diff --git a/mobile/index.html b/mobile/index.html index e8f1d71..5bcf3fa 100644 --- a/mobile/index.html +++ b/mobile/index.html @@ -647,8 +647,14 @@ function renderAsk() { const a = S.approvals.find((x) => x.bot === S.open); if (!a) { $("ask").innerHTML = ""; return; } if (S.role === "member") { $("ask").innerHTML = `
${esc(a.title)}
等擁有者或管理員決定
`; return; } + // 需要處理 (a step only you can do on a computer): its screen is one tap away, then 我完成了 / 跳過. + if (a.kind === "handoff") { + $("ask").innerHTML = `
需要處理 · ${esc(a.computer || "")}
${esc(a.reason)}
`; + $("ask").querySelector("[data-screen]").onclick = () => { S.back = S.view; show("screen"); $("seg").children[0].textContent = ownComputer(); $("seg").children[1].style.display = S.mac ? "" : "none"; startFrames(); }; + } else { $("ask").innerHTML = `
${esc(a.title)}
${esc(a.reason)}
${a.can_always ? `` : ""}
`; - $("ask").querySelectorAll("button").forEach((b) => b.onclick = async () => { + } + $("ask").querySelectorAll("button[data-a]").forEach((b) => b.onclick = async () => { $("ask").innerHTML = ""; try { await api(base() + `api/bots/${S.open}/answer`, {json: {answer: b.dataset.a}}); } catch (e) { toast(e.message); } }); diff --git a/src/agent.rs b/src/agent.rs index 2733b65..0c31819 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -515,6 +515,46 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< send(StreamEvent::NeedApproval(Approval { call_id: call.id.clone(), tool: call.name.clone(), reason, grant, reply: atx, kind })); wait_answer(arx, &cancel) }; + // 需要處理 (the original's box hand-off): a step only the person can do — signing + // in, a captcha, a code from their phone. A card with the computer's live screen + // appears in the chat; they take over, do it, and say 「我完成了」 or 「跳過」. A + // message typed meanwhile answers it too. + if call.name == tools::HANDOFF_TOOL { + let instruction = tools::args_of(&call.args)["instruction"].as_str().unwrap_or("").trim().to_string(); + let (output, status) = match screen_target(&job, &call.args) { + Err(e) => (e, ToolStatus::Failed), + Ok(_) if instruction.is_empty() => ("請在 instruction 寫清楚要使用者做什麼。".to_string(), ToolStatus::Failed), + Ok(t) => { + let (atx, arx) = channel(); + send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Waiting)); + send(StreamEvent::NeedApproval(Approval { call_id: call.id.clone(), tool: call.name.clone(), reason: instruction.clone(), grant: t.name.clone(), reply: atx, kind: ApprovalKind::Handoff })); + let answer = loop { + if cancel.load(Ordering::Relaxed) { + return; + } + match arx.recv_timeout(Duration::from_millis(200)) { + Ok(a) => break Some(a), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + if !job.steer.lock().unwrap().is_empty() { + break None; + } + } + Err(_) => break Some(ApprovalAnswer::Reject), + } + }; + match answer { + Some(ApprovalAnswer::Reject) => ("使用者選擇跳過這一步。不要再要求同一件事：沒有它就繼續其餘的工作，做不到的照實說。".to_string(), ToolStatus::Denied), + Some(_) => (format!("[{}] 使用者按了「我完成了」。先截圖確認畫面，再接著做。", t.name), ToolStatus::Done), + None => ("使用者沒有按按鈕，而是傳了訊息（就在下面）。照他說的做。".to_string(), ToolStatus::Done), + } + } + }; + send(StreamEvent::ToolStatus(call.id.clone(), status)); + let result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &output)); + send(StreamEvent::ToolResult(result.clone())); + history.push(result); + continue; + } // 生活卡片: drawn by the window from the call itself. if call.name == "show_card" { send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Done)); @@ -1374,6 +1414,71 @@ mod tests { assert!(result.contains("note.txt"), "{result}"); } + /// 需要處理: the model hands a step to the person; the card's request carries the instruction + /// and the computer, 「我完成了」 tells the bot to carry on, 「跳過」 that the step is skipped. + #[test] + fn a_hand_off_waits_for_the_person_and_says_what_they_chose() { + use std::io::{Read, Write}; + use std::net::TcpListener; + for (answer, expect, status) in [(ApprovalAnswer::Once, "我完成了", ToolStatus::Done), (ApprovalAnswer::Reject, "跳過", ToolStatus::Denied)] { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = l.local_addr().unwrap().port(); + std::thread::spawn(move || { + let replies = [ + "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"h1\",\"function\":{\"name\":\"hand_over_computer\",\"arguments\":\"{\\\"instruction\\\":\\\"請登入 Threads\\\"}\"}}]}}]}\n\ndata: [DONE]\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"好\"}}]}\n\ndata: [DONE]\n\n", + ]; + for body in replies { + let (mut s, _) = l.accept().unwrap(); + let mut req = Vec::new(); + let mut buf = [0u8; 8192]; + loop { + let n = s.read(&mut buf).unwrap_or(0); + if n == 0 { + break; + } + req.extend_from_slice(&buf[..n]); + let t = String::from_utf8_lossy(&req).to_string(); + if let Some(end) = t.find("\r\n\r\n") { + let len = t[..end].lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::().unwrap_or(0))).unwrap_or(0); + if req.len() >= end + 4 + len { + break; + } + } + } + let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n"); + let _ = s.write_all(body.as_bytes()); + } + }); + let job = Job { + req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("幫我登入 Threads")], tools: vec![], session: String::new() }, + computers: vec![tools::Target { name: "王小明的Mac mini".into(), place: tools::Place::Device { host: "m".into() } }], + default_computer: "王小明的Mac mini".into(), + roles: tools::Roles { own: "Master 的電腦".into(), person: Some("王小明的Mac mini".into()), server: true }, + ..test_job() + }; + let (tx, rx) = channel(); + spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default()); + let (mut asked, mut result, mut last) = (None, String::new(), None); + loop { + match rx.recv_timeout(Duration::from_secs(10)).expect("loop stalled") { + (_, StreamEvent::NeedApproval(a)) => { + asked = Some((a.kind, a.reason.clone(), a.grant.clone())); + let _ = a.reply.send(answer.clone()); + } + (_, StreamEvent::ToolStatus(_, st)) => last = Some(st), + (_, StreamEvent::ToolResult(m)) => result = m.text, + (_, StreamEvent::Done) => break, + (_, StreamEvent::Error(e)) => panic!("{e}"), + _ => {} + } + } + assert_eq!(asked, Some((ApprovalKind::Handoff, "請登入 Threads".to_string(), "王小明的Mac mini".to_string()))); + assert!(result.contains(expect), "{result}"); + assert_eq!(last, Some(status)); + } + } + /// 直接操作: tools off, this Mac off, a command that changes files — and still no question. /// Dangerous commands stay blocked. #[test] diff --git a/src/app.rs b/src/app.rs index 86b7144..10fb1a8 100644 --- a/src/app.rs +++ b/src/app.rs @@ -44,6 +44,9 @@ pub const SIDEBAR_DEFAULT_WIDTH: f32 = 280.0; pub const SIDEBAR_MIN_WIDTH: f32 = 240.0; pub const SIDEBAR_MAX_WIDTH: f32 = 400.0; pub const SIDEBAR_RAIL_WIDTH: f32 = 88.0; +/// Mac: the window has no title bar of its own (the content reaches the top); this strip at the +/// top moves the window, and the traffic lights sit in its left corner. +pub const TITLEBAR_H: f32 = 28.0; pub const SIDEBAR_COLLAPSE_BELOW: f32 = 210.0; pub const SIDEBAR_EXPAND_FROM: f32 = 220.0; /// `--sand-chat-min-width:424px` (index CSS, class sand-1fiktjy). @@ -491,6 +494,13 @@ impl App { } } } + // 需要處理: the bot waits for the person on its own computer (the hand-off card). + Ok("handoff") => { + app.state.selected = 1; + let (reply, _) = channel(); + let own = app.state.bot(1).map(|b| crate::model::own_computer_name(&b.name)).unwrap_or_default(); + app.approvals.insert(1, Approval { call_id: "h1".into(), tool: tools::HANDOFF_TOOL.into(), reason: "請在 Threads 登入頁輸入帳號和密碼後登入，好了按「我完成了」。".into(), grant: own, reply, kind: ApprovalKind::Handoff }); + } Ok("panel") => { // The real box, so the 電腦 card shows its screen. crate::cloudbox::start_background(); @@ -550,6 +560,13 @@ impl App { app.show_settings = true; app.settings_tab = SettingsTab::Remote; } + // …or the side panel, or the computer viewer, as they look with that data. + if app.screenshot.is_some() && !app.demo { + match std::env::var("ANYBOT_SHOT_VIEW").as_deref() { + Ok("panel") => app.panel = Some(panel_ui::PanelTab::Activity), + _ => {} + } + } app } @@ -756,6 +773,13 @@ impl App { if !screens.is_empty() { let default_screen = if screens.contains(&default_computer) { default_computer.clone() } else { screens[0].clone() }; tool_specs.extend(crate::computer::specs_for(&screens, &default_screen)); + // 需要處理: hand a step to the person on that screen (the original's box hand-off). + tool_specs.push(serde_json::json!({"name": tools::HANDOFF_TOOL, + "description": "Ask the person to do one step themselves on a computer's screen: signing in (you never type their passwords), a captcha or puzzle, a code sent to their phone or email, approving a payment, or anything else only they may do. First get the computer to that point (open the page, click into the sign-in form), then call this with a short instruction. A card with that computer's live screen appears in the chat; they take over, do it, and press 「我完成了」 (or 「跳過」). The result says which; after 「我完成了」 take a screenshot and carry on. Don't ask them in words to find a window themselves.", + "parameters": {"type": "object", "properties": { + "instruction": {"type": "string", "description": "What they should do, one or two short sentences in their language, e.g. 「請在 Threads 登入頁輸入帳號和密碼後登入」."}, + "computer": {"type": "string", "enum": screens.clone(), "description": format!("Which computer's screen. Default: {default_screen}.")}}, + "required": ["instruction"]}})); } // 看圖小幫手: the person's models that may see pictures, for a bot whose model can't. providers::set_eyes(self.eyes()); @@ -1293,6 +1317,7 @@ impl App { } } ApprovalKind::EnableThisComputer => self.state.local_execution = true, + ApprovalKind::Handoff => {} ApprovalKind::Action => { if let Some(b) = self.state.bot_mut(bot_id) { if !g.is_empty() && !b.allow_rules.contains(g) { @@ -1428,6 +1453,10 @@ impl App { if let Some(c) = b.messages.iter_mut().rev().flat_map(|m| m.tool_calls.iter_mut()).find(|c| c.id == call_id) { c.status = st; } + // The step moved on without the buttons (a message answered a hand-off). + if st != ToolStatus::Waiting && self.approvals.get(&id).map(|a| a.call_id == call_id).unwrap_or(false) { + self.approvals.remove(&id); + } } other => { let Some(m) = b.messages.last_mut() else { continue }; @@ -1734,6 +1763,10 @@ impl App { fn rail(&mut self, ui: &mut egui::Ui) { let p = self.pal; ui.vertical_centered(|ui| { + // (The Mac's window buttons sit in the top corner.) + if cfg!(target_os = "macos") { + ui.add_space(TITLEBAR_H - 6.0); + } if icon_button(ui, "◨", "展開側邊欄", p).clicked() { self.state.sidebar_collapsed = false; self.mark(); @@ -2020,9 +2053,11 @@ impl App { self.voice_toggle(&ctx, bot.id); } } - if self.approvals.contains_key(&bot.id) { - if crate::icons::pill(&mut right, crate::icons::Icon::Shield, &t("需要你允許"), p.accent, Color32::WHITE, 36.0).on_hover_text(t("打開請求視窗")).clicked() { + if let Some(kind) = self.approvals.get(&bot.id).map(|a| a.kind) { + let label = if kind == ApprovalKind::Handoff { t("等你處理") } else { t("需要你允許") }; + if crate::icons::pill(&mut right, crate::icons::Icon::Shield, &label, p.accent, Color32::WHITE, 36.0).on_hover_text(t("打開請求視窗")).clicked() { self.popup_hidden.remove(&bot.id); + self.stick_bottom = true; } } let right_w = bar.right() - right.min_rect().left(); @@ -2096,11 +2131,12 @@ impl App { ApprovalKind::EnableThisComputer => format!("允許 {} 在你的電腦上執行任務嗎？", bot.name), ApprovalKind::Action if machine.is_empty() => format!("允許 {} {}嗎？", bot.name, what), ApprovalKind::Action => format!("允許 {} 在 {machine} 上{what}嗎？", bot.name), + ApprovalKind::Handoff => format!("{}：需要你", bot.name), }; let note = match kind { ApprovalKind::EnableTools => Some("這個 Bot 的電腦工具還沒打開。它想做的事："), ApprovalKind::EnableThisComputer => Some("「在這台電腦上執行」還沒打開，Bot 平常只在它自己的電腦上做事。它想在你的電腦上："), - ApprovalKind::Action => None, + ApprovalKind::Action | ApprovalKind::Handoff => None, }; let mut answer = None; ui.label(RichText::new(title).strong().size(if big { 17.0 } else { 14.0 })); @@ -2128,7 +2164,7 @@ impl App { ApprovalKind::EnableTools => Some("一律允許這個 Bot 使用電腦".to_string()), ApprovalKind::EnableThisComputer => Some("一律允許（打開「在這台電腦上執行」）".to_string()), ApprovalKind::Action if !grant.is_empty() => Some(format!("一律允許{}", grant_label(&grant))), - ApprovalKind::Action => None, + ApprovalKind::Action | ApprovalKind::Handoff => None, }; if let Some(label) = always { if ui.button(label).clicked() { @@ -2153,7 +2189,9 @@ impl App { fn approval_card(&mut self, ui: &mut egui::Ui, ctx: &egui::Context, bot: &Bot) { let p = self.pal; - if !self.approvals.contains_key(&bot.id) { + let Some(kind) = self.approvals.get(&bot.id).map(|a| a.kind) else { return }; + if kind == ApprovalKind::Handoff { + self.handoff_card(ui, ctx, bot); return; } let mut answer = None; @@ -2173,6 +2211,91 @@ impl App { } } + /// 需要處理 (the original's box hand-off card): what the bot needs the person to do, the + /// computer's live screen with 「接管」 on it, and 跳過 / 我完成了 / 接管. + fn handoff_card(&mut self, ui: &mut egui::Ui, ctx: &egui::Context, bot: &Bot) { + let p = self.pal; + let Some(a) = self.approvals.get(&bot.id) else { return }; + let (instruction, computer) = (a.reason.clone(), a.grant.clone()); + // Its own computer shows here (on a screen of a brain, the brain's); another computer of + // the person's is the one in front of them. + let own = computer == crate::model::own_computer_name(&bot.name) || computer == crate::model::BOT_COMPUTER_NAME; + let shows = own && (self.mir.on || crate::cloudbox::available()); + let thumb = if shows { self.box_thumbnail(ctx) } else { None }; + let width = (ui.available_width() - 32.0 - 28.0).clamp(200.0, 560.0); + enum Act { + Skip, + Done, + TakeOver, + } + let mut act = None; + ui.horizontal_top(|ui| { + ui.add_space(32.0); + Frame::new().fill(p.elevated).stroke(Stroke::new(2.0, p.accent)).corner_radius(14).inner_margin(Margin::symmetric(14, 12)).show(ui, |ui| { + ui.vertical(|ui| { + ui.set_width(width); + ui.horizontal(|ui| { + crate::icons::label(ui, crate::icons::Icon::Monitor, &t("電腦"), FONT_XS, p.text3); + ui.label(RichText::new(sidebar_ui::shorten_width(ui, &format!("· {computer}"), (ui.available_width() - 4.0).max(40.0), FONT_XS)).size(FONT_XS).color(p.text3)); + }); + ui.add_space(2.0); + ui.label(RichText::new(t("需要處理")).strong().size(15.0)); + ui.add(egui::Label::new(RichText::new(&instruction).size(13.5).color(p.text2)).wrap()); + ui.add_space(8.0); + // The screen, 16:10, with 「接管」 on it. + let sw = width.min(420.0); + let (rect, r) = ui.allocate_exact_size(vec2(sw, sw * 0.625), if shows { egui::Sense::click() } else { egui::Sense::hover() }); + match thumb { + Some((tex, size)) => { + egui::Image::new((tex, size)).fit_to_exact_size(rect.size()).corner_radius(10).paint_at(ui, rect); + } + None => { + ui.painter().rect_filled(rect, 10, Color32::from_rgb(0x2b, 0x2f, 0x36)); + let text = if shows { t("連結中") } else { t("請到「{name}」上操作").replace("{name}", &computer) }; + ui.painter().text(rect.center(), egui::Align2::CENTER_CENTER, sidebar_ui::shorten_width(ui, &text, rect.width() - 20.0, FONT_BASE), egui::FontId::proportional(FONT_BASE), Color32::from_gray(220)); + } + } + ui.painter().rect_stroke(rect, 10, Stroke::new(1.0, p.border), egui::StrokeKind::Inside); + if shows { + let label = t("接管"); + let g = ui.painter().layout_no_wrap(label.clone(), egui::FontId::proportional(13.0), Color32::WHITE); + let pill = egui::Rect::from_min_size(rect.right_bottom() - vec2(g.size().x + 28.0, 38.0), vec2(g.size().x + 20.0, 28.0)); + ui.painter().rect_filled(pill, 14, Color32::from_black_alpha(if r.hovered() { 200 } else { 160 })); + ui.painter().galley(pill.center() - g.size() / 2.0, g, Color32::WHITE); + if r.on_hover_cursor(egui::CursorIcon::PointingHand).on_hover_text(t("接管電腦")).clicked() { + act = Some(Act::TakeOver); + } + } + ui.add_space(10.0); + ui.horizontal(|ui| { + if ui.button(t("跳過")).on_hover_text(t("取消此要求且不執行該步驟；Bot 會跳過它並繼續")).clicked() { + act = Some(Act::Skip); + } + if ui.button(t("我完成了")).clicked() { + act = Some(Act::Done); + } + if shows && ui.add(egui::Button::new(RichText::new(t("接管")).color(p.on_primary)).fill(p.primary).corner_radius(8)).clicked() { + act = Some(Act::TakeOver); + } + }); + }); + }); + }); + ui.add_space(10.0); + match act { + Some(Act::Skip) => { + self.give_back_computer(); + self.answer(ctx, bot.id, ApprovalAnswer::Reject); + } + Some(Act::Done) => { + self.give_back_computer(); + self.answer(ctx, bot.id, ApprovalAnswer::Once); + } + Some(Act::TakeOver) => self.take_over_computer(), + None => {} + } + } + /// A new request: show it as a pop-up (even over another bot's chat) and get the person's /// attention (the Dock icon bounces when AnyBot isn't in front). fn on_new_approval(&mut self, ctx: &egui::Context, bot_id: u64) { @@ -2183,10 +2306,12 @@ impl App { if self.state.notify && self.away() { if let (Some(b), Some(a)) = (self.state.bot(bot_id), self.approvals.get(&bot_id)) { let what = a.reason.split('\n').next().unwrap_or("").to_string(); + // 「{callee} 需要你」 for a step only the person can do. + let title = if a.kind == ApprovalKind::Handoff { format!("{}：需要你", b.name) } else { format!("{} 需要你允許", b.name) }; if !self.headless { - notify::banner(&format!("{} 需要你允許", b.name), &what); + notify::banner(&title, &what); } - crate::remote::notify(&format!("{} 需要你允許", b.name), &what, bot_id); + crate::remote::notify(&title, &what, bot_id); } } } @@ -2213,7 +2338,8 @@ impl App { /// The pop-up: a dimmed window with the request in the middle, so it can't be missed. fn approval_popup(&mut self, ctx: &egui::Context) { let selected = self.state.selected; - let mut ids: Vec = self.approvals.keys().copied().filter(|id| !self.popup_hidden.contains(id)).collect(); + // (A hand-off is the card in the chat, with the live screen, like the original's.) + let mut ids: Vec = self.approvals.iter().filter(|(id, a)| !self.popup_hidden.contains(id) && a.kind != ApprovalKind::Handoff).map(|(id, _)| *id).collect(); ids.sort_by_key(|id| (*id != selected, *id)); let Some(&bot_id) = ids.first() else { return }; let Some(bot) = self.state.bot(bot_id).cloned() else { return }; @@ -3291,6 +3417,11 @@ impl App { ctx.send_viewport_cmd(egui::ViewportCommand::InnerSize(vec2(w, h))); } } + // Real data: the computer viewer opens a little after start, as a person would open it (a + // screen of a brain has said so by then, so it shows the brain's computer). + if self.frames == 10 && !self.demo && std::env::var("ANYBOT_SHOT_VIEW").as_deref() == Ok("viewer") { + self.open_viewer(); + } // ANYBOT_SHOT_PAGE=id: open that page late (a screen gets its pages from the brain first). if self.frames == need * 3 / 4 { if let Some(id) = std::env::var("ANYBOT_SHOT_PAGE").ok().and_then(|v| v.parse().ok()) { @@ -3605,6 +3736,20 @@ impl eframe::App for App { if ctx.input(|i| i.modifiers.command && i.key_pressed(Key::Comma)) { self.show_settings = true; } + // Mac: the top strip moves the window like a title bar would (buttons drawn over it still + // take their clicks: they come later, so they are on top); a double click zooms. + if cfg!(target_os = "macos") { + let full = ui.max_rect(); + let strip = egui::Rect::from_min_max(full.min, egui::pos2(full.max.x, full.min.y + TITLEBAR_H)); + let r = ui.interact(strip, egui::Id::unique("titlebar-drag"), egui::Sense::click_and_drag()); + if r.drag_started_by(egui::PointerButton::Primary) { + ctx.send_viewport_cmd(egui::ViewportCommand::StartDrag); + } + if r.double_clicked() { + let zoomed = ctx.input(|i| i.viewport().maximized.unwrap_or(false)); + ctx.send_viewport_cmd(egui::ViewportCommand::Maximized(!zoomed)); + } + } self.sidebar(ui); self.main_pane(ui); self.bot_editor(ctx); @@ -3886,8 +4031,9 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu - 公開網頁：web_fetch。\n\ - 要登入才看得到的（Threads、Instagram、Facebook、Gmail、銀行…）：用 browser_open 在 AnyBot 自己的瀏覽器視窗打開，再用 browser_read 讀分頁內容。例如看 Threads 帳號：browser_open(\"https://www.threads.net\") 然後 browser_read。\n\ - 在網頁上做事（後台、報表、表單）要像人一樣操作：先 browser_screenshot 看一眼（圖上每個可點的東西都有編號），再用 browser_click／browser_type／browser_select／browser_press 照編號操作；表格用 browser_table 讀，找字用 browser_find，要等載入用 browser_wait，匯出報表用 browser_download。不要猜網址；找不到想要的功能（例如排序、篩選），就看欄位標題、篩選器、下拉選單或分頁標籤，還是沒有就照頁面現有的資料自己整理，並說明網站沒有這個功能。 - - 絕對不要直接呼叫網站的內部 API、下載網站的程式碼來研究，或用程式大量抓資料：網站會把帳號當成機器人（例如跳出拼圖驗證、封鎖帳號）。出現驗證碼、拼圖或簡訊驗證時，停下來請使用者在 AnyBot 瀏覽器視窗完成。 - - 如果讀到的是登入頁，請使用者在跳出來的 AnyBot 瀏覽器視窗登入一次（之後會記住），登入好再讀。絕對不要叫使用者去改瀏覽器的設定或選單（例如「允許來自 Apple 事件的 JavaScript」），那不需要。也不要說要從 Chrome 複製登入或 Cookie，沒有這個功能。\n\ + - 絕對不要直接呼叫網站的內部 API、下載網站的程式碼來研究，或用程式大量抓資料：網站會把帳號當成機器人（例如跳出拼圖驗證、封鎖帳號）。 + - 需要使用者親手做的步驟（登入、輸入密碼、驗證碼、拼圖、簡訊或 email 驗證、確認付款）：先在電腦上把那一頁打開、點到要輸入的地方，再呼叫 hand_over_computer，instruction 寫清楚要他做什麼。聊天裡會出現那台電腦的即時畫面，他點「接管」做完後按「我完成了」，你再截圖確認、接著做。不要只用文字叫他自己去找視窗；也絕對不要替他打密碼。 + - 如果讀到的是登入頁：把登入頁開在那台電腦上，用 hand_over_computer 請使用者登入一次（之後會記住），他按「我完成了」再讀。絕對不要叫使用者去改瀏覽器的設定或選單（例如「允許來自 Apple 事件的 JavaScript」），那不需要。也不要說要從 Chrome 複製登入或 Cookie，沒有這個功能。\n\ - AnyBot 瀏覽器有設定檔（profile），每個有自己的登入。使用者說要換 profile／設定檔／另一個帳號時，用 browser_profiles：list 看有哪些，use 切換，add 新增（新的要登入一次）。也可以在 browser_open / browser_read 帶 profile 參數。使用者想用自己平常的 Chrome（自己的 Chrome 設定檔、已經登入的帳號）時，用 profile「你的瀏覽器」：請使用者在 Chrome 切到想要的設定檔，再 browser_read。\n\ - 你也能像人一樣操作螢幕（{screens}）。computer_use_screenshot 看畫面，再用 computer_use_click / computer_use_typing / computer_use_press_key / computer_use_scroll / computer_use_drag 操作；座標用最新截圖上的像素。每做一步就再截圖確認。用在檔案和網頁工具做不到的事（例如操作其他 App、在 Chrome 切換設定檔）。\n\ {}\ diff --git a/src/app/computers_ui.rs b/src/app/computers_ui.rs index 9c97895..8ef0f23 100644 --- a/src/app/computers_ui.rs +++ b/src/app/computers_ui.rs @@ -977,6 +977,40 @@ impl App { } } + /// 接管: open the computer and take control (the bot waits for the person anyway). + pub(super) fn take_over_computer(&mut self) { + self.open_viewer(); + if self.cu.control { + return; + } + self.cu.control = true; + match self.cu.remote.clone() { + Some(s) => { + std::thread::spawn(move || { + let _ = crate::remote::sync_post(&s, "screen", &serde_json::json!({"kind": "hold", "on": true})); + }); + } + None => cloudbox::HELD_BY_PERSON.store(true, Ordering::SeqCst), + } + } + + /// 我完成了 / 跳過: the bot gets its computer back (its steps would otherwise wait for the + /// person to let go). + pub(super) fn give_back_computer(&mut self) { + if !self.cu.control { + return; + } + self.cu.control = false; + match self.cu.remote.clone() { + Some(s) => { + std::thread::spawn(move || { + let _ = crate::remote::sync_post(&s, "screen", &serde_json::json!({"kind": "hold", "on": false})); + }); + } + None => cloudbox::HELD_BY_PERSON.store(false, Ordering::SeqCst), + } + } + fn close_viewer(&mut self) { self.cu.viewer_open = false; self.cu.control = false; @@ -1116,8 +1150,11 @@ impl App { let default = vec2((screen.width() * 0.62).min(900.0), (screen.width() * 0.62).min(900.0) * 0.625 + 52.0); let busy = if brain { self.mir.box_state["busy"].as_bool().unwrap_or(false) } else { cloudbox::is_busy() }; let title = self.state.bot(self.state.selected).map(|b| crate::model::own_computer_name(&b.name)).unwrap_or_else(|| crate::model::BOT_COMPUTER_NAME.to_string()); - let title = if brain { format!("{title}（在大腦「{}」上）", self.state.remote.brain_name) } else { title }; + let mut close_clicked = false; + let mut done_clicked = false; + let selected = self.state.selected; + let handoff = self.approvals.get(&selected).map(|a| a.kind == crate::providers::ApprovalKind::Handoff).unwrap_or(false); egui::Window::new("box-viewer") .id(egui::Id::unique("box-viewer")) .title_bar(false) @@ -1135,16 +1172,9 @@ impl App { .shadow(egui::epaint::Shadow { offset: [0, 14], blur: 40, spread: 0, color: Color32::from_black_alpha(60) }), ) .show(ctx, |ui| { + // Buttons first, from the right; the title and state get what is left and end in + // … when it's short (the state used to slide under 「取得控制權」 in a narrow window). ui.horizontal(|ui| { - monitor_icon(ui, 20.0, p.text); - ui.label(RichText::new(&title).strong().size(FONT_LG)); - status_pill(ui, &run, p); - if busy && !self.cu.control { - crate::icons::label(ui, crate::icons::Icon::Bot, &t("Bot 正在操作"), FONT_XS, p.accent); - } - if self.cu.control { - crate::icons::label(ui, crate::icons::Icon::Users, &t("你正在操作，Bot 已暫停"), FONT_XS, p.accent); - } ui.with_layout(Layout::right_to_left(Align::Center), |ui| { let (rect, r) = ui.allocate_exact_size(vec2(28.0, 28.0), egui::Sense::click()); if r.hovered() { @@ -1158,6 +1188,10 @@ impl App { close_clicked = true; } ui.add_space(4.0); + // The bot is waiting for the person here: done, and it carries on. + if handoff && ui.add(egui::Button::new(RichText::new(t("我完成了")).strong().color(p.on_primary)).fill(p.primary).corner_radius(8).min_size(vec2(84.0, 30.0))).clicked() { + done_clicked = true; + } if !brain && ui.add(secondary_button("全螢幕")).on_hover_text(t("在瀏覽器用 noVNC 全螢幕開啟")).clicked() { let control = self.cu.control; self.box_task(move || { @@ -1184,6 +1218,19 @@ impl App { None => cloudbox::HELD_BY_PERSON.store(self.cu.control, Ordering::SeqCst), } } + ui.add_space(8.0); + ui.with_layout(Layout::left_to_right(Align::Center), |ui| { + monitor_icon(ui, 20.0, p.text); + let note = if self.cu.control { Some(t("你正在操作，Bot 已暫停")) } else if busy { Some(t("Bot 正在操作")) } else { None }; + let room = (ui.available_width() - 96.0 - if note.is_some() { 120.0 } else { 0.0 }).max(60.0); + ui.label(RichText::new(super::sidebar_ui::shorten_width(ui, &title, room, FONT_LG)).strong().size(FONT_LG)); + if ui.available_width() > 80.0 { + status_pill(ui, &run, p); + } + if let Some(n) = note.filter(|_| ui.available_width() > 110.0) { + crate::icons::label(ui, if self.cu.control { crate::icons::Icon::Users } else { crate::icons::Icon::Bot }, &n, FONT_XS, p.accent); + } + }); }); }); ui.add_space(6.0); @@ -1200,7 +1247,17 @@ impl App { }; egui::ScrollArea::both().auto_shrink([false, false]).show(ui, |ui| { let sense = if self.cu.control { egui::Sense::click_and_drag() } else { egui::Sense::hover() }; - let resp = ui.add(egui::Image::new((tex_id, size)).corner_radius(8).sense(sense)); + // Centred in the window (it sat at the left with an empty band beside it). + let resp = if self.cu.fit { + let pad = ((ui.available_width() - size.x) / 2.0).max(0.0); + ui.horizontal(|ui| { + ui.add_space(pad); + ui.add(egui::Image::new((tex_id, size)).corner_radius(8).sense(sense)) + }) + .inner + } else { + ui.add(egui::Image::new((tex_id, size)).corner_radius(8).sense(sense)) + }; if self.cu.control { ui.painter().rect_stroke(resp.rect, 8, Stroke::new(2.0, p.accent), egui::StrokeKind::Outside); self.forward_input(ui, &resp, (fw, fh)); @@ -1233,7 +1290,11 @@ impl App { } } }); - if close_clicked || !open { + if done_clicked { + self.give_back_computer(); + self.answer(ctx, selected, crate::providers::ApprovalAnswer::Once); + self.close_viewer(); + } else if close_clicked || !open { self.close_viewer(); } ctx.request_repaint_after(std::time::Duration::from_millis(VIEWER_FRAME_MS)); diff --git a/src/app/mirror_ui.rs b/src/app/mirror_ui.rs index 6708af2..db17f4c 100644 --- a/src/app/mirror_ui.rs +++ b/src/app/mirror_ui.rs @@ -533,6 +533,7 @@ impl App { ApprovalKind::Action => "action", ApprovalKind::EnableTools => "tools", ApprovalKind::EnableThisComputer => "local", + ApprovalKind::Handoff => "handoff", }; json!({"bot": id, "call_id": a.call_id, "tool": a.tool, "reason": a.reason, "grant": a.grant, "kind": kind}) }) @@ -1015,6 +1016,7 @@ impl App { let kind = match a["kind"].as_str().unwrap_or("") { "tools" => ApprovalKind::EnableTools, "local" => ApprovalKind::EnableThisComputer, + "handoff" => ApprovalKind::Handoff, _ => ApprovalKind::Action, }; let (reply, _) = channel(); diff --git a/src/app/panel_ui.rs b/src/app/panel_ui.rs index aac19fa..df01a27 100644 --- a/src/app/panel_ui.rs +++ b/src/app/panel_ui.rs @@ -105,7 +105,8 @@ impl App { let status = self.streams.get(&bot.id).and_then(|l| l.status.clone()).unwrap_or_else(|| t("工作中…")); let mut clicked = false; panel_card(ui, p, |ui| { - clicked = if waiting { panel_row(ui, p, Some(crate::icons::Icon::Shield), &t("需要你允許"), "", true).clicked() } else { panel_row(ui, p, Some(crate::icons::Icon::Stop), &status, &t("停止"), false).clicked() }; + let label = if self.approvals.get(&bot.id).map(|a| a.kind == crate::providers::ApprovalKind::Handoff).unwrap_or(false) { t("等你處理") } else { t("需要你允許") }; + clicked = if waiting { panel_row(ui, p, Some(crate::icons::Icon::Shield), &label, "", true).clicked() } else { panel_row(ui, p, Some(crate::icons::Icon::Stop), &status, &t("停止"), false).clicked() }; }); if clicked { if waiting { @@ -173,21 +174,28 @@ impl App { let run = crate::cloudbox::run_state(); let running = if brain { self.mir.box_state["running"].as_bool().unwrap_or(false) } else { run.is_running() }; let state = if brain { - let label = self.mir.box_state["label"].as_str().map(t).unwrap_or_else(|| t("連線中…")); - format!("{label} · {}", t("在大腦「{name}」上").replace("{name}", &brain_name)) + self.mir.box_state["label"].as_str().map(t).unwrap_or_else(|| t("連線中…")) } else if available { t(&run.label()) } else { t("沒有 Docker") }; - let (rect, r) = ui.allocate_exact_size(vec2(ui.available_width(), 64.0), egui::Sense::click()); + // Where it is, on its own line (it ran into the picture when it shared the state's). + let place = brain.then(|| t("在大腦「{name}」上").replace("{name}", &brain_name)); + let (rect, r) = ui.allocate_exact_size(vec2(ui.available_width(), if place.is_some() { 72.0 } else { 64.0 }), egui::Sense::click()); if r.hovered() { ui.painter().rect_filled(rect, 9, p.hover); } - ui.painter().text(rect.left_top() + vec2(10.0, 15.0), egui::Align2::LEFT_TOP, &own_name, egui::FontId::proportional(13.0), p.text); - ui.painter().text(rect.left_top() + vec2(10.0, 33.0), egui::Align2::LEFT_TOP, state, egui::FontId::proportional(12.0), p.text3); + // Every line stops before the picture (long names end in …). + let text_w = (rect.width() - 86.0 - 10.0 - 8.0).max(40.0); + let top = if place.is_some() { 9.0 } else { 15.0 }; + ui.painter().text(rect.left_top() + vec2(10.0, top), egui::Align2::LEFT_TOP, super::sidebar_ui::shorten_width(ui, &own_name, text_w, 13.0), egui::FontId::proportional(13.0), p.text); + ui.painter().text(rect.left_top() + vec2(10.0, top + 18.0), egui::Align2::LEFT_TOP, super::sidebar_ui::shorten_width(ui, &state, text_w, 12.0), egui::FontId::proportional(12.0), p.text3); + if let Some(place) = &place { + ui.painter().text(rect.left_top() + vec2(10.0, top + 35.0), egui::Align2::LEFT_TOP, super::sidebar_ui::shorten_width(ui, place, text_w, 12.0), egui::FontId::proportional(12.0), p.text3); + } // The screen: 16:10, rounded, the real desktop when there is a picture. - let box_r = egui::Rect::from_min_size(rect.right_top() + vec2(-86.0, 8.0), vec2(78.0, 48.0)); + let box_r = egui::Rect::from_min_size(egui::pos2(rect.right() - 86.0, rect.center().y - 24.0), vec2(78.0, 48.0)); match thumb { Some((tex, size)) => { let mut img = egui::Image::new((tex, size)).fit_to_exact_size(box_r.size()).corner_radius(7); @@ -211,7 +219,8 @@ impl App { let tip = if running { t("看「{name}」的螢幕，也可以取得控制權自己操作") } else { t("叫醒「{name}」並看它的螢幕") }.replace("{name}", &own_name); view = r.on_hover_cursor(egui::CursorIcon::PointingHand).on_hover_text(tip).clicked(); if !choices.is_empty() { - panel_select(ui, p, "做事的電腦", ("computer", bot.id), ¤t, |ui| { + let shown = super::sidebar_ui::shorten_width(ui, ¤t, (ui.available_width() - 120.0).max(60.0), 13.0); + panel_select(ui, p, "做事的電腦", ("computer", bot.id), &shown, |ui| { for (name, kind) in &choices { let what = t(match kind.as_str() { "own" => "它自己的電腦", diff --git a/src/app/remote_ui.rs b/src/app/remote_ui.rs index d1c3a20..7585c6b 100644 --- a/src/app/remote_ui.rs +++ b/src/app/remote_ui.rs @@ -41,8 +41,10 @@ fn approval_json(bot: &Bot, a: &Approval) -> serde_json::Value { ApprovalKind::EnableTools => format!("允許 {} 使用電腦工具嗎？", bot.name), ApprovalKind::EnableThisComputer => format!("允許 {} 在你的電腦上執行任務嗎？", bot.name), ApprovalKind::Action => format!("{} 想要{what}", bot.name), + ApprovalKind::Handoff => format!("{}：需要你", bot.name), }; - json!({"bot": bot.id, "title": title, "reason": a.reason, "tool": a.tool, "can_always": a.kind != ApprovalKind::Action || !a.grant.is_empty()}) + let handoff = a.kind == ApprovalKind::Handoff; + json!({"bot": bot.id, "title": title, "reason": a.reason, "tool": a.tool, "can_always": !handoff && (a.kind != ApprovalKind::Action || !a.grant.is_empty()), "kind": if handoff { "handoff" } else { "ask" }, "computer": if handoff { a.grant.as_str() } else { "" }}) } impl App { diff --git a/src/app/sidebar_ui.rs b/src/app/sidebar_ui.rs index 30d0472..0ebf064 100644 --- a/src/app/sidebar_ui.rs +++ b/src/app/sidebar_ui.rs @@ -745,7 +745,7 @@ impl App { }; painter.galley(egui::pos2(tx, rect.top() + 9.0), one_line(b.name.clone(), 15.0, p.text), p.text); let sub = if waiting { - t("需要你允許") + if self.approvals.get(&b.id).map(|a| a.kind == crate::providers::ApprovalKind::Handoff).unwrap_or(false) { t("等你處理") } else { t("需要你允許") } } else if streaming { t("正在工作…") } else { diff --git a/src/i18n_table.rs b/src/i18n_table.rs index 43e08a1..e98af91 100644 --- a/src/i18n_table.rs +++ b/src/i18n_table.rs @@ -614,4 +614,13 @@ pub const ROWS: &[(&str, &str, &str)] = &[ ("在「{name}」上做事 · 點一下換電腦", "Works on “{name}” · click to switch computers", "「{name}」で作業 · クリックで切り替え"), ("一台 Linux 電腦，你所有的 Bot 共用，每個 Bot 有自己的資料夾（例如「Master 的電腦」）。一直開著，不會休眠。", "A Linux computer all your bots share; each bot has its own folder (e.g. “Master's computer”). Always on, it never sleeps.", "すべての Bot が共有する Linux コンピューター。Bot ごとにフォルダーがあります（例：「Master のコンピューター」）。常に起動していて、スリープしません。"), ("把 Bot 的檔案備份到這台電腦；每天也會自動存一份", "Back up the bots' files to this computer; a copy is also saved every day", "Bot のファイルをこのコンピューターにバックアップします。毎日自動でも保存します"), + ("等你處理", "Waiting for you", "対応待ち"), + ("需要處理", "Action needed", "対応が必要です"), + ("接管", "Take over", "操作を代わる"), + ("接管電腦", "Take over the computer", "コンピューターの操作を代わる"), + ("我完成了", "I'm done", "完了しました"), + ("跳過", "Skip", "スキップ"), + ("取消此要求且不執行該步驟；Bot 會跳過它並繼續", "Cancel this request without doing the step; the Bot continues without it", "この依頼を取り消し、その手順は行いません。Bot はそれを飛ばして続けます"), + ("連結中", "Connecting", "接続中"), + ("請到「{name}」上操作", "Please do it on “{name}”", "「{name}」で操作してください"), ]; diff --git a/src/main.rs b/src/main.rs index 2d2619e..e1b1e19 100644 --- a/src/main.rs +++ b/src/main.rs @@ -235,15 +235,18 @@ fn main() -> eframe::Result<()> { app::App::serve(&ctx); } - let options = eframe::NativeOptions { - viewport: egui::ViewportBuilder::default() - .with_title("AnyBot") - .with_inner_size(shot_size()) - .with_min_inner_size([app::SIDEBAR_MIN_WIDTH + app::CHAT_MIN_WIDTH, 480.0]) - .with_app_id("anybot") - .with_icon(app_icon()), - ..Default::default() - }; + let mut viewport = egui::ViewportBuilder::default() + .with_title("AnyBot") + .with_inner_size(shot_size()) + .with_min_inner_size([app::SIDEBAR_MIN_WIDTH + app::CHAT_MIN_WIDTH, 480.0]) + .with_app_id("anybot") + .with_icon(app_icon()); + // On a Mac the window is one surface to the very top (the grey system title bar didn't match + // a dark window); the traffic lights sit over the sidebar's empty corner. + if cfg!(target_os = "macos") { + viewport = viewport.with_fullsize_content_view(true).with_titlebar_shown(false).with_title_shown(false); + } + let options = eframe::NativeOptions { viewport, ..Default::default() }; eframe::run_native( "AnyBot", options, diff --git a/src/providers.rs b/src/providers.rs index 594951f..2d04023 100644 --- a/src/providers.rs +++ b/src/providers.rs @@ -223,6 +223,10 @@ pub enum ApprovalKind { EnableTools, /// 在這台電腦上執行 is off: let it work on this Mac? ("always" turns the setting on.) EnableThisComputer, + /// 需要處理: a step only the person can do on a computer (signing in, a captcha…). `reason` + /// is what to do, `grant` the computer's name; Once = 「我完成了」, Reject = 「跳過」 (the + /// original's box hand-off card). + Handoff, } pub struct AppCall { diff --git a/src/tools.rs b/src/tools.rs index f3c3a87..2550aab 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -60,8 +60,12 @@ pub fn specs() -> Vec { ] } +/// 請你接手: the step that asks the person to do something by hand on a computer. +pub const HANDOFF_TOOL: &str = "hand_over_computer"; + pub fn label(name: &str) -> &'static str { match name { + HANDOFF_TOOL => "請你接手電腦", "list_dir" => "查看資料夾", "search" => "搜尋檔案", "read_file" => "讀取檔案",