From 76a76c7d7a4be5e93226046151c072df8424cc6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=80=A7=E9=A9=8A?= Date: Fri, 6 Nov 2026 17:44:59 +0800 Subject: [PATCH] Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Web - web_fetch kept pages with a
only up to it (taken for ): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4 --- FEATURES.md | 9 +- box/Dockerfile | 4 +- box/a11y | 58 ++++++++++--- src/agent.rs | 214 +++++++++++++++++++++++++++------------------- src/app.rs | 15 ++-- src/appctl.rs | 18 ---- src/botbrowser.rs | 91 ++++++-------------- src/browser.rs | 92 ++++++++++++++++++-- src/cloudbox.rs | 210 +++++++++++++++++++++++++++------------------ src/computer.rs | 62 ++++++++++---- src/desktop.rs | 17 +++- src/providers.rs | 9 +- src/tools.rs | 22 ++++- src/vnc.rs | 104 +++++++++++++++++----- src/web.rs | 44 ++++++++-- 15 files changed, 629 insertions(+), 340 deletions(-) diff --git a/FEATURES.md b/FEATURES.md index 494ef5e..00e371d 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -896,15 +896,16 @@ AnyBot 是一個桌面聊天 App，介面和運作方式照 Grok Bot 重做。 - 最多等 20 秒。 - 最多下載 3,000,000 位元組（估計值）。 - 最多回傳 20,000 字（估計值），超過時結尾加「…（後面還有，已截斷）」。 -- 會拿掉程式碼、樣式和頁首，把段落、標題、清單、表格列分行，特殊符號轉回一般文字。 +- 會拿掉程式碼、樣式和 ``（只認完整的標籤名：`
` 不是 head，以前會把有 header 的網站截斷成只剩選單），把段落、標題、清單、表格列分行，表格儲存格之間空一格，特殊符號轉回一般文字。 - 輸出：「網址：…」「標題：…」加內文。 - 可能看到的訊息： - - 網站回應 400 以上：「網站回應 N。如果這個頁面要登入，請改用 browser_open + browser_read。」 - - 文字少於 200 字：「（頁面幾乎沒有文字，可能要登入或靠程式載入。可以改用 browser_open 打開，再用 browser_read 讀。）」 + - 網站回應 400 以上：「網站回應 N。如果這個頁面要登入，請改用 browser_open。」 + - 文字少於 200 字：「（頁面幾乎沒有文字，可能要登入或靠程式載入。可以改用 browser_open 打開，它會等頁面載入再回傳內容。）」 ### 瀏覽器工具 -- 「在瀏覽器打開」（browser_open）：參數是網址和設定檔。 +- 「在瀏覽器打開」（browser_open）：參數是網址和設定檔。等頁面載入完成（最多 12 秒，看載入狀態、網址和文字停下來，不是固定秒數），直接回傳和 browser_read 一樣的內容（最多 12,000 字，加上可以點的編號），所以打開後不用再讀一次。 +- 點擊、輸入、按鍵、選擇、捲動、上一頁之後：等頁面停下來就回來（頁面有變且穩定，或 0.5–0.9 秒內沒有開始變），最多 4 秒；以前是固定 1–2.5 秒。 - 「讀取瀏覽器分頁」（browser_read）：最多讀多少字（預設 20,000，最多 40,000），加上設定檔。 - 「瀏覽器設定檔」（browser_profiles）：列出、新增、切換。 diff --git a/box/Dockerfile b/box/Dockerfile index 5d7db30..853b732 100644 --- a/box/Dockerfile +++ b/box/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ at-spi2-core libatk-adaptor python3-gi gir1.2-atspi-2.0 libglib2.0-bin gsettings-desktop-schemas dconf-gsettings-backend \ novnc python3-websockify chromium fonts-noto-cjk fonts-noto-color-emoji \ ca-certificates curl wget git jq unzip zip less nano vim-tiny procps psmisc \ - python3 python3-pip python3-venv nodejs npm sudo tini dbus-x11 xclip \ + python3 python3-pip python3-venv nodejs npm sudo tini dbus-x11 xclip poppler-utils \ && apt-get clean && rm -rf /var/lib/apt/lists/* \ && useradd -m -s /bin/bash bot && echo 'bot ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/bot \ && ln -sf /usr/share/novnc/vnc.html /usr/share/novnc/index.html @@ -27,6 +27,6 @@ RUN chmod +x /usr/local/bin/anybot-box-start /usr/local/bin/anybot-a11y /usr/loc # Accessibility on for every app, including those started over `docker exec`. ENV DISPLAY=:1 SCREEN_SIZE=1280x800x24 GTK_MODULES=atk-bridge GTK_A11Y=atspi GNOME_ACCESSIBILITY=1 NO_AT_BRIDGE=0 EXPOSE 6080 -LABEL anybot.box.version="5" +LABEL anybot.box.version="6" ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/usr/local/bin/anybot-box-start"] diff --git a/box/a11y b/box/a11y index 9752cd1..c098c9f 100644 --- a/box/a11y +++ b/box/a11y @@ -29,6 +29,24 @@ NAMED = {"heading", "label", "static", "text", "image", "table cell", "column he SKIP_APPS = {"fluxbox", "a11y", "python3", "xterm-wrapper"} +def stacked_windows(): + """The desktop's shown windows, bottom to top: (x, y, w, h, pid).""" + out = [] + try: + ids = os.popen("xprop -root _NET_CLIENT_LIST_STACKING 2>/dev/null").read().split("#")[-1] + for wid in [w.strip() for w in ids.split(",") if w.strip().startswith("0x")]: + info = os.popen(f"xwininfo -id {wid} 2>/dev/null").read() + if "IsViewable" not in info: + continue + num = lambda k: int(info.split(k)[1].split()[0]) # noqa: E731 + pid = os.popen(f"xprop -id {wid} _NET_WM_PID 2>/dev/null").read().split("=")[-1].strip() + out.append((num("Absolute upper-left X:"), num("Absolute upper-left Y:"), num("Width:"), num("Height:"), + int(pid) if pid.isdigit() else 0)) + except Exception: # noqa: BLE001 + pass + return out + + def main(): limit, budget = 400, 3.0 args = sys.argv[1:] @@ -65,6 +83,7 @@ def main(): pass deadline = time.monotonic() + budget out = [] + windows = stacked_windows() def state(node): try: @@ -79,13 +98,20 @@ def main(): except Exception: # noqa: BLE001 return None - def onscreen(b): - x, y, w, h = b - if w <= 0 or h <= 0 or x + w <= 0 or y + h <= 0: - return False - return not (sw and x >= sw) and not (sh and y >= sh) + def cut(b, clip): + """The part of box b inside clip (both x, y, w, h), or None.""" + x1, y1 = max(b[0], clip[0]), max(b[1], clip[1]) + x2, y2 = min(b[0] + b[2], clip[0] + clip[2]), min(b[1] + b[3], clip[1] + clip[3]) + return (x1, y1, x2 - x1, y2 - y1) if x2 > x1 and y2 > y1 else None - def walk(node, depth, window): + def covered(cx, cy, pid): + """Another app's window is the top one at this spot (a terminal over the browser).""" + for (x, y, w, h, wpid) in reversed(windows): + if x <= cx < x + w and y <= cy < y + h: + return bool(pid) and bool(wpid) and wpid != pid + return False + + def walk(node, depth, window, clip, pid): if time.monotonic() > deadline or len(out) >= limit or depth > 45: return try: @@ -102,8 +128,13 @@ def main(): # took itself for minimized under this window manager) while its page is on the screen. # Visible with a place on the screen is what counts; containers are walked through. box = extents(node) - if depth > 1 and (not visible or box is None or not onscreen(box)): + # What is really seen of it: inside the screen, its window and the page's visible area + # (a link scrolled up under the toolbar was numbered over the Back button). + seen = cut(box, clip) if box is not None and box[2] > 0 and box[3] > 0 else None + if depth > 1 and (not visible or seen is None): return + if seen is not None and (role in ("frame", "dialog", "window") or role in ("document web", "document frame")): + clip = seen try: name = (node.get_name() or "").strip() except Exception: # noqa: BLE001 @@ -120,7 +151,8 @@ def main(): except Exception: # noqa: BLE001 pass active = role in ACTIVE or (editable and role in ("text", "paragraph", "section", "document web")) - if depth > 1 and box is not None and (active or (role in NAMED and (name or value))): + if depth > 1 and seen is not None and (active or (role in NAMED and (name or value))) \ + and not covered(seen[0] + seen[2] // 2, seen[1] + seen[3] // 2, pid): flags = [] if st is not None: for s, word in ((Atspi.StateType.FOCUSED, "focused"), (Atspi.StateType.CHECKED, "checked"), @@ -129,7 +161,7 @@ def main(): flags.append(word) if not st.contains(Atspi.StateType.ENABLED): flags.append("disabled") - x, y, w, h = box + x, y, w, h = seen out.append({"role": role, "name": name[:120], "value": value[:200], "x": x, "y": y, "w": w, "h": h, "active": active, "flags": flags, "window": window}) try: @@ -142,7 +174,7 @@ def main(): except Exception: # noqa: BLE001 continue if c is not None: - walk(c, depth + 1, window) + walk(c, depth + 1, window, clip, pid) for i in range(apps): try: @@ -152,7 +184,11 @@ def main(): continue if name in SKIP_APPS: continue - walk(app, 0, name) + try: + pid = app.get_process_id() + except Exception: # noqa: BLE001 + pid = 0 + walk(app, 0, name, (0, 0, sw or 100000, sh or 100000), pid) json.dump({"elements": out, "truncated": len(out) >= limit or time.monotonic() > deadline}, sys.stdout, ensure_ascii=False) return 0 diff --git a/src/agent.rs b/src/agent.rs index 06553ad..668aa1b 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -213,8 +213,6 @@ fn mentions(text: &str, t: &tools::Target, roles: &tools::Roles) -> bool { &["bot 的電腦", "bot的電腦", "你的電腦", "你自己的電腦", "你的虛擬", "虛擬電腦", "雲端電腦"] } else if roles.person.as_deref() == Some(t.name.as_str()) { &["我的電腦", "我電腦", "我這台", "這台電腦", "本機", "我的筆電", "my computer"] - } else if roles.server && matches!(t.place, tools::Place::Local { .. }) { - &["伺服器", "server"] } else { &[] }; @@ -227,11 +225,14 @@ fn mentions(text: &str, t: &tools::Target, roles: &tools::Roles) -> bool { /// A step naming a computer other than the chosen one, which the person didn't ask for, goes /// to the chosen one — and the bot is told so (quietly moving it made a bot "ask your Mac" six /// times while every step landed on its own computer). -fn keep_to_chosen(job: &Job, history: &[Message], args: &str) -> (String, Option) { +fn keep_to_chosen(job: &Job, history: &[Message], name: &str, args: &str) -> (String, Option) { let mut v = tools::args_of(args); let Some(named) = v.get("computer").and_then(|c| c.as_str()).map(|c| c.trim().to_string()).filter(|c| !c.is_empty()) else { return (args.to_string(), None) }; - if named == job.default_computer || !job.computers.iter().any(|t| t.name == job.default_computer) { - return (args.to_string(), None); + // The chosen computer can't do this kind of step (no screen, no browser): moving it there + // would only fail, so the step stays where the bot sent it. + match job.computers.iter().find(|t| t.name == job.default_computer) { + Some(chosen) if named != job.default_computer && can_do(chosen, name) => {} + _ => return (args.to_string(), None), } let said = history.iter().rev().filter(|m| m.role == Role::User && !m.summary).take(3).map(|m| m.text.as_str()).collect::>().join("\n"); match job.computers.iter().find(|t| t.name == named) { @@ -247,6 +248,23 @@ fn keep_to_chosen(job: &Job, history: &[Message], args: &str) -> (String, Option } } +/// Can this computer do this kind of step: a screen step needs a screen there, a browser step a +/// browser; anything else runs anywhere. +fn can_do(t: &tools::Target, tool: &str) -> bool { + let screen = computer::is_computer_tool(tool); + let browser = web::is_web_tool(tool) && tool != "web_fetch"; + if !screen && !browser { + return true; + } + match &t.place { + tools::Place::Device { .. } => true, + tools::Place::Box { .. } => crate::cloudbox::available(), + tools::Place::Local { .. } if screen => computer::screen_supported(), + tools::Place::Local { .. } => crate::botbrowser::available(), + _ => false, + } +} + /// What element `e7` is, from the latest app list in the chat (「按鈕「7」」). fn element_named(history: &[Message], r: &str) -> Option { let tag = format!("[{}] ", crate::marks::number_of(r)?); @@ -257,9 +275,12 @@ fn element_named(history: &[Message], r: &str) -> Option { /// What makes two screen or browser steps "the same" (None for other tools): the tool, the /// computer and the target, a click's spot on a 16-pixel grid (an inch beside an empty strip is -/// the same miss). +/// the same miss). Looking, waiting, and moving through a page (scrolling, keys, back) aren't +/// counted: doing those again and again is how a long page or a list is read. fn step_key(name: &str, args: &str) -> Option { - if !(computer::is_computer_tool(name) || web::is_web_tool(name)) || matches!(name, "computer_use_screenshot" | "browser_read" | "browser_screenshot" | "browser_wait") { + if !(computer::is_computer_tool(name) || web::is_web_tool(name)) + || matches!(name, "computer_use_screenshot" | "browser_read" | "browser_screenshot" | "browser_wait" | "computer_use_scroll" | "browser_scroll" | "computer_use_press_key" | "browser_press" | "browser_back") + { return None; } let mut v = tools::args_of(args); @@ -273,6 +294,32 @@ fn step_key(name: &str, args: &str) -> Option { Some(format!("{name} {v}")) } +/// The bot's own computer can come or go mid-turn (Docker started or stopped): its place follows +/// what is true now, and the bot is told when it changed. (Fixed at the start of the turn, a box +/// that came up mid-turn stayed "no screen" and every step there ran in the wrong place.) +fn refresh_own(job: &mut Job, bot_id: u64) -> Option { + let own = job.roles.own.clone(); + let t = job.computers.iter_mut().find(|t| t.name == own)?; + match (&t.place, crate::cloudbox::available()) { + (tools::Place::Sealed { .. }, true) => { + t.place = tools::Place::Box { dir: crate::cloudbox::bot_dir(bot_id) }; + Some(format!("（「{own}」現在可以用了：它是你自己的 Linux 桌面，可以執行指令、看畫面、用瀏覽器。）")) + } + (tools::Place::Box { .. }, false) => { + t.place = tools::Place::Sealed { root: crate::model::bot_computer_dir(bot_id) }; + Some(format!("（「{own}」現在連不到 Docker：暫時不能執行指令、看畫面。）")) + } + _ => None, + } +} + +/// "Always allow" for this turn: the grant is kept (once). +fn keep_grant(grants: &mut Vec, g: String) { + if !g.is_empty() && !grants.contains(&g) { + grants.push(g); + } +} + /// Every CHECK_IN_STEPS steps, the newest result carries a question for the bot: still getting /// somewhere, or stuck? Only the model sees it (the chat keeps the result as it was). fn check_in(history: &mut [Message], step: usize) { @@ -451,6 +498,7 @@ fn wait_answer(rx: std::sync::mpsc::Receiver, cancel: &AtomicBoo pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc, ctx: eframe::egui::Context) { std::thread::spawn(move || { + let mut job = job; let send = |ev: StreamEvent| { let _ = tx.send((bot_id, ev)); ctx.request_repaint(); @@ -468,6 +516,11 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< maybe_compact(&job, &mut history, COMPACT_THRESHOLD_PERCENT + COMPACT_TOLERANCE_PERCENT, &cancel, &send); drain_steer(&job, &mut history, &send); check_in(&mut history, step); + if let Some(note) = refresh_own(&mut job, bot_id) { + if let Some(m) = history.last_mut().filter(|m| m.role == Role::Tool) { + m.text.push_str(&format!("\n\n{note}")); + } + } send(StreamEvent::NextStep(job.via.clone())); } let req = ChatRequest { history: history.clone(), ..job.req.clone() }; @@ -485,7 +538,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< // Steps moved back to the chosen computer: what the bot is told with each result. let mut moved: std::collections::HashMap = std::collections::HashMap::new(); for c in out.tool_calls.iter_mut() { - let (args, note) = keep_to_chosen(&job, &history, &c.args); + let (args, note) = keep_to_chosen(&job, &history, &c.name, &c.args); c.args = args; if let Some(n) = note { moved.insert(c.id.clone(), n); @@ -533,6 +586,15 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< repeat_note = "⚠️ 這一步你已經做了 3 次。如果畫面還不是你要的，下一步一定要換方法，不要再做同一步。\n".into(); } } + // Every step ends the same way: its status, then its result (with the notes) and + // any picture, shown and kept. + let finish = |history: &mut Vec, output: &str, status: ToolStatus, image: Option| { + send(StreamEvent::ToolStatus(call.id.clone(), status)); + let mut result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, output))); + result.images.extend(image); + send(StreamEvent::ToolResult(result.clone())); + history.push(result); + }; if appctl::is_app_tool(&call.name) { // App actions run on the window's side, which also asks for deletions. send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running)); @@ -548,21 +610,14 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< Err(_) => break ("App 沒有回應".to_string(), false), } }; - send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed })); - let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, &output, if ok { ToolStatus::Done } else { ToolStatus::Failed }, None); continue; } // The person's own recording, shown back to the bot: nothing to ask. if call.name == "view_demo_step" { send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running)); let (output, ok, img) = crate::teach::view_step(&job.demos, &call.args); - send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed })); - let mut result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); - result.images.extend(img); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, &output, if ok { ToolStatus::Done } else { ToolStatus::Failed }, img); continue; } // Public web pages need no computer and no permission. @@ -581,11 +636,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< }; // Which computer's browser it was, like every other step. let output = if on.is_empty() { output } else { format!("[{on}] {output}") }; - send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed })); - let mut result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); - result.images.extend(img); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, &output, if ok { ToolStatus::Done } else { ToolStatus::Failed }, img); continue; } let is_web = web::is_web_tool(&call.name); @@ -643,28 +694,19 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< } } }; - send(StreamEvent::ToolStatus(call.id.clone(), status)); - let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, &output, status, None); continue; } // 生活卡片: drawn by the window from the call itself. if call.name == "show_card" { - send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Done)); - let result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, "卡片已顯示給使用者。")); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, "卡片已顯示給使用者。", ToolStatus::Done, None); continue; } // 表情回應: an emoji on the person's latest message (no words needed). if call.name == "react" { let emoji: String = tools::args_of(&call.args)["emoji"].as_str().unwrap_or("👍").chars().take(4).collect(); send(StreamEvent::React(emoji.clone())); - send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Done)); - let result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &format!("已在使用者的訊息上按了 {emoji}。"))); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, &format!("已在使用者的訊息上按了 {emoji}。"), ToolStatus::Done, None); continue; } // 傳檔案: put a file the bot made into the chat as a card the person can open. @@ -693,10 +735,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< } Err(e) => (e, false), }; - send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed })); - let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, &output, if ok { ToolStatus::Done } else { ToolStatus::Failed }, None); continue; } // 檔案預覽 / Bot 做的頁面: open what the bot made on the person's screen. @@ -728,10 +767,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< (None, true) => ("不知道使用者現在在用哪台電腦，沒辦法打開給他看。把網址直接寫在回覆裡。".to_string(), false), (_, false) => ("AnyBot 在伺服器上，檔案沒辦法直接在使用者的螢幕打開。改用 send_file 把檔案傳到聊天裡，使用者點一下就能看。".to_string(), false), }; - send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed })); - let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, &output, if ok { ToolStatus::Done } else { ToolStatus::Failed }, None); continue; } let local = match (&on_box, target_for(&job, &call.args)) { @@ -751,10 +787,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< Ok(p) => (format!("找不到 {}", p.display()), false), Err(e) => (e, false), }; - send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed })); - let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, &output, if ok { ToolStatus::Done } else { ToolStatus::Failed }, None); continue; } // 打電話、傳簡訊、在 X 發文: speaks for the person, so asked every single time @@ -774,10 +807,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< (t, false) => (t, ToolStatus::Failed), } }; - send(StreamEvent::ToolStatus(call.id.clone(), status)); - let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, &output, status, None); continue; } // 發布網站: put a page or folder the bot made on the web (asked once per site name). @@ -791,9 +821,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< match ask_person(ApprovalKind::Action, format!("把 {path} 發布成公開網站「{name}」（任何知道網址的人都看得到）"), grant.clone()) { ApprovalAnswer::Once => allowed = true, ApprovalAnswer::Always(g) => { - if !g.is_empty() && !grants.contains(&g) { - grants.push(g); - } + keep_grant(&mut grants, g); allowed = true; } ApprovalAnswer::Reject => {} @@ -818,10 +846,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< Err(e) => (e, ToolStatus::Failed), } }; - send(StreamEvent::ToolStatus(call.id.clone(), status)); - let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, &output, status, None); continue; } // 保險庫: the bot sees only metadata; filling asks the person first. A login on a @@ -829,10 +854,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< // a card is asked every time (like the original's 允許一次 / 拒絕). if call.name == crate::vault::LIST_TOOL { let output = crate::vault::list_for_bot(&call.args); - send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Done)); - let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, &output, ToolStatus::Done, None); continue; } if call.name == crate::vault::TOOL || call.name == crate::vault::CARD_TOOL || call.name == crate::vault::CONTACT_TOOL { @@ -860,9 +882,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< match ask_person(ApprovalKind::Action, question, grant.clone()) { ApprovalAnswer::Once => allowed = true, ApprovalAnswer::Always(g) => { - if !g.is_empty() && !grants.contains(&g) { - grants.push(g); - } + keep_grant(&mut grants, g); allowed = true; } ApprovalAnswer::Reject => {} @@ -887,10 +907,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< } } }; - send(StreamEvent::ToolStatus(call.id.clone(), status)); - let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, &output, status, None); continue; } // MCP connectors: asked once per server (or always allowed), like the original's @@ -904,9 +921,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< match ask_person(ApprovalKind::Action, format!("使用連接器「{server}」的工具：{tool} {args_short}"), grant.clone()) { ApprovalAnswer::Once => allowed = true, ApprovalAnswer::Always(g) => { - if !g.is_empty() && !grants.contains(&g) { - grants.push(g); - } + keep_grant(&mut grants, g); allowed = true; } ApprovalAnswer::Reject => {} @@ -922,11 +937,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< } else { ("使用者拒絕了這個操作。請不要再試同樣的操作，改問使用者想怎麼做。".to_string(), ToolStatus::Denied, None) }; - send(StreamEvent::ToolStatus(call.id.clone(), status)); - let mut result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); - result.images.extend(img); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, &output, status, img); continue; } // The person's own apps and everyday browser are on the computer they are using: @@ -1056,7 +1067,8 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< } } (Ok(t), Ok(())) => tools::decide_on(&call.name, &call.args, t, &grants), - (Err(e), _) => Decision::Deny(e.clone()), + // Can't reach that computer: reported as a failed step below, not a safety block. + (Err(_), _) => Decision::Allow, // The person said no to the pop-up: nothing more to ask. (_, Err(_)) => Decision::Allow, }; @@ -1073,9 +1085,9 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< }; match verdict { ReviewVerdict::Ask(rule) => Decision::Ask { reason: format!("{reason}（符合你的「一律詢問」規則：{rule}）"), grant: String::new() }, + // (No note on the result: the model doesn't need it, and it was on every step.) _ => { let _ = grant; - review_note = "（直接操作模式：已自動允許）\n".into(); Decision::Allow } } @@ -1099,6 +1111,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< }; let approved = match decision { _ if gate.is_err() => Err((gate.clone().unwrap_err(), ToolStatus::Denied)), + _ if target.is_err() => Err((target.clone().err().unwrap_or_default(), ToolStatus::Failed)), Decision::Allow => Ok(()), // Sending (Messages, mail…) still asks every time, even right after a switch. Decision::Ask { .. } if said_yes && !(is_mac && crate::macapps::is_send(&call.name)) => Ok(()), @@ -1109,9 +1122,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< match ask_person(ApprovalKind::Action, reason, grant) { ApprovalAnswer::Once => Ok(()), ApprovalAnswer::Always(g) => { - if !g.is_empty() && !grants.contains(&g) { - grants.push(g); - } + keep_grant(&mut grants, g); Ok(()) } ApprovalAnswer::Reject => Err(("使用者拒絕了這個操作。請不要再試同樣的操作，改問使用者想怎麼做。".into(), ToolStatus::Denied)), @@ -1158,11 +1169,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< (Ok(()), Err(e)) => (e.clone(), ToolStatus::Failed), (Err(x), _) => x, }; - send(StreamEvent::ToolStatus(call.id.clone(), status)); - let mut result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); - result.images.extend(image); - send(StreamEvent::ToolResult(result.clone())); - history.push(result); + finish(&mut history, &output, status, image); if cancel.load(Ordering::Relaxed) { return; } @@ -1281,6 +1288,28 @@ mod tests { port } + /// A reply with nothing in it (a dropped stream) is asked again, not taken as the answer. + #[test] + fn an_empty_reply_is_asked_again() { + let port = fake_model(vec![("", serde_json::json!("")), ("", serde_json::json!("好了"))]); + let job = Job { + req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("在嗎")], tools: vec![], session: String::new() }, + ..test_job() + }; + let (tx, rx) = channel(); + spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default()); + let mut texts = vec![]; + loop { + match rx.recv_timeout(Duration::from_secs(20)).expect("loop stalled") { + (_, StreamEvent::StepDone(m)) => texts.push(m.text), + (_, StreamEvent::Done) => break, + (_, StreamEvent::Error(e)) => panic!("{e}"), + _ => {} + } + } + assert_eq!(texts, vec!["好了".to_string()]); + } + /// A brain with no Chrome of its own — a cloud brain — browses on its own box: the bot's /// browser_open and browser_read for its computer reach the box's Chromium, not the /// person's browser on the server. Run where no Chrome-family browser is installed, against @@ -1336,6 +1365,9 @@ mod tests { assert_eq!(step_key("computer_use_screenshot", "{}"), None); assert_eq!(step_key("shell", r#"{"command":"ls"}"#), None); assert!(step_key("browser_click", r#"{"ref":"5"}"#).is_some()); + // Reading down a page is not going in circles. + assert_eq!(step_key("browser_scroll", r#"{"direction":"down"}"#), None); + assert_eq!(step_key("computer_use_press_key", r#"{"key":"Down"}"#), None); } #[test] @@ -1356,7 +1388,6 @@ mod tests { let own = tools::Target { name: "Master 的電腦".into(), place: tools::Place::Box { dir: String::new() } }; let mac = tools::Target { name: "王小明的Mac mini".into(), place: tools::Place::Device { host: "m".into(), dir: String::new() } }; let desk = tools::Target { name: "ai".into(), place: tools::Place::Device { host: "a".into(), dir: String::new() } }; - let server = tools::Target { name: "雲端大腦".into(), place: tools::Place::Local { workdir: Default::default() } }; assert!(mentions("在 Master 的電腦打開 Threads", &own, &roles)); assert!(mentions("用master的電腦", &own, &roles)); assert!(mentions("你都用 Bot的電腦", &own, &roles)); @@ -1366,7 +1397,6 @@ mod tests { assert!(!mentions("截圖我的電腦", &own, &roles)); assert!(mentions("在 ai 上跑", &desk, &roles)); assert!(!mentions("用 AI 幫我想標題", &mac, &roles)); - assert!(mentions("在伺服器上看看磁碟", &server, &roles)); } #[test] @@ -1382,11 +1412,15 @@ mod tests { ..test_job() }; // Not asked for: back to the chosen one, with a word for the bot. - let (args, note) = keep_to_chosen(&job, &[Message::user("幫我截圖")], r#"{"computer":"王小明的Mac mini"}"#); + let (args, note) = keep_to_chosen(&job, &[Message::user("幫我截圖")], "shell", r#"{"computer":"王小明的Mac mini"}"#); assert!(args.contains("Master 的電腦") && note.unwrap().contains("王小明的Mac mini")); // Named by the person: it stays. - let (args, note) = keep_to_chosen(&job, &[Message::user("看一下我的電腦畫面")], r#"{"computer":"王小明的Mac mini"}"#); + let (args, note) = keep_to_chosen(&job, &[Message::user("看一下我的電腦畫面")], "shell", r#"{"computer":"王小明的Mac mini"}"#); assert!(args.contains("王小明的Mac mini") && note.is_none()); + // The chosen computer has no screen (no Docker): a screen step isn't moved there to fail. + let sealed = Job { computers: vec![tools::Target { name: "Master 的電腦".into(), place: tools::Place::Sealed { root: std::env::temp_dir() } }, job.computers[1].clone()], ..job }; + let (args, note) = keep_to_chosen(&sealed, &[Message::user("幫我截圖")], "computer_use_screenshot", r#"{"computer":"王小明的Mac mini"}"#); + assert!(args.contains("王小明的Mac mini") && note.is_none(), "{args}"); } #[test] diff --git a/src/app.rs b/src/app.rs index 7f2bb5d..b955a6f 100644 --- a/src/app.rs +++ b/src/app.rs @@ -770,7 +770,7 @@ impl App { } if !screens.is_empty() { let default_screen = if screens.contains(&default_computer) { default_computer.clone() } else { screens[0].clone() }; - tool_specs.extend(crate::computer::specs_for(&screens, &default_screen)); + tool_specs.extend(crate::computer::specs_for(&screens, &default_screen, screens.contains(&default_computer))); // 需要處理: hand a step to the person on that screen (the original's box hand-off). tool_specs.push(serde_json::json!({"name": tools::HANDOFF_TOOL, "description": "Ask the person to do one step themselves on a computer's screen: signing in (you never type their passwords), a captcha or puzzle, a code sent to their phone or email, approving a payment, or anything else only they may do. First get the computer to that point (open the page, click into the sign-in form), then call this with a short instruction. A card with that computer's live screen appears in the chat; they take over, do it, and press 「我完成了」 (or 「跳過」). The result says which; after 「我完成了」 take a screenshot and carry on. Don't ask them in words to find a window themselves.", @@ -972,8 +972,8 @@ impl App { if secrets.is_empty() { String::new() } else { format!("使用者設好的環境密鑰（已是環境變數，值不要印出來）：{}。", secrets.join("、")) } ) } - tools::Place::Sealed { root } => format!("「{}」：你自己的電腦（獨立資料夾 {}，看不到使用者的檔案；沒有螢幕），一律可用。", t.name, root.display()), - tools::Place::Local { workdir } if roles.server => format!("「{}」：AnyBot 大腦所在的伺服器（起始資料夾 {}），不是使用者在用的電腦；只有使用者指名它才用。", t.name, workdir.display()), + tools::Place::Sealed { root } if tools::sealed_runs_commands() => format!("「{}」：你自己的電腦（獨立資料夾 {}，看不到使用者的檔案；沒有螢幕），一律可用。", t.name, root.display()), + tools::Place::Sealed { root } => format!("「{}」：你自己的電腦現在沒有開（AnyBot 所在的這台沒有 Docker）：只能在你的資料夾 {} 讀寫檔案，不能執行指令，也沒有螢幕。", t.name, root.display()), tools::Place::Local { workdir } => format!( "「{}」：AnyBot 開著的這台{}（起始資料夾 {}）{}，{}。", t.name, @@ -3588,7 +3588,6 @@ impl App { } pub fn serve(ctx: &egui::Context) -> ! { - crate::appctl::SERVING.store(true, std::sync::atomic::Ordering::Relaxed); let mut app = App::new(ctx, None); app.headless = true; if let Ok(pw) = std::env::var("ANYBOT_PHONE_PASSWORD") { @@ -4025,14 +4024,12 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu 做完 App 操作後，用一句話告訴使用者結果（例如排程的時間）。\n\n\ 你也可以上網和用使用者的瀏覽器：\n\ - 公開網頁：web_fetch。\n\ - - 要登入才看得到的（Threads、Instagram、Facebook、Gmail、銀行…）：用 browser_open 在 AnyBot 自己的瀏覽器視窗打開，再用 browser_read 讀分頁內容。例如看 Threads 帳號：browser_open(\"https://www.threads.net\") 然後 browser_read。\n\ + - 要登入才看得到的（Threads、Instagram、Facebook、Gmail、銀行…）：用 browser_open 在 AnyBot 自己的瀏覽器視窗打開：它會等頁面載入，直接回傳頁面內容和可以點的編號（不用再 browser_read；之後頁面變了才用 browser_read 重讀）。例如看 Threads 帳號：browser_open(\"https://www.threads.net\")。\n\ - 網頁上的事（任何電腦、任何網站）一律用 browser_* 工具：先 browser_read（或 browser_screenshot）拿到每個連結、按鈕、欄位的編號和網址，再用 browser_click／browser_type／browser_select／browser_press 照編號操作；要去的頁面有網址就直接 browser_open。網頁上絕對不要看截圖猜座標點。表格用 browser_table 讀，找字用 browser_find，要等載入用 browser_wait，匯出報表用 browser_download。不要猜網址；找不到想要的功能（例如排序、篩選），就看欄位標題、篩選器、下拉選單或分頁標籤，還是沒有就照頁面現有的資料自己整理，並說明網站沒有這個功能。 - 絕對不要直接呼叫網站的內部 API、下載網站的程式碼來研究，或用程式大量抓資料：網站會把帳號當成機器人（例如跳出拼圖驗證、封鎖帳號）。也絕對不要用指令去讀瀏覽器的 Cookie、瀏覽紀錄、密碼或設定檔資料庫。 - - 需要使用者親手做的步驟（登入、輸入密碼、驗證碼、拼圖、簡訊或 email 驗證、確認付款）：先在電腦上把那一頁打開、點到要輸入的地方，再呼叫 hand_over_computer，instruction 寫清楚要他做什麼。聊天裡會出現那台電腦的即時畫面，他點「接管」做完後按「我完成了」，你再截圖確認、接著做。不要只用文字叫他自己去找視窗；也絕對不要替他打密碼。 - - 如果讀到的是登入頁：把登入頁開在那台電腦上，用 hand_over_computer 請使用者登入一次（之後會記住），他按「我完成了」再讀。絕對不要叫使用者去改瀏覽器的設定或選單（例如「允許來自 Apple 事件的 JavaScript」），那不需要。也不要說要從 Chrome 複製登入或 Cookie，沒有這個功能。\n\ + - 需要使用者親手做的步驟（讀到登入頁、輸入密碼、驗證碼、拼圖、簡訊或 email 驗證、確認付款）：先在那台電腦上把那一頁打開、點到要輸入的地方，再呼叫 hand_over_computer，instruction 寫清楚要他做什麼。聊天裡會出現那台電腦的即時畫面，他點「接管」做完後按「我完成了」，你再確認、接著做（登入一次之後就會記住）。不要只用文字叫他自己去找視窗，絕對不要替他打密碼，也不要叫他改瀏覽器的設定或選單、或說要從 Chrome 複製登入或 Cookie（都不需要，也沒有這個功能）。\n\ - AnyBot 瀏覽器有設定檔（profile），每個有自己的登入。使用者說要換 profile／設定檔／另一個帳號時，用 browser_profiles：list 看有哪些，use 切換，add 新增（新的要登入一次）。也可以在 browser_open / browser_read 帶 profile 參數。使用者想用自己平常的 Chrome（自己的 Chrome 設定檔、已經登入的帳號）時，用 profile「你的瀏覽器」：請使用者在 Chrome 切到想要的設定檔，再 browser_read。\n\ - - 你也能像人一樣操作螢幕（{screens}），用在檔案、指令和網頁工具做不到的事（例如操作其他 App）。先 computer_use_screenshot：你會拿到一張每個按鈕、連結、欄位、選單項目都框起來、標了編號的截圖，和同樣編號的清單。一律照編號操作（click／typing／scroll 的 ref，例如 ref 填 12），點下去就是那個東西的正中央；只有圖上沒有編號的東西（圖片裡的一個點、遊戲）才用 x、y。每個動作做完都會附上新的編號截圖，不要另外截圖。結果說「畫面沒有變化」就是沒點到：換一個編號或換方法，絕對不要重複同一步。要在某個 App 裡打字：先 computer_use_open_app，再 computer_use_typing（ref 是輸入框的編號；submit=true 順便按 return）\n\ - - 已經從目前畫面知道接下來的幾個動作時（填一張表：點欄位、打字、下一欄、打字、按 return；一組捷徑），用 computer_use_batch 一次做完：actions 是動作陣列，每個 {{\"action\":\"click\",\"ref\":\"12\"}} 這樣（最多 8 個）。會依序執行，最後只附一張截圖；第 2 步要看第 1 步的結果才知道怎麼做的，就拆開用單個動作。每次看截圖用一句話說畫面顯示什麼、這步成功與否。\n\ + - 你也能像人一樣操作螢幕（{screens}），用在檔案、指令和網頁工具做不到的事（例如操作其他 App）。照截圖上的編號操作（工具說明裡有怎麼用）；只有沒有編號的東西才用 x、y。結果說「畫面和上一步完全一樣」就是沒點到：換一個編號或換方法，絕對不要重複同一步。已經從畫面知道接下來幾步時（填一張表、一組捷徑），用 computer_use_batch 一次做完；要看上一步結果才知道下一步的，就一步一步來。每次看截圖用一句話說畫面顯示什麼、這步成功與否。\n\ {}\ - 不要說「我沒有瀏覽器或工具」：先試這些工具。需要權限時 App 會自己跳出視窗問使用者；工具回報要開什麼設定時，照實轉告。", now.format("%Y-%m-%d %H:%M"), diff --git a/src/appctl.rs b/src/appctl.rs index 9e4226a..7c9f392 100644 --- a/src/appctl.rs +++ b/src/appctl.rs @@ -145,10 +145,6 @@ pub fn needs_approval(name: &str, args: &str, state: &State) -> Option { None } -/// True while running as a headless brain (`anybot --serve`): "this computer" is a server, -/// not a person's desktop, and the computer-wording tools must say so. Set in App::serve. -pub static SERVING: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); - pub struct Outcome { pub text: String, pub ok: bool, @@ -546,19 +542,6 @@ pub fn apply(state: &mut State, bot_id: u64, name: &str, args: &str) -> Outcome let name = state.bot(target).map(|b| b.name.clone()).unwrap_or_default(); Outcome { text: format!("已交給「{name}」，它的回覆會出現在它自己的聊天裡。"), ok: true, wake: vec![(target, msg)], theme_changed: false, page: None } } - "list_computers" => { - // (The window answers this with the account's computers too; this is the plain list.) - let serving = SERVING.load(std::sync::atomic::Ordering::Relaxed); - let own = state.bot(bot_id).map(|b| model::own_computer_name(&b.name)).unwrap_or_else(|| model::BOT_COMPUTER_NAME.into()); - let here = state.this_computer_name(); - let mut lines = vec![format!("「{own}」：你自己的電腦（AI 的電腦，不是使用者的電腦，獨立資料夾），一律可用")]; - // A brain on a server doesn't work on the server itself. - if !serving { - lines.push(format!("「{here}」：AnyBot 開著的這台{}，{}", if cfg!(target_os = "macos") { " Mac" } else { "電腦" }, if state.local_execution { "已允許" } else { "使用者還沒在設定裡允許（設定 → 電腦 → 在這台電腦上執行）" })); - } - lines.extend(state.remote_computers.iter().map(|c| format!("「{}」：透過 SSH 連到 {}", c.name, c.ssh_target))); - ok(lines.join("\n")) - } "update_settings" => { let mut out = Outcome { text: String::new(), ok: true, wake: vec![], theme_changed: false, page: None }; let mut changed = vec![]; @@ -656,7 +639,6 @@ mod tests { assert!(apply(&mut st, 1, "update_bot", r#"{"default_computer":"火星"}"#).text.contains("沒有")); assert!(apply(&mut st, 1, "update_settings", r#"{"theme":"dark","timezone":"Asia/Tokyo"}"#).theme_changed); assert_eq!(st.timezone, "Asia/Tokyo"); - assert!(apply(&mut st, 1, "list_computers", "{}").text.contains("還沒")); } #[test] diff --git a/src/botbrowser.rs b/src/botbrowser.rs index 881f262..446c495 100644 --- a/src/botbrowser.rs +++ b/src/botbrowser.rs @@ -8,9 +8,7 @@ // browser keeps its own: 「預設」 plus any the person or the bot adds (工作, 私人…). Each is its own // browser folder and window with its own logins; switching picks which one the bot uses. -use serde_json::{json, Value}; -use std::io::ErrorKind; -use std::net::{SocketAddr, TcpStream}; +use serde_json::Value; use std::path::PathBuf; use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; @@ -19,8 +17,6 @@ use std::time::{Duration, Instant}; pub const PORT: u16 = 9333; /// How long to wait for the browser to come up (estimated). const START_TIMEOUT_SECS: u64 = 20; -/// How long a page read may take (estimated). -const READ_TIMEOUT_SECS: u64 = 20; /// The first profile: the browser folder made before profiles existed (its logins stay). pub const DEFAULT_PROFILE: &str = "預設"; /// The person's own browser (read through accessibility, like Grok Bot's Computer Use helper): @@ -322,10 +318,19 @@ pub fn open(url: &str, profile: Option<&str>) -> Result { return Err(format!("{name} 沒辦法開新分頁（{}）", r.status())); } } - std::thread::sleep(Duration::from_secs(3)); - Ok(format!("已在 AnyBot 瀏覽器（{name}，設定檔「{prof}」）打開 {url}。接著用 browser_read 讀內容。如果頁面要登入，請使用者在那個視窗登入一次，之後會記住。")) + // Back when the page has loaded (not after a fixed 3 s: a slow page was still blank then), + // with what it shows — the same as browser_read, one step fewer for every page. + let loaded = crate::browser::wait_loaded(Some(&prof), OPEN_WAIT_MS); + let page = read(PAGE_CHARS_ON_OPEN, Some(&prof)).unwrap_or_default(); + let head = format!("已在 AnyBot 瀏覽器（{name}，設定檔「{prof}」）打開 {url}{}", if loaded { "。" } else { "，還在載入（可以用 browser_wait 等它，再 browser_read）。" }); + Ok(if page.is_empty() { head } else { format!("{head}\n\n{page}") }) } +/// Longest wait for a page browser_open opened (estimated). +const OPEN_WAIT_MS: u64 = 12_000; +/// Text of the page handed back by browser_open (browser_read gives more when asked). +const PAGE_CHARS_ON_OPEN: usize = 12_000; + /// Bring a profile's window up (Settings button): start it, or bring its front tab forward. pub fn show(profile: &str) -> Result<(), String> { let (_, port, fresh) = ensure(profile, None)?; @@ -362,50 +367,11 @@ pub fn profiles_tool(action: &str, name: &str) -> Result { } } -/// The page to read: the first ordinary page tab (Chrome lists the most recent first). -pub fn pick_page(list: &Value) -> Option<(String, String, String)> { - list.as_array()?.iter().find(|t| t["type"] == "page" && !t["url"].as_str().unwrap_or("").starts_with("devtools://")).map(|t| { - (t["webSocketDebuggerUrl"].as_str().unwrap_or("").to_string(), t["title"].as_str().unwrap_or("").to_string(), t["url"].as_str().unwrap_or("").to_string()) - }) -} - -fn evaluate(port: u16, ws_url: &str, expression: &str) -> Result { - let addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap(); - let stream = TcpStream::connect_timeout(&addr, Duration::from_secs(5)).map_err(|e| e.to_string())?; - stream.set_read_timeout(Some(Duration::from_secs(READ_TIMEOUT_SECS))).ok(); - let (mut ws, _) = tungstenite::client::client(ws_url, stream).map_err(|e| format!("連不上 AnyBot 瀏覽器：{e}"))?; - let msg = json!({"id": 1, "method": "Runtime.evaluate", "params": {"expression": expression, "returnByValue": true, "awaitPromise": true}}); - ws.send(tungstenite::Message::Text(msg.to_string().into())).map_err(|e| e.to_string())?; - let start = Instant::now(); - loop { - if start.elapsed() > Duration::from_secs(READ_TIMEOUT_SECS) { - return Err("讀取頁面太久了".into()); - } - match ws.read() { - Ok(tungstenite::Message::Text(t)) => { - let v: Value = serde_json::from_str(t.as_str()).unwrap_or(Value::Null); - if v["id"] == 1 { - let _ = ws.close(None); - if let Some(e) = v.get("error") { - return Err(format!("頁面拒絕讀取：{e}")); - } - return Ok(v["result"]["result"]["value"].clone()); - } - } - Ok(_) => {} - Err(tungstenite::Error::Io(e)) if e.kind() == ErrorKind::WouldBlock || e.kind() == ErrorKind::TimedOut => return Err("讀取頁面太久了".into()), - Err(e) => return Err(e.to_string()), - } - } -} - /// Run JavaScript in the current tab of the bot's browser. Returns (address, result). pub fn eval_current(profile: Option<&str>, expression: &str) -> Result<(String, Value), String> { - let prof = resolve(profile)?; - let (_, port, _) = ensure(&prof, None)?; - let list: Value = local().get(&endpoint(port, "/json/list")).call().map_err(|e| e.to_string())?.into_body().read_json().map_err(|e| e.to_string())?; - let Some((ws_url, _, url)) = pick_page(&list) else { return Err("AnyBot 瀏覽器沒有開著的分頁，請先用 browser_open 打開登入頁。".into()) }; - let v = evaluate(port, &ws_url, expression)?; + let mut tab = crate::browser::Tab::current(profile)?; + let url = tab.eval("location.href")?.as_str().unwrap_or("").to_string(); + let v = tab.eval(expression)?; Ok((url, v)) } @@ -414,14 +380,14 @@ const READ_EXPR: &str = "JSON.stringify({t:document.title,u:location.href,x:docu /// Title, address and visible text of the current tab of the bot's browser. pub fn read(max_chars: usize, profile: Option<&str>) -> Result { let prof = resolve(profile)?; - let (name, port, _) = ensure(&prof, None)?; - let list: Value = local().get(&endpoint(port, "/json/list")).call().map_err(|e| e.to_string())?.into_body().read_json().map_err(|e| e.to_string())?; - let Some((ws_url, title, url)) = pick_page(&list) else { return Err("AnyBot 瀏覽器沒有開著的分頁，請先用 browser_open 打開網址。".into()) }; - let v = evaluate(port, &ws_url, READ_EXPR)?; + let (name, _, _) = ensure(&prof, None)?; + // One connection to the tab for the text and the numbered elements. + let mut tab = crate::browser::Tab::current(Some(&prof))?; + let v = tab.eval(READ_EXPR)?; let page: Value = v.as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or(Value::Null); let text: String = page["x"].as_str().unwrap_or("").lines().map(str::trim).filter(|l| !l.is_empty()).collect::>().join("\n"); - let title = page["t"].as_str().map(str::to_string).unwrap_or(title); - let url = page["u"].as_str().map(str::to_string).unwrap_or(url); + let title = page["t"].as_str().unwrap_or("").to_string(); + let url = page["u"].as_str().unwrap_or("").to_string(); let mut clipped: String = text.chars().take(max_chars).collect(); if text.chars().count() > max_chars { clipped.push_str("\n…（後面還有，已截斷）"); @@ -432,7 +398,7 @@ pub fn read(max_chars: usize, profile: Option<&str>) -> Result { }; let mut out = format!("瀏覽器：AnyBot 瀏覽器（{name}，設定檔「{prof}」）\n標題：{title}\n網址：{url}\n\n{clipped}"); // The links, buttons and fields, numbered for browser_click / browser_type / browser_select. - if let Ok((_, _, list, _)) = crate::browser::Tab::current(Some(&prof)).and_then(|mut t| crate::browser::elements(&mut t)) { + if let Ok((_, _, list, _)) = crate::browser::elements(&mut tab) { if !list.is_empty() { out.push_str("\n\n可以操作的元素（用 browser_click、browser_type、browser_select 等工具的 ref；連結的網址在 → 後面，不要自己猜網址；看不懂時用 browser_screenshot）：\n"); out.push_str(&list.join("\n")); @@ -452,8 +418,9 @@ mod tests { #[test] #[ignore] fn reads_a_real_page() { + // Opening hands back the loaded page itself (no browser_read needed). let r = open("https://example.com", None).unwrap(); - assert!(r.contains("example.com"), "{r}"); + assert!(r.contains("example.com") && r.contains("Example Domain") && r.contains("[1] 連結"), "{r}"); let page = read(2000, None).unwrap(); assert!(page.contains("Example Domain"), "{page}"); println!("{page}"); @@ -464,16 +431,6 @@ mod tests { assert_eq!(encode_component("https://www.threads.net/@me?x=1&y=2"), "https://www.threads.net/%40me%3Fx%3D1%26y%3D2"); } - #[test] - fn first_real_page_is_read() { - let list = serde_json::json!([ - {"type": "service_worker", "url": "https://x", "webSocketDebuggerUrl": "ws://a"}, - {"type": "page", "url": "devtools://devtools", "webSocketDebuggerUrl": "ws://b"}, - {"type": "page", "url": "https://www.threads.net/", "title": "Threads", "webSocketDebuggerUrl": "ws://c"} - ]); - assert_eq!(pick_page(&list).unwrap(), ("ws://c".to_string(), "Threads".to_string(), "https://www.threads.net/".to_string())); - } - #[test] fn profiles_are_found_by_name() { let list = vec![DEFAULT_PROFILE.to_string(), "工作".to_string(), "Personal".to_string()]; diff --git a/src/browser.rs b/src/browser.rs index de705cd..027a0de 100644 --- a/src/browser.rs +++ b/src/browser.rs @@ -28,7 +28,12 @@ pub struct Tab { fn targets(port: u16) -> Result, String> { let v: Value = botbrowser::local().get(&format!("http://127.0.0.1:{port}/json/list")).call().map_err(|e| e.to_string())?.into_body().read_json().map_err(|e| e.to_string())?; - Ok(v.as_array().cloned().unwrap_or_default().into_iter().filter(|t| t["type"] == "page" && !t["url"].as_str().unwrap_or("").starts_with("devtools://")).collect()) + Ok(pages(&v)) +} + +/// The ordinary page tabs of a /json/list answer, most recently used first. +fn pages(list: &Value) -> Vec { + list.as_array().cloned().unwrap_or_default().into_iter().filter(|t| t["type"] == "page" && !t["url"].as_str().unwrap_or("").starts_with("devtools://")).collect() } /// The browser's port for a profile (starting it if needed). @@ -305,6 +310,56 @@ fn settle(ms: u64) { std::thread::sleep(Duration::from_millis(ms)); } +/// What the page shows, cheaply: load state, address and how much text. +const SIG_JS: &str = "document.readyState+'|'+location.href+'|'+(document.body?document.body.innerText.length:0)"; + +fn page_sig(tab: &mut Tab) -> String { + tab.eval(SIG_JS).ok().and_then(|v| v.as_str().map(String::from)).unwrap_or_default() +} + +/// Wait for an action's effect, not a fixed time: back as soon as the page has changed and holds +/// still (loaded, same address and text twice in a row), or when nothing has started to change +/// within `quiet_ms`; never longer than `max_ms`. (Fixed sleeps read slow pages too early and +/// kept the bot waiting on fast ones.) +pub fn settle_after(tab: &mut Tab, before: &str, quiet_ms: u64, max_ms: u64) { + let start = Instant::now(); + let mut last = String::new(); + let mut changed = false; + loop { + settle(150); + let now = page_sig(tab); + changed |= now != before; + let elapsed = start.elapsed().as_millis() as u64; + let still = now.starts_with("complete|") && now == last; + if (still && (changed || elapsed >= quiet_ms)) || elapsed >= max_ms { + return; + } + last = now; + } +} + +/// Wait until the tab in front has loaded a real page (not the blank tab it starts as) and holds +/// still, at most `max_ms`. Says whether it did. Connects afresh each look: a navigation can swap +/// the page behind the tab. +pub fn wait_loaded(profile: Option<&str>, max_ms: u64) -> bool { + let start = Instant::now(); + let mut last = String::new(); + while (start.elapsed().as_millis() as u64) < max_ms { + let now = Tab::current(profile).map(|mut t| { + let sig = page_sig(&mut t); + t.close(); + sig + }); + let now = now.unwrap_or_default(); + if now.starts_with("complete|") && !now.contains("|about:blank|") && now == last { + return true; + } + last = now; + settle(200); + } + false +} + // ---------- the tools ---------- fn s(args: &Value, k: &str) -> String { @@ -455,6 +510,7 @@ pub fn run(name: &str, args: &Value) -> Result<(String, Option), String> Ok((format!("標題：{title}\n網址：{url}\n\n{}{}", list.join("\n"), if more { "\n…還有更多（捲動後再列一次）" } else { "" }), None)) } "browser_click" | "browser_hover" => { + let sig = page_sig(&mut tab); let before = snapshot(&mut tab); let (x, y, what) = locate(&mut tab, &rf, &text)?; settle(150); @@ -476,11 +532,12 @@ pub fn run(name: &str, args: &Value) -> Result<(String, Option), String> }; click_at(&mut tab, x, y, button, if args["double"].as_bool().unwrap_or(false) { 2 } else { 1 })?; } - settle(1500); + settle_after(&mut tab, &sig, 900, 4000); let after = snapshot(&mut tab); Ok((format!("已{} {what}。\n{}", if name == "browser_hover" { "停在" } else { "點" }, changes(&before, &after)), None)) } "browser_type" => { + let sig = page_sig(&mut tab); let before = snapshot(&mut tab); let (x, y, what) = locate(&mut tab, &rf, &text)?; click_at(&mut tab, x, y, "left", 1)?; @@ -495,12 +552,13 @@ pub fn run(name: &str, args: &Value) -> Result<(String, Option), String> settle(150); press(&mut tab, "Enter")?; } - settle(1200); + settle_after(&mut tab, &sig, 700, 4000); let after = snapshot(&mut tab); let now: String = tab.eval("(() => { const e = document.activeElement; return e && 'value' in e ? (e.type === 'password' ? '' : String(e.value)) : ''; })()")?.as_str().unwrap_or("").chars().take(80).collect(); Ok((format!("已在 {what} 輸入。{}\n{}", if now.is_empty() { String::new() } else { format!("欄位現在是「{now}」。") }, changes(&before, &after)), None)) } "browser_press" => { + let sig = page_sig(&mut tab); let before = snapshot(&mut tab); if !rf.is_empty() || !text.is_empty() { let (x, y, _) = locate(&mut tab, &rf, &text)?; @@ -508,12 +566,13 @@ pub fn run(name: &str, args: &Value) -> Result<(String, Option), String> settle(150); } press(&mut tab, &s(args, "key"))?; - settle(1000); + settle_after(&mut tab, &sig, 700, 4000); let after = snapshot(&mut tab); Ok((format!("已按 {}。\n{}", s(args, "key"), changes(&before, &after)), None)) } "browser_select" => { let option = s(args, "option"); + let sig = page_sig(&mut tab); let before = snapshot(&mut tab); let js = format!( r#"(() => {{ const el = {}; if (!el) return 'missing'; if (el.tagName !== 'SELECT') return 'custom'; @@ -539,11 +598,12 @@ pub fn run(name: &str, args: &Value) -> Result<(String, Option), String> let (ox, oy, _) = locate(&mut tab, "", &option).map_err(|_| format!("已打開選單，但找不到「{option}」這個選項。用 browser_elements 看選單裡有什麼。"))?; click_at(&mut tab, ox, oy, "left", 1)?; } - settle(1200); + settle_after(&mut tab, &sig, 700, 4000); let after = snapshot(&mut tab); Ok((format!("已選「{option}」。\n{}", changes(&before, &after)), None)) } "browser_scroll" => { + let sig = page_sig(&mut tab); let before = snapshot(&mut tab); if !rf.is_empty() || !text.is_empty() { let (_, _, what) = locate(&mut tab, &rf, &text)?; @@ -559,7 +619,7 @@ pub fn run(name: &str, args: &Value) -> Result<(String, Option), String> }; // Pages that scroll an inner box: scroll the biggest scrollable element too. tab.eval(&format!("(() => {{ {js}; const boxes = [...document.querySelectorAll('*')].filter(e => e.scrollHeight > e.clientHeight + 40 && /(auto|scroll)/.test(getComputedStyle(e).overflowY)).sort((a, b) => b.clientHeight - a.clientHeight); if (boxes[0]) {{ const b = boxes[0]; const d = '{}'; if (d === 'top') b.scrollTop = 0; else if (d === 'bottom') b.scrollTop = b.scrollHeight; else b.scrollTop += (d === 'up' ? -1 : 1) * b.clientHeight * 0.85; }} return 1; }})()", s(args, "direction")))?; - settle(1200); + settle_after(&mut tab, &sig, 500, 2500); let after = snapshot(&mut tab); let pos = tab.eval("JSON.stringify([Math.round(scrollY), document.documentElement.scrollHeight, innerHeight])")?; Ok((format!("已捲動（位置 {}）。\n{}", pos.as_str().unwrap_or(""), changes(&before, &after)), None)) @@ -631,13 +691,14 @@ pub fn run(name: &str, args: &Value) -> Result<(String, Option), String> Ok((text_out, Some(b64))) } "browser_back" => { + let sig = page_sig(&mut tab); let before = snapshot(&mut tab); match s(args, "direction").as_str() { "forward" => tab.eval("history.forward()").map(|_| ())?, "reload" => tab.call("Page.reload", json!({})).map(|_| ())?, _ => tab.eval("history.back()").map(|_| ())?, } - settle(2000); + settle_after(&mut tab, &sig, 900, 5000); let after = snapshot(&mut tab); Ok((format!("好了。\n{}", changes(&before, &after)), None)) } @@ -752,8 +813,8 @@ fn tabs(args: &Value, profile: Option<&str>) -> Result { "new" => { let url = crate::web::normalize_url(&s(args, "url")).unwrap_or_else(|_| "about:blank".into()); http(&format!("/json/new?{}", botbrowser::encode_component(&url)), true)?; - settle(2500); - Ok(format!("已開新分頁：{url}")) + let loaded = wait_loaded(profile, 10_000); + Ok(format!("已開新分頁：{url}{}", if loaded { "" } else { "（還在載入，可以用 browser_wait 等它）" })) } _ => Ok(if list.is_empty() { "沒有開著的分頁。".into() @@ -836,6 +897,19 @@ mod tests { assert_eq!(NAMES.len(), specs(&json!({"type": "string"})).len()); } + #[test] + fn the_front_page_tab_is_used() { + let list = json!([ + {"type": "service_worker", "url": "https://x", "webSocketDebuggerUrl": "ws://a"}, + {"type": "page", "url": "devtools://devtools", "webSocketDebuggerUrl": "ws://b"}, + {"type": "page", "url": "https://www.threads.net/", "title": "Threads", "webSocketDebuggerUrl": "ws://c"}, + {"type": "page", "url": "https://example.com/", "webSocketDebuggerUrl": "ws://d"} + ]); + let p = pages(&list); + assert_eq!(p.len(), 2); + assert_eq!(p[0]["webSocketDebuggerUrl"], "ws://c"); + } + /// A dashboard like Shopee's affiliate pages: a React-style date picker that only reacts to /// real clicks, a custom dropdown, a table, a hover menu, an export. Run with ANYBOT_DATA_DIR /// set to a scratch folder; opens a browser window for a moment. diff --git a/src/cloudbox.rs b/src/cloudbox.rs index 7fa230f..59d9c94 100644 --- a/src/cloudbox.rs +++ b/src/cloudbox.rs @@ -811,12 +811,27 @@ pub fn browser_port() -> Result { static START_LOCK: Mutex<()> = Mutex::new(()); /// Make sure the box is up, waking or creating it as needed. Blocks until it's ready. +/// The box start (Docker host and StartedAt) last seen ready. While the box is running from that +/// same start, a step needs one `docker inspect` instead of four calls (version, inspect, the +/// ready probe, chown) — the ready file only matters right after a start. +static READY_START: Mutex = Mutex::new(String::new()); + +fn box_start() -> Option { + let s = dk(&["inspect", "-f", "{{.State.Status}} {{.State.StartedAt}}", container()]).ok()?; + Some(format!("{}|{}", docker_host(), s.trim())) +} + pub fn ensure_running() -> Result<(), String> { let _g = START_LOCK.lock().unwrap(); if shared().lock().unwrap().rebuilding { return Err("Bot 的電腦正在更新，請等它完成再試。".into()); } touch(); + let known = READY_START.lock().unwrap_or_else(|e| e.into_inner()).clone(); + if !known.is_empty() && box_start().as_deref() == Some(known.as_str()) { + set_run(Run::Running); + return Ok(()); + } let r = probe(); let result = match r { Run::Running => Ok(()), @@ -859,6 +874,7 @@ pub fn ensure_running() -> Result<(), String> { Ok(()) => { set_run(Run::Running); let _ = dk(&["exec", container(), "sh", "-c", &format!("mkdir -p {HOME} && chown bot:bot {HOME}")]); + *READY_START.lock().unwrap_or_else(|e| e.into_inner()) = box_start().filter(|s| s.contains("|running ")).unwrap_or_default(); } Err(e) => { log(e); @@ -1424,7 +1440,7 @@ fn box_scale() -> Result<(f64, f64), String> { /// times `scale` (desktop pixels per picture pixel; 1 for a person's viewer, which shows the /// desktop at full size). `inline` puts typed text inside the script quoted (a batch runs its /// steps in one exec with one stdin, so steps can't ride on it). -pub fn box_script(name: &str, v: &Value, scale: (f64, f64), inline: bool) -> Result<(String, Option, String), String> { +pub fn box_script(name: &str, v: &Value, scale: (f64, f64)) -> Result<(String, String), String> { // A control's number from the latest picture: its centre on the desktop. let by_number = |p: &Value| -> Option> { let r = match &p["ref"] { @@ -1465,9 +1481,9 @@ pub fn box_script(name: &str, v: &Value, scale: (f64, f64), inline: bool) -> Res Ok(match name { "computer_use_click" => { let n = v["count"].as_u64().unwrap_or(1).clamp(1, 3); - (wrap(format!("{} click --repeat {n} --delay {CLICK_REPEAT_MS} {button}", mv(pt(v)?))), None, format!("已{}{} {}。", if button == "3" { "右鍵" } else { "" }, if n >= 2 { format!("連點 {n} 下") } else { "點".into() }, at(v))) + (wrap(format!("{} click --repeat {n} --delay {CLICK_REPEAT_MS} {button}", mv(pt(v)?))), format!("已{}{} {}。", if button == "3" { "右鍵" } else { "" }, if n >= 2 { format!("連點 {n} 下") } else { "點".into() }, at(v))) } - "computer_use_mouse_move" => (format!("xdotool {}", mv(pt(v)?)), None, format!("已把滑鼠移到 {}。", at(v))), + "computer_use_mouse_move" => (format!("xdotool {}", mv(pt(v)?)), format!("已把滑鼠移到 {}。", at(v))), "computer_use_scroll" => { let amount = crate::computer::scroll_amount(v["amount"].as_i64()); let dir = v["direction"].as_str().unwrap_or("").trim().to_lowercase(); @@ -1479,7 +1495,7 @@ pub fn box_script(name: &str, v: &Value, scale: (f64, f64), inline: bool) -> Res _ => return Err(format!("方向「{dir}」要是 up、down、left 或 right")), }; let go = if v["x"].is_null() && v["ref"].is_null() { String::new() } else { format!("{} ", mv(pt(v)?)) }; - (wrap(format!("{go}click --repeat {amount} --delay 20 {wheel}")), None, format!("已捲動 {dir} ×{amount}。")) + (wrap(format!("{go}click --repeat {amount} --delay 20 {wheel}")), format!("已捲動 {dir} ×{amount}。")) } "computer_use_drag" => { // A point that fails to parse is an error, not a silently dropped waypoint. @@ -1494,7 +1510,7 @@ pub fn box_script(name: &str, v: &Value, scale: (f64, f64), inline: bool) -> Res body.push_str(&format!(" mousemove {} {} sleep 0.008", x.round() as i64, y.round() as i64)); } body.push_str(&format!(" mouseup {button}")); - (wrap(body), None, format!("已從 {} 拖曳到 {}。", at(&raw[0]), at(&raw[raw.len() - 1]))) + (wrap(body), format!("已從 {} 拖曳到 {}。", at(&raw[0]), at(&raw[raw.len() - 1]))) } "computer_use_typing" => { let text = v["value"].as_str().unwrap_or("").to_string(); @@ -1513,13 +1529,13 @@ pub fn box_script(name: &str, v: &Value, scale: (f64, f64), inline: bool) -> Res None => String::new(), }; if !text.is_empty() { - script.push_str(&format!(" && {}", if inline { type_script_inline(&text) } else { type_script(&text) })); + script.push_str(&format!(" && {}", type_script(&text))); } if submit { script.push_str(" && sleep 0.06 && xdotool key --clearmodifiers Return"); } let script = script.trim_start_matches(" && ").to_string(); - (script, if inline { None } else { Some(text.clone()).filter(|t| !t.is_empty()) }, format!("已打字（{} 個字）{field}{}。", text.chars().count(), if submit { "，再按了 return" } else { "" })) + (script, format!("已打字（{} 個字）{field}{}。", text.chars().count(), if submit { "，再按了 return" } else { "" })) } "computer_use_press_key" => { let keys = v["key"].as_str().unwrap_or(""); @@ -1530,14 +1546,16 @@ pub fn box_script(name: &str, v: &Value, scale: (f64, f64), inline: bool) -> Res Ok(c) if hold > 0 => parts.push(format!("keydown {c} sleep {:.3} keyup {c}", hold as f64 / 1000.0)), Ok(c) => parts.push(format!("key --clearmodifiers {c}")), // 「@」「？」: a character, typed. - Err(_) if k.chars().count() == 1 => parts.push(format!("type --clearmodifiers -- {}", crate::tools::sh_quote(k))), + // `--args 1`: a plain `type` swallows every word after it as more text, so + // 「@ Return」 typed "@sleep0.02key…" and never pressed Return. + Err(_) if k.chars().count() == 1 => parts.push(format!("type --clearmodifiers --args 1 -- {}", crate::tools::sh_quote(k))), Err(e) => return Err(e), } } if parts.is_empty() { return Err("請給要按的鍵".into()); } - (format!("xdotool {}", parts.join(" sleep 0.02 ")), None, format!("已按 {keys}。")) + (format!("xdotool {}", parts.join(" sleep 0.02 ")), format!("已按 {keys}。")) } other => return Err(format!("沒有「{other}」這個工具")), }) @@ -1547,25 +1565,21 @@ pub fn box_script(name: &str, v: &Value, scale: (f64, f64), inline: bool) -> Res /// (Chinese, emoji) is pasted, because X keyboards can't type characters they have no key for in /// every program: both selections hold it (xclip keeps serving after the exec ends, setsid), and /// terminals paste with shift+Insert, everything else with ctrl+v. +/// Type text into the window in front: short plain text key by key; anything else (another +/// language, or long text, which key by key could outlast the step's 30 s and then get typed +/// twice when the bot tried again) pasted through the clipboard. fn type_script(text: &str) -> String { - if text.is_ascii() { - return format!("xdotool type --clearmodifiers --delay {TYPE_DELAY_MS} --file -"); - } - "cat > /tmp/anybot-paste.txt && for s in clipboard primary; do setsid xclip -selection $s -i /tmp/anybot-paste.txt /dev/null 2>&1 & done; sleep 0.15; \ - case \"$(xprop -id $(xdotool getactivewindow) WM_CLASS 2>/dev/null | tr A-Z a-z)\" in *term*) k=shift+Insert;; *) k=ctrl+v;; esac; xdotool key --clearmodifiers $k" - .to_string() -} - -/// The same, with the text quoted into the script (batch steps: one stdin for the whole chain). -fn type_script_inline(text: &str) -> String { let q = crate::tools::sh_quote(text); - if text.is_ascii() { + if text.is_ascii() && text.chars().count() <= TYPE_KEYS_MAX_CHARS { return format!("xdotool type --clearmodifiers --delay {TYPE_DELAY_MS} -- {q}"); } format!("printf '%s' {q} > /tmp/anybot-paste.txt && for s in clipboard primary; do setsid xclip -selection $s -i /tmp/anybot-paste.txt /dev/null 2>&1 & done; sleep 0.15; \ case \"$(xprop -id $(xdotool getactivewindow) WM_CLASS 2>/dev/null | tr A-Z a-z)\" in *term*) k=shift+Insert;; *) k=ctrl+v;; esac; xdotool key --clearmodifiers $k") } +/// Longest text typed key by key (more is pasted). +const TYPE_KEYS_MAX_CHARS: usize = 200; + /// Wait for the screen to stop changing before the picture comes back: two matching probes mean /// it settled. A probe is two small grayscale captures (top-left, center: ~60 ms on the box), so /// a settled screen takes ~250 ms where every step used to pay a fixed 0.5 s — and a page that @@ -1573,24 +1587,15 @@ fn type_script_inline(text: &str) -> String { /// when `import` is missing. const SETTLE_SH: &str = "{ probe() { { import -window root -crop 160x140+80+30 -depth 8 pgm:-; import -window root -crop 160x160+560+320 -depth 8 pgm:-; } 2>/dev/null | md5sum; }; if command -v import >/dev/null 2>&1; then a=$(probe); i=0; while [ $i -lt 5 ]; do sleep 0.1; b=$(probe); [ \"$a\" = \"$b\" ] && break; a=$b; i=$((i+1)); done; else sleep 0.35; fi; }"; -/// Run a step's script in the box; with `picture`, wait for the screen to settle and return it. -fn box_exec(script: &str, stdin: Option<&str>, picture: bool) -> Result, String> { - let full = if picture { format!("{{ {script}; }} || exit $?; {SETTLE_SH}; import -window root -quality 90 jpeg:- 2>/dev/null || true") } else { script.to_string() }; - // Always an stdin (closed at once when there's no text), as `docker exec -i` expects. - let out = match dk_with_input(&["exec", "-i", "-u", "bot", "-e", "DISPLAY=:1", container(), "sh", "-c", &full], Some(stdin.unwrap_or("").as_bytes()), 30_000) { - Ok(o) => o, - Err(e) if e == "逾時" => return Err("這台電腦太久沒有回應".into()), - Err(e) if e.is_empty() => return Err("這一步沒有成功".into()), - Err(e) => return Err(format!("xdotool：{e}")), - }; - if !picture || out.is_empty() { - return Ok(None); +/// Run a script in the box (as the bot user, on its desktop). Gives back what it printed. +fn box_exec(script: &str) -> Result { + // An stdin closed at once, as `docker exec -i` expects. + match dk_with_input(&["exec", "-i", "-u", "bot", "-e", "DISPLAY=:1", container(), "sh", "-c", script], Some(b""), 30_000) { + Ok(o) => Ok(String::from_utf8_lossy(&o).to_string()), + Err(e) if e == "逾時" => Err("這台電腦太久沒有回應".into()), + Err(e) if e.is_empty() => Err("這一步沒有成功".into()), + Err(e) => Err(format!("xdotool：{e}")), } - let img = image::load_from_memory(&out).map(|i| i.to_rgb8()).ok(); - if let Some(i) = &img { - shared().lock().unwrap_or_else(|e| e.into_inner()).size = Some(i.dimensions()); - } - Ok(img) } /// The numbered controls of the box's latest look (centres in desktop pixels). @@ -1661,7 +1666,7 @@ pub fn elements_of(v: &Value) -> Vec { /// Do a step's script (if any), wait for the screen to settle, and look: the picture with the /// controls numbered and the list, in one `docker exec`. After a step it also says when nothing /// changed, so the bot doesn't click the same empty place again and again. -fn box_look(script: Option<(&str, Option<&str>)>, after: Option) -> Result<(String, Option), String> { +fn box_look(script: Option<&str>, after: Option, pointer: bool) -> Result<(String, Option), String> { let t_all = Instant::now(); // The picture is captured by a background `import` while the controls are read (the a11y // read is the slow part on web pages); a JPEG costs less to make and to ship than a PNG. @@ -1669,11 +1674,10 @@ fn box_look(script: Option<(&str, Option<&str>)>, after: Option) -> Resu sleep 0.18; anybot-a11y --budget 3 > /tmp/anybot-look.json 2>/dev/null || echo '{\"elements\":[]}' > /tmp/anybot-look.json; \ wc -c < /tmp/anybot-look.json; cat /tmp/anybot-look.json; wait $ip; cat /tmp/anybot-look.jpg )"; let full = match script { - Some((s, _)) => format!("{{ {s}; }} || exit $?; {SETTLE_SH}; {look}"), + Some(s) => format!("{{ {s}; }} || exit $?; {SETTLE_SH}; {look}"), None => look.to_string(), }; - let input = script.and_then(|(_, i)| i).unwrap_or(""); - let out = match dk_with_input(&["exec", "-i", "-u", "bot", "-e", "DISPLAY=:1", container(), "sh", "-c", &full], Some(input.as_bytes()), 30_000) { + let out = match dk_with_input(&["exec", "-i", "-u", "bot", "-e", "DISPLAY=:1", container(), "sh", "-c", &full], Some(b""), 30_000) { Ok(o) => o, Err(e) if e == "逾時" => return Err("這台電腦太久沒有回應".into()), Err(e) if e.is_empty() => return Err("這一步沒有成功".into()), @@ -1700,7 +1704,7 @@ fn box_look(script: Option<(&str, Option<&str>)>, after: Option) -> Resu text.push_str(&format!("螢幕截圖（{fw}×{fh}，Linux 桌面）。")); let unchanged = { let mut last = BOX_LAST.lock().unwrap_or_else(|e| e.into_inner()); - let same = after.is_some() && *last == Some(fp); + let same = pointer && after.is_some() && *last == Some(fp); *last = Some(fp); same }; @@ -1771,19 +1775,24 @@ fn open_on_box(app: &str) -> Result { "w=$(xdotool search --onlyvisible --classname {q} 2>/dev/null | tail -1); \ if [ -z \"$w\" ]; then command -v {q} >/dev/null || {{ echo \"沒有 {prog} 這個程式\" >&2; exit 3; }}; (setsid {q} >/dev/null 2>&1 &); \ for i in $(seq 1 40); do sleep 0.2; w=$(xdotool search --onlyvisible --classname {q} 2>/dev/null | tail -1); [ -n \"$w\" ] && break; done; fi; \ - [ -n \"$w\" ] && xdotool windowactivate $w 2>/dev/null; true" + if [ -n \"$w\" ]; then xdotool windowactivate $w 2>/dev/null; echo shown; fi; true" ); - box_exec(&script, None, false)?; - Ok(format!("已打開「{prog}」，放在最前面。")) + let out = box_exec(&script)?; + Ok(if out.contains("shown") { format!("已打開「{prog}」，放在最前面。") } else { format!("已啟動「{prog}」，但 8 秒內還沒有出現視窗（可能還在開，看下面的畫面）。") }) } /// Run a computer_use tool on the box. Returns (text, screenshot as base64 JPEG). +/// One screen step on the box at a time: two bots acting at once shared the picture files and the +/// numbers (a torn picture, or a number resolved against the other bot's picture). +static BOX_STEP: Mutex<()> = Mutex::new(()); + pub fn computer_run(name: &str, v: &Value) -> Result<(String, Option), String> { + let _one = BOX_STEP.lock().unwrap_or_else(|e| e.into_inner()); ensure_running()?; let _busy = Busy::new(true); match name { "computer_use_check_permissions" => return Ok(("這台電腦不需要另外的權限：可以截圖，也可以操作滑鼠和鍵盤。".into(), None)), - "computer_use_screenshot" => return box_look(None, None), + "computer_use_screenshot" => return box_look(None, None, false), _ => {} } if HELD_BY_PERSON.load(Ordering::SeqCst) || crate::vnc::person_active() { @@ -1791,13 +1800,13 @@ pub fn computer_run(name: &str, v: &Value) -> Result<(String, Option), S } if name == "computer_use_open_app" { let note = open_on_box(v["app"].as_str().unwrap_or(""))?; - return box_look(Some(("true", None)), Some(note)); + return box_look(None, Some(note), false); } if name == "computer_use_batch" { return box_batch(v, box_scale()?); } - let (script, stdin, note) = box_script(name, v, box_scale()?, false)?; - box_look(Some((&script, stdin.as_deref())), Some(note)) + let (script, note) = box_script(name, v, box_scale()?)?; + box_look(Some(&script), Some(note), crate::computer::is_pointer_step(name)) } const BATCH_MARK: &str = "/tmp/anybot-batch.done"; @@ -1818,24 +1827,25 @@ fn box_batch(v: &Value, scale: (f64, f64)) -> Result<(String, Option), S let mut notes = Vec::new(); for (i, a) in acts.iter().enumerate() { let name = crate::computer::action_tool(a["action"].as_str().unwrap_or(""))?; - let (script, _, note) = box_script(name, a, scale, true)?; + let (script, note) = box_script(name, a, scale)?; parts.push(format!("{{ {script} && echo x >> {BATCH_MARK}; }}")); notes.push(format!("{}. {note}", i + 1)); } - box_exec(&format!("rm -f {BATCH_MARK}; {} ; true", parts.join(" && ")), None, false)?; - let done = dk(&["exec", container(), "sh", "-c", &format!("wc -l < {BATCH_MARK} 2>/dev/null || echo 0")]) + // The chain runs in the same call as the picture; one more call reads how far it got. + let chain = format!("rm -f {BATCH_MARK}; {} ; true", parts.join(" && ")); + let (text, pic) = box_look(Some(&chain), Some(String::new()), false)?; + let done = dk(&["exec", container(), "sh", "-c", &format!("wc -l < {BATCH_MARK} 2>/dev/null || echo 0; rm -f {BATCH_MARK}")]) .ok() .and_then(|s| s.trim().parse::().ok()) .unwrap_or(0) .min(acts.len()); - let _ = dk(&["exec", container(), "sh", "-c", &format!("rm -f {BATCH_MARK}")]); let note = if done == acts.len() { format!("已依序做完 {done} 步：\n{}", notes.join("\n")) } else { let ran: Vec<&str> = notes.iter().take(done).map(String::as_str).collect(); format!("做完 {done}/{} 步，第 {} 步沒有成功（後面沒有做）：\n{}", acts.len(), done + 1, ran.join("\n")) }; - box_look(Some(("true", None)), Some(note)) + Ok((format!("{note}{text}"), pic)) } // ---------- text a person types in a viewer ---------- @@ -1844,7 +1854,7 @@ fn box_batch(v: &Value, scale: (f64, f64)) -> Result<(String, Option), S /// the bot's typing. (Their mouse and keys go through VNC itself.) pub fn viewer_text(text: &str) { if !text.is_empty() { - let _ = box_exec(&type_script(text), Some(text), false); + let _ = box_exec(&type_script(text)); } touch(); } @@ -1995,40 +2005,34 @@ mod tests { let s = (1.25, 1.25); // Picture pixels → desktop pixels; never `mousemove --sync` (it hangs when the pointer is // already there). - let (sc, input, note) = box_script("computer_use_click", &serde_json::json!({"x": 512, "y": 320}), s, false).unwrap(); + let (sc, note) = box_script("computer_use_click", &serde_json::json!({"x": 512, "y": 320}), s).unwrap(); assert_eq!(sc, "xdotool mousemove 640 400 sleep 0.03 click --repeat 1 --delay 80 1"); - assert!(input.is_none() && note.contains("(512, 320)")); + assert!(note.contains("(512, 320)")); // Keys held for a step are let go even when it fails. - let (sc, _, _) = box_script("computer_use_click", &serde_json::json!({"x": 8, "y": 8, "modifier_keys": ["cmd", "shift"], "count": 2}), s, false).unwrap(); + let (sc, _) = box_script("computer_use_click", &serde_json::json!({"x": 8, "y": 8, "modifier_keys": ["cmd", "shift"], "count": 2}), s).unwrap(); assert!(sc.starts_with("xdotool keydown ctrl keydown shift mousemove 10 10") && sc.ends_with("rc=$?; xdotool keyup shift keyup ctrl; [ $rc = 0 ]"), "{sc}"); assert!(!sc.contains("--sync")); - // Text goes in on stdin (no quoting to get wrong); return after it when asked. - let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "echo 'hi' && ls", "submit": true}), s, false).unwrap(); - assert_eq!(input.as_deref(), Some("echo 'hi' && ls")); - assert!(sc.starts_with("xdotool type --clearmodifiers --delay 12 --file -") && sc.ends_with("xdotool key --clearmodifiers Return"), "{sc}"); - let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "中文"}), s, false).unwrap(); - assert!(sc.contains("xclip") && input.as_deref() == Some("中文")); - // Keys by name, a character typed, several in order. - let (sc, _, _) = box_script("computer_use_press_key", &serde_json::json!({"key": "slash @ cmd+a"}), s, false).unwrap(); - assert_eq!(sc, "xdotool key --clearmodifiers slash sleep 0.02 type --clearmodifiers -- '@' sleep 0.02 key --clearmodifiers ctrl+a"); + // Text rides in the script, quoted; return after it when asked. + let (sc, _) = box_script("computer_use_typing", &serde_json::json!({"value": "it's a 'test'", "submit": true}), s).unwrap(); + assert!(sc.starts_with(r"xdotool type --clearmodifiers --delay 12 -- 'it'\''s a '\''test'\'''") && sc.ends_with("xdotool key --clearmodifiers Return"), "{sc}"); + // Another language, or long text: pasted (key by key, 2,000 letters outlasted the step). + let (sc, _) = box_script("computer_use_typing", &serde_json::json!({"value": "中文"}), s).unwrap(); + assert!(sc.contains("printf '%s' '中文' > /tmp/anybot-paste.txt") && sc.contains("xclip"), "{sc}"); + let (sc, _) = box_script("computer_use_typing", &serde_json::json!({"value": "a".repeat(TYPE_KEYS_MAX_CHARS + 1)}), s).unwrap(); + assert!(sc.contains("xclip") && !sc.contains("xdotool type"), "{sc}"); + // Keys by name, a character typed (without swallowing the keys after it), several in order. + let (sc, _) = box_script("computer_use_press_key", &serde_json::json!({"key": "slash @ cmd+a"}), s).unwrap(); + assert_eq!(sc, "xdotool key --clearmodifiers slash sleep 0.02 type --clearmodifiers --args 1 -- '@' sleep 0.02 key --clearmodifiers ctrl+a"); // A drag walks its path in small steps and lets go at the end. - let (sc, _, _) = box_script("computer_use_drag", &serde_json::json!({"path": [{"x": 0, "y": 0}, {"x": 48, "y": 0}]}), s, false).unwrap(); + let (sc, _) = box_script("computer_use_drag", &serde_json::json!({"path": [{"x": 0, "y": 0}, {"x": 48, "y": 0}]}), s).unwrap(); assert!(sc.starts_with("xdotool mousemove 0 0 sleep 0.03 mousedown 1") && sc.contains("mousemove 60 0") && sc.ends_with("mouseup 1"), "{sc}"); // No app numbers on this desktop: said, not guessed. - assert!(box_script("computer_use_click", &serde_json::json!({"ref": "e3"}), s, false).is_err()); - assert!(box_script("computer_use_scroll", &serde_json::json!({"direction": "sideways"}), s, false).is_err()); + assert!(box_script("computer_use_click", &serde_json::json!({"ref": "e3"}), s).is_err()); + assert!(box_script("computer_use_scroll", &serde_json::json!({"direction": "sideways"}), s).is_err()); } #[test] - fn batch_steps_carry_their_text_inline() { - let s = (1.0, 1.0); - // A batch runs its steps in one exec with one stdin: the text rides in the script, quoted. - let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "it's a 'test'"}), s, true).unwrap(); - assert!(input.is_none(), "no stdin for a batch step"); - assert!(sc.starts_with("xdotool type --clearmodifiers --delay 12 -- 'it'\\''s a '\\''test'\\'''"), "{sc}"); - // Chinese in a batch: the paste path, text written inline. - let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "中文", "submit": true}), s, true).unwrap(); - assert!(input.is_none() && sc.contains("printf '%s' '中文' > /tmp/anybot-paste.txt") && sc.ends_with("xdotool key --clearmodifiers Return"), "{sc}"); + fn batch_action_names() { // Batch action names, and the ones a batch must not carry. assert_eq!(crate::computer::action_tool("Click").unwrap(), "computer_use_click"); assert_eq!(crate::computer::action_tool("typing").unwrap(), "computer_use_typing"); @@ -2094,6 +2098,48 @@ mod tests { assert!(again.contains("畫面和上一步完全一樣"), "{again}"); } + /// Numbers only on what can be seen: links scrolled up under the browser's toolbar get none (or + /// their visible part's centre), and nothing is numbered under a window that covers the page + /// (xterm has no accessibility; clicking a number there used to type into the terminal). + #[test] + #[ignore] + fn box_numbers_only_on_what_is_seen() { + ensure_running().expect("box starts"); + let cancel = AtomicBool::new(false); + let links: String = (1..=60).map(|i| format!("

L{i}

")).collect(); + let page = format!("seen{links}"); + dk_with_input(&["exec", "-i", "-u", "bot", container(), "sh", "-c", "mkdir -p /tmp/anybot-seen && cat > /tmp/anybot-seen/index.html && (cd /tmp/anybot-seen && setsid python3 -m http.server 8767 --bind 127.0.0.1 >/dev/null 2>&1 &) ; sleep 0.5"], Some(page.as_bytes()), 10_000).expect("page served"); + let opened = crate::botbrowser::in_box(|| crate::web::run_full("browser_open", r#"{"url":"http://127.0.0.1:8767/"}"#, &cancel)); + assert!(opened.1, "{}", opened.0); + // Scroll so a link straddles the top of the page area. + crate::botbrowser::in_box(|| crate::botbrowser::eval_current(None, "window.scrollTo(0, 1220); 1")).unwrap(); + std::thread::sleep(Duration::from_millis(500)); + let page_spots = || -> Vec<(f64, f64, String)> { + computer_run("computer_use_screenshot", &serde_json::json!({})).unwrap(); + let spots = BOX_SPOTS.lock().unwrap().clone().unwrap(); + spots.by_number.values().filter(|(_, w)| w.contains("「L")).map(|((x, y), w)| (*x, *y, w.clone())).collect() + }; + let top = crate::botbrowser::in_box(|| crate::botbrowser::eval_current(None, "window.outerHeight - window.innerHeight")).map(|(_, v)| v.as_f64().unwrap_or(87.0)).unwrap_or(87.0); + let seen = page_spots(); + assert!(!seen.is_empty()); + for (x, y, w) in &seen { + assert!(*y >= top && *y < 800.0, "{w} numbered at ({x}, {y}), above the page area (top {top})"); + } + assert!(!seen.iter().any(|(_, _, w)| w.contains("「L1」")), "L1 is scrolled away: {seen:?}"); + // A terminal over the page: nothing numbered under it. + // (Found by its process: the shell inside renames the window.) + let geo = box_exec("setsid xterm -geometry 60x20+0+300 >/dev/null 2>&1 & echo $! > /tmp/anybot-seen.pid; sleep 1.2; w=$(xdotool search --pid $(cat /tmp/anybot-seen.pid) | head -1); xdotool getwindowgeometry --shell $w | sed -n 's/^[A-Z]*=//p' | sed -n '2,5p' | tr '\\n' ' '").unwrap(); + let g: Vec = geo.split_whitespace().filter_map(|v| v.parse().ok()).collect(); + assert_eq!(g.len(), 4, "{geo}"); + let covered = page_spots(); + let _ = box_exec("kill $(cat /tmp/anybot-seen.pid); rm -f /tmp/anybot-seen.pid; pkill -f 'http[.]server 8767'; rm -rf /tmp/anybot-seen"); + for (x, y, w) in &covered { + assert!(!(*x >= g[0] && *x < g[0] + g[2] && *y >= g[1] && *y < g[1] + g[3]), "{w} numbered at ({x}, {y}) under the terminal {g:?}"); + } + assert!(covered.len() < seen.len(), "some links are under the terminal: {} vs {}", covered.len(), seen.len()); + println!("page links numbered: {} in view, {} with a terminal over the page (top of page area {top})", seen.len(), covered.len()); + } + /// Uses the real Docker: `cargo test --release -- --ignored box_screen_is_exact --nocapture`. /// The bot's clicks land where its picture says, the same spot twice doesn't hang, typing (ASCII, Chinese, a key by name) reaches the app, /// and every step comes back with the screen after it. @@ -2135,7 +2181,7 @@ mod tests { let out = dk(&["exec", "-u", "bot", container(), "cat", "/home/bot/exact.txt"]).unwrap_or_default(); println!("typed: {out}"); assert_eq!(out.trim(), "a-b 中文 /"); - let _ = box_exec("xdotool key --clearmodifiers ctrl+d", None, false); + let _ = box_exec("xdotool key --clearmodifiers ctrl+d"); } /// Uses the real Docker: `cargo test --release -- --ignored box_batch_in_one_round_trip --nocapture`. @@ -2168,7 +2214,7 @@ mod tests { assert!(computer_run("computer_use_batch", &serde_json::json!({"actions": []})).is_err()); let many: Vec<_> = (0..9).map(|_| serde_json::json!({"action": "press_key", "key": "f15"})).collect(); assert!(computer_run("computer_use_batch", &serde_json::json!({"actions": many})).is_err()); - let _ = box_exec("xdotool key --clearmodifiers ctrl+d", None, false); + let _ = box_exec("xdotool key --clearmodifiers ctrl+d"); } /// Uses the real Docker: `cargo test --release -- --ignored box_real_usage_drill --nocapture`. @@ -2395,7 +2441,7 @@ mod tests { assert!(!in_schedule_window("03:00", t("02:59"))); assert!(in_schedule_window("23:30", t("00:10"))); assert!(!in_schedule_window("bad", t("00:10"))); - assert_eq!(wanted_version(), "5"); + assert_eq!(wanted_version(), "6"); } #[test] diff --git a/src/computer.rs b/src/computer.rs index 462d5f0..41869c9 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -213,6 +213,13 @@ pub fn scale_here() -> (f64, f64) { (pw / w.max(1) as f64, ph / h.max(1) as f64) } +/// A click, scroll or drag: a step whose whole effect should show on the screen. Only these are +/// called out when the picture didn't change (typing a few letters or a key can change too little +/// for the coarse fingerprint, and the bot was told to stop typing that worked). +pub fn is_pointer_step(name: &str) -> bool { + matches!(name, "computer_use_click" | "computer_use_scroll" | "computer_use_drag") +} + pub fn is_computer_tool(name: &str) -> bool { NAMES.contains(&name) } @@ -268,7 +275,7 @@ pub fn describe(name: &str, args: &str) -> String { /// The screen tools, each with a `computer` choice. One way to see (the numbered picture), one /// way to act (a real click or keystroke, best by number). -pub fn specs_for(computers: &[String], default: &str) -> Vec { +pub fn specs_for(computers: &[String], default: &str, chosen: bool) -> Vec { let mods = json!({"type": "array", "items": {"type": "string", "enum": ["cmd", "shift", "option", "control"]}}); let rf = json!({"type": "string", "description": "The number of a control on the latest picture (e.g. 12). Always use a number when the control has one."}); let mut v = vec![ @@ -296,7 +303,10 @@ pub fn specs_for(computers: &[String], default: &str) -> Vec { // Always named: each computer goes by its own name (the instructions say which is which), // so the bot says whose screen it means and the person can tell where it worked. if !computers.is_empty() { - let desc = format!("Whose screen, by the computer's name. Default: {default} (the one the person chose). Use another only when the person names it."); + // The person's chosen computer may have no screen (a bot's computer without Docker): then + // the default is only the first screen there is, and the bot is told so. + let why = if chosen { "the one the person chose" } else { "the computer the person chose has no screen" }; + let desc = format!("Whose screen, by the computer's name. Default: {default} ({why}). Use another only when the person names it."); for t in v.iter_mut() { t["parameters"]["properties"]["computer"] = json!({"type": "string", "enum": computers, "description": desc}); } @@ -504,7 +514,7 @@ pub fn ask_permission(pane: &str) { } const NO_ACCESSIBILITY: &str = "macOS 還沒讓 AnyBot 使用「輔助使用」，所以不能讀 App，也不能動滑鼠和鍵盤。這台電腦上已經跳出詢問並打開「系統設定 → 隱私權與安全性 → 輔助使用」：請使用者把 AnyBot 打開（只要一次），再叫我試一次。"; -const NO_SCREEN_RECORDING: &str = "macOS 還沒讓 AnyBot 錄製螢幕，所以看不到畫面。這台電腦上已經打開「系統設定 → 隱私權與安全性 → 螢幕與系統錄音」：請使用者把 AnyBot 打開（只要一次）。在那之前可以用 computer_use_read_app 讀 App 的文字。"; +const NO_SCREEN_RECORDING: &str = "macOS 還沒讓 AnyBot 錄製螢幕，所以看不到畫面。這台電腦上已經打開「系統設定 → 隱私權與安全性 → 螢幕與系統錄音」：請使用者把 AnyBot 打開（只要一次）。在那之前 computer_use_screenshot 還是會列出最前面 App 的文字和按鈕編號（只是沒有圖）。"; /// Reading apps and using the mouse and keyboard need 輔助使用 on macOS (other systems: always). fn controls_allowed() -> bool { @@ -667,7 +677,7 @@ pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result { let button = v["button"].as_str().unwrap_or("left"); match name { "computer_use_click" => { - let (x, y, at) = place_of(v, scale)?.ok_or("請給 x、y（截圖上的位置）或 ref（computer_use_read_app 的編號）")?; + let (x, y, at) = place_of(v, scale)?.ok_or("請給 x、y（截圖上的位置）或 ref（截圖上的編號）")?; let count = v["count"].as_u64().unwrap_or(1).clamp(1, 3) as i64; mac::click(x, y, button, count, flags_of(v)); let how = format!("{}{}", if button == "right" { "右鍵" } else { "" }, if count >= 2 { format!("連點 {count} 下") } else { "點".into() }); @@ -747,14 +757,31 @@ pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result { /// numbered, and the list of them. Without a picture (no 螢幕錄製), the list with the app's text. /// After a step (`after`), it also says when nothing changed, so the bot doesn't click the same /// empty place again and again. -fn look(after: Option) -> Result<(String, Option), String> { +fn look(after: Option, pointer: bool) -> Result<(String, Option), String> { let picture = if mac::screen_capture_preflight() { Some(capture_screen().map_err(|e| format!("看不到畫面：{e}"))?) } else { None }; - let found = if controls_allowed() { crate::desktop::elements("").ok() } else { None }; + // The app in front: on Linux the window system's own answer when it gives one (agent-desktop + // alone goes by AT-SPI's "active" window, which some desktops never mark: no numbers, 「」). + let front = if cfg!(target_os = "linux") { mac::front_app().map(|(n, _)| n).unwrap_or_default() } else { String::new() }; + let (found, why) = if controls_allowed() { + match crate::desktop::elements(&front).or_else(|e| if front.is_empty() { Err(e) } else { crate::desktop::elements("") }) { + Ok(f) => (Some(f), String::new()), + Err(e) => (None, e), + } + } else { + (None, String::new()) + }; if picture.is_none() && found.is_none() { ask_permission("Privacy_ScreenCapture"); return Err(NO_SCREEN_RECORDING.into()); } let (app, window, mut els) = found.unwrap_or_default(); + // Only what is on this screen gets a number (a link below the fold was numbered at the + // picture's edge, and a click there landed on the edge). + let (sw, sh) = mac::screen_points(); + els.retain(|e| { + let (cx, cy) = (e.x + e.w / 2.0, e.y + e.h / 2.0); + sw <= 0.0 || (cx >= 0.0 && cy >= 0.0 && cx < sw && cy < sh) + }); // A number is a promise that a click there lands in this app: a control another window // covers (or one on another desktop) gets none. let hidden = cover_check(window, &mut els); @@ -776,7 +803,7 @@ fn look(after: Option) -> Result<(String, Option), String> { } let unchanged = { let mut last = LAST_LOOK.lock().unwrap_or_else(|e| e.into_inner()); - let same = after.is_some() && *last == Some(fp); + let same = pointer && after.is_some() && *last == Some(fp); *last = Some(fp); same }; @@ -786,7 +813,10 @@ fn look(after: Option) -> Result<(String, Option), String> { if hidden > 0 { text.push_str(&format!("\n（「{app}」有 {hidden} 個按鈕被別的視窗擋住或不在這個桌面，沒有編號：先 computer_use_open_app 把它叫到前面。）")); } - if lines.is_empty() { + if lines.is_empty() && app.is_empty() { + let why = if why.is_empty() { String::new() } else { format!("（{}）", why.lines().next().unwrap_or("")) }; + text.push_str(&format!("\n看不出最前面是哪個 App{why}，所以沒有編號：只好照圖上的位置點 x、y。")); + } else if lines.is_empty() { text.push_str(&format!("\n「{app}」沒有可以照編號點的東西（有些 App 不提供）：只好照圖上的位置點 x、y。")); } else { text.push_str(&format!("\n「{app}」上可以操作的東西（圖上同樣的編號，用 ref 點）：\n{}", lines.join("\n"))); @@ -818,9 +848,9 @@ fn cover_check(window: Option, els: &mut [crate::marks::Element]) -> usize } /// After a step: wait for the screen to settle, then look. -fn observe(note: String) -> (String, Option) { +fn observe(note: String, pointer: bool) -> (String, Option) { std::thread::sleep(std::time::Duration::from_millis(SETTLE_MS)); - match look(Some(note.clone())) { + match look(Some(note.clone()), pointer) { Ok(r) => r, Err(e) => (format!("{note}（看不到結果：{e}）"), None), } @@ -835,7 +865,7 @@ fn run_inner(name: &str, v: &Value) -> Result<(String, Option), String> return Err(crate::desk::describe().into()); } match name { - "computer_use_screenshot" => return look(None), + "computer_use_screenshot" => return look(None, false), "computer_use_check_permissions" => { #[cfg(target_os = "linux")] return Ok((crate::desk::describe().into(), None)); @@ -861,7 +891,7 @@ fn run_inner(name: &str, v: &Value) -> Result<(String, Option), String> "computer_use_batch" => batch(v, scale_here())?, _ => act(name, v, scale_here()).map_err(plain_words)?, }; - Ok(observe(note)) + Ok(observe(note, is_pointer_step(name))) } /// agent-desktop's error codes, as words the bot can pass on. @@ -1031,8 +1061,6 @@ mod mac { fn CGEventSetIntegerValueField(e: Ev, field: u32, v: i64); fn CGEventPost(tap: u32, e: Ev); fn CGEventCreateScrollWheelEvent(src: Src, units: u32, count: u32, wheel1: i32, ...) -> Ev; - fn CGDisplayPixelsWide(d: u32) -> usize; - fn CGDisplayPixelsHigh(d: u32) -> usize; fn CGWindowListCopyWindowInfo(option: u32, relative_to: u32) -> *const c_void; fn CGRectMakeWithDictionaryRepresentation(dict: *const c_void, rect: *mut CGRect) -> bool; static kCGWindowOwnerName: *const c_void; @@ -1127,12 +1155,10 @@ mod mac { } pub fn screen_pixels() -> (u32, u32) { - let d = unsafe { CGMainDisplayID() }; + // Points times the backing scale (CGDisplayPixelsWide reports points on Retina, while + // screencapture gives backing pixels). let (pw, ph) = screen_points(); - // CGDisplayPixelsWide reports points on Retina; screencapture gives backing pixels. - let (w, h) = unsafe { (CGDisplayPixelsWide(d) as f64, CGDisplayPixelsHigh(d) as f64) }; let scale = backing_scale(); - let _ = (w, h); ((pw * scale) as u32, (ph * scale) as u32) } diff --git a/src/desktop.rs b/src/desktop.rs index 9be5dd8..ee32104 100644 --- a/src/desktop.rs +++ b/src/desktop.rs @@ -133,8 +133,10 @@ pub fn elements_of(tree: &Value, app: &str) -> Vec { let b = &n["bounds"]; if let (Some(x), Some(y), Some(w), Some(h)) = (b["x"].as_f64(), b["y"].as_f64(), b["width"].as_f64(), b["height"].as_f64()) { let is_active = n["ref_id"].is_string() && active(role); - if w >= 1.0 && h >= 1.0 && x + w > 0.0 && y + h > 0.0 && (is_active || !name.is_empty() || !value.trim().is_empty()) && role != "window" && role != "group" { - let has = |k: &str| n["states"].as_array().map(|a| a.iter().any(|x| x == k)).unwrap_or(false) || n[k] == true; + let has = |k: &str| n["states"].as_array().map(|a| a.iter().any(|x| x == k)).unwrap_or(false) || n[k] == true; + // Hidden or scrolled out of view: nothing a click could reach. + let unseen = has("hidden") || has("offscreen"); + if !unseen && w >= 1.0 && h >= 1.0 && x + w > 0.0 && y + h > 0.0 && (is_active || !name.is_empty() || !value.trim().is_empty()) && role != "window" && role != "group" { let mut flags = vec![]; for (k, word) in [("checked", "已勾"), ("selected", "已選"), ("disabled", "停用"), ("focused", "游標在這")] { if has(k) && (k != "focused" || role.contains("text") || role.contains("field")) { @@ -177,6 +179,17 @@ pub fn launch(app: &str) -> Result<(), String> { mod tests { use super::*; + #[test] + fn hidden_and_offscreen_controls_get_no_number() { + let tree = serde_json::json!({"role": "window", "children": [ + {"role": "button", "name": "Close", "ref_id": "a", "bounds": {"x": 8.0, "y": 38.0, "width": 16.0, "height": 16.0}}, + {"role": "scrollbar", "name": "bar", "ref_id": "b", "states": ["disabled", "hidden"], "bounds": {"x": 1902.0, "y": 61.0, "width": 19.0, "height": 1000.0}}, + {"role": "link", "name": "Below the fold", "ref_id": "c", "states": ["offscreen"], "bounds": {"x": 20.0, "y": 2400.0, "width": 80.0, "height": 20.0}} + ]}); + let names: Vec = elements_of(&tree, "App").into_iter().map(|e| e.name).collect(); + assert_eq!(names, vec!["Close".to_string()]); + } + #[test] fn controls_and_text_come_with_their_place() { let tree = serde_json::json!({"role": "window", "name": "計算機", "bounds": {"x": 0.0, "y": 0.0, "width": 300.0, "height": 400.0}, "children": [ diff --git a/src/providers.rs b/src/providers.rs index 537797e..c077135 100644 --- a/src/providers.rs +++ b/src/providers.rs @@ -425,6 +425,13 @@ pub fn run_step(req: &ChatRequest, cancel: &AtomicBool, send: &dyn Fn(StreamEven _ => stream_openai_compatible(req, cancel, &tracked), }; match result { + // Nothing at all came back (no words, no tool call): a dropped stream or a hiccup. The + // turn used to end on 「（沒有回應內容）」; ask again instead. + Ok(out) if out.text.trim().is_empty() && out.tool_calls.is_empty() && !produced.get() && attempt < RETRY_MAX_ATTEMPTS && !cancel.load(Ordering::Relaxed) => { + send(StreamEvent::Retrying(format!("模型這次沒有回任何內容，再問一次（第 {} 次，共 {} 次）…", attempt + 1, RETRY_MAX_ATTEMPTS))); + std::thread::sleep(Duration::from_millis(backoff_delay_ms(attempt))); + attempt += 1; + } Ok(out) => return Ok(out), Err(f) if image_refused(&f.msg) && sees_images(req) && !produced.get() && !cancel.load(Ordering::Relaxed) => { mark_blind(req); @@ -1417,7 +1424,7 @@ mod tests { assert!(low.contains("x-api-key: sk-go") && low.contains("x-opencode-session: anybot-bot-7"), "{seen}"); // Responses-style models go through OpenCode's chat-completions translation. - let (_, seen) = ask("grok-4.7", "data: [DONE]\n\n"); + let (_, seen) = ask("grok-4.7", "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}]}\n\ndata: [DONE]\n\n"); assert!(seen.starts_with("POST /openai/v1/chat/completions "), "{seen}"); } diff --git a/src/tools.rs b/src/tools.rs index 37b7907..bef09ab 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -489,6 +489,15 @@ fn sandbox_exec() -> Option<&'static str> { (cfg!(target_os = "macos") && Path::new(p).exists()).then_some(p) } +/// Can the bot's own computer run commands without Docker? Only inside a sandbox (macOS): +/// elsewhere a command there would run on this computer itself, as AnyBot, keys and all. +pub fn sealed_runs_commands() -> bool { + sandbox_exec().is_some() +} + +/// Why the bot's own computer can't run a command right now. +pub const NO_BOX_FOR_COMMANDS: &str = "你自己的電腦現在不能執行指令：AnyBot 所在的這台沒有 Docker（或連不到 Docker），沒有隔離的地方可以跑指令。請告訴使用者把 Docker 打開，或請他指名別台電腦來做。"; + fn real(p: &Path) -> String { // The sandbox matches real paths, so resolve links like /var → /private/var. std::fs::canonicalize(p).unwrap_or_else(|_| p.to_path_buf()).display().to_string() @@ -704,7 +713,10 @@ pub fn decide_on(name: &str, args: &str, target: &Target, grants: &[String]) -> Place::Local { workdir } => decide(name, args, workdir, grants), Place::Sealed { root } => { if name == "shell" { - // Its own computer: the sandbox keeps it inside its folder. + // Its own computer: the sandbox keeps it inside its folder. No sandbox, no command. + if !sealed_runs_commands() { + return Decision::Deny(NO_BOX_FOR_COMMANDS.into()); + } return denied(&arg_str(&v, "command")).unwrap_or(Decision::Allow); } if name == "read_terminal" || name == "write_terminal" { @@ -802,6 +814,9 @@ pub fn run_on(name: &str, args: &str, target: &Target, cancel: &AtomicBool, ids: } Place::Sealed { root } => { let _ = std::fs::create_dir_all(root); + if name == "shell" && !sealed_runs_commands() { + return (NO_BOX_FOR_COMMANDS.into(), false); + } if name == "shell" { let home = dirs::home_dir().unwrap_or_else(|| PathBuf::from("/Users")); return done(shell(&arg_str(&v, "command"), root, timeout_of(&v), cancel, ids, Some(&home))); @@ -1372,6 +1387,11 @@ mod tests { #[test] fn bot_computer_is_sealed() { if sandbox_exec().is_none() { + // No sandbox (Linux, Windows): no command runs there at all, asked or not. + let t = Target { name: crate::model::BOT_COMPUTER_NAME.into(), place: Place::Sealed { root: std::env::temp_dir().join("anybot-no-seal") } }; + assert!(matches!(decide_on("shell", r#"{"command":"cat ~/.ssh/id_rsa"}"#, &t, &[]), Decision::Deny(_))); + let (r, ok) = run_on("shell", r#"{"command":"echo ran"}"#, &t, &AtomicBool::new(false), &RunIds::default()); + assert!(!ok && !r.contains("ran"), "{r}"); return; } let Some(home) = dirs::home_dir() else { return }; diff --git a/src/vnc.rs b/src/vnc.rs index a2cefb9..c74d73a 100644 --- a/src/vnc.rs +++ b/src/vnc.rs @@ -182,36 +182,68 @@ impl Session { } let mut at = 0; loop { - let rest = &self.pending[at..]; - let Some(&kind) = rest.first() else { break }; - let len = match kind { - 0 => 20, - 2 if rest.len() >= 4 => 4 + 4 * u16::from_be_bytes([rest[2], rest[3]]) as usize, - 3 => 10, - 4 => 8, - 5 => 6, - 6 if rest.len() >= 8 => 8 + u32::from_be_bytes([rest[4], rest[5], rest[6], rest[7]]) as usize, - 2 | 6 => break, - _ => { + match next_message(&self.pending[at..]) { + Msg::Need => break, + Msg::Unknown => { self.sniffing = false; self.pending.clear(); return; } - }; - if rest.len() < len { - break; + Msg::Is { len, input } => { + if input { + person_acted(); + } + at += len; + } } - match kind { - 4 if rest[1] == 1 => person_acted(), - 5 if rest[1] != 0 => person_acted(), - _ => {} - } - at += len; } self.pending.drain(..at); } } +enum Msg { + /// Not all of it has arrived yet. + Need, + /// A message this doesn't know (watching stops; forwarding goes on). + Unknown, + /// A whole message of `len` bytes; `input` when it is the person pressing a key or button. + Is { len: usize, input: bool }, +} + +/// The viewer's next message (RFC 6143 §7.5 and the extensions noVNC uses with TigerVNC: +/// continuous updates, fences, desktop size, xvp and QEMU key events — without them the watching +/// stopped at the first fence, and the bot went on clicking while the person worked). +fn next_message(rest: &[u8]) -> Msg { + let Some(&kind) = rest.first() else { return Msg::Need }; + let need = |n: usize| rest.len() >= n; + let len = match kind { + 0 => 20, + 2 if need(4) => 4 + 4 * u16::from_be_bytes([rest[2], rest[3]]) as usize, + 3 => 10, + 4 => 8, + 5 => 6, + 6 if need(8) => 8 + u32::from_be_bytes([rest[4], rest[5], rest[6], rest[7]]) as usize, + 150 => 10, + 248 if need(9) => 9 + rest[8] as usize, + 250 => 4, + 251 if need(7) => 8 + 16 * rest[6] as usize, + // QEMU extended key event (submessage 0). + 255 if need(2) && rest[1] == 0 => 12, + 2 | 6 | 248 | 251 | 255 if !need(9) => return Msg::Need, + _ => return Msg::Unknown, + }; + if !need(len) { + return Msg::Need; + } + let input = match kind { + 4 => rest[1] == 1, + 5 => rest[1] != 0, + 255 => u16::from_be_bytes([rest[2], rest[3]]) != 0, + _ => false, + }; + Msg::Is { len, input } +} + impl Drop for Session { fn drop(&mut self) { let _ = self.child.kill(); @@ -244,6 +276,38 @@ pub fn person_typed(text: &str) { mod tests { use super::*; + #[test] + fn a_persons_keys_are_seen_past_novncs_extra_messages() { + let walk = |b: &[u8]| -> (usize, bool, bool) { + let (mut at, mut input) = (0, false); + loop { + match next_message(&b[at..]) { + Msg::Need => return (at, input, false), + Msg::Unknown => return (at, input, true), + Msg::Is { len, input: i } => { + input |= i; + at += len; + } + } + } + }; + let mut b = vec![150, 1, 0, 0, 0, 0, 5, 0, 3, 32]; // EnableContinuousUpdates + b.extend([248, 0, 0, 0, 0, 0, 0, 1, 2, 7, 7]); // ClientFence with 2 bytes of payload + b.extend([3, 1, 0, 0, 0, 0, 5, 0, 3, 32]); // FramebufferUpdateRequest + let (at, input, stopped) = walk(&b); + assert_eq!((at, input, stopped), (b.len(), false, false)); + // A QEMU key press (noVNC's keys with TigerVNC), then its release. + let mut k = b.clone(); + k.extend([255, 0, 0, 1, 0, 0, 0, 0x61, 0, 0, 0, 30]); + k.extend([255, 0, 0, 0, 0, 0, 0, 0x61, 0, 0, 0, 30]); + assert_eq!(walk(&k), (k.len(), true, false)); + // A plain pointer click still counts; a half-arrived message waits for the rest. + assert_eq!(walk(&[5, 1, 0, 10, 0, 20]), (6, true, false)); + assert_eq!(walk(&[248, 0, 0]), (0, false, false)); + // Something unknown stops the watching. + assert!(walk(&[99, 0, 0, 0]).2); + } + #[test] fn the_bridge_keeps_its_indentation() { // Python needs its indentation (a Rust "\" line continuation would strip it). diff --git a/src/web.rs b/src/web.rs index a5aba47..7d93645 100644 --- a/src/web.rs +++ b/src/web.rs @@ -84,9 +84,9 @@ fn specs_inner(browser: bool) -> Vec { }; let profile = json!({"type": "string", "description": format!("Which AnyBot browser profile (設定檔) to use; each has its own logins (e.g. a different account). Profiles now: {names}. \"{}\" is the person's own everyday browser (read through accessibility), in whichever Chrome profile they have open there. Leave out to use the current one ({current}).", crate::botbrowser::PERSON_PROFILE)}); let mut v = vec![ - json!({"name": "web_fetch", "description": "Download a public web page and return its text (no login). For pages that need the person's login, use browser_open then browser_read.", + json!({"name": "web_fetch", "description": "Download a public web page and return its text (no login). For pages that need the person's login, or that only show text after scripts run, use browser_open.", "parameters": {"type": "object", "properties": {"url": {"type": "string"}}, "required": ["url"]}}), - json!({"name": "browser_open", "description": "Open a link in AnyBot's own browser window (a separate browser profile the person logs in to once, for sites like Threads, Instagram, Facebook, Gmail). Waits a few seconds for it to load; then call browser_read. Never ask the person to change browser settings.", + json!({"name": "browser_open", "description": "Open a link in AnyBot's own browser window (a separate browser profile the person logs in to once, for sites like Threads, Instagram, Facebook, Gmail). Waits until the page has loaded and returns what it shows, like browser_read (title, address, text and the numbered links, buttons and fields), so there is no need to call browser_read right after. Never ask the person to change browser settings.", "parameters": {"type": "object", "properties": {"url": {"type": "string"}, "profile": profile}, "required": ["url"]}}), json!({"name": "browser_read", "description": "Read the title, address and visible text of the current tab in AnyBot's browser (logged-in pages included), plus a numbered list of its links (with their addresses), buttons, fields and dropdowns. Use those numbers with browser_click / browser_type / browser_select instead of guessing URLs. Each computer has its own AnyBot browser with its own logins (on a bot's own computer it is the Chromium on its desktop). For anything on a web page, read it here and act by these numbers; never guess screen coordinates on a web page. If it shows a login page, hand the computer to the person (hand_over_computer) to log in once.", "parameters": {"type": "object", "properties": {"max_chars": {"type": "integer", "description": "Longest text to return (default 20000)."}, "profile": profile}}}), @@ -188,14 +188,14 @@ fn fetch(url: &str) -> Result { let status = resp.status().as_u16(); let body = resp.into_body().with_config().limit(MAX_FETCH_BYTES).read_to_string().map_err(|e| format!("讀取失敗：{e}"))?; if status >= 400 { - return Err(format!("網站回應 {status}。如果這個頁面要登入，請改用 browser_open + browser_read。")); + return Err(format!("網站回應 {status}。如果這個頁面要登入，請改用 browser_open。")); } let title = between(&body, "").map(|t| t.split_once('>').map(|x| x.1).unwrap_or(t).trim().to_string()).unwrap_or_default(); let text = html_to_text(&body); let thin = text.chars().count() < 200; let mut out = format!("網址：{url}\n標題：{}\n\n{}", decode_entities(&title), clip(&text, MAX_PAGE_CHARS)); if thin { - out.push_str("\n\n（頁面幾乎沒有文字，可能要登入或靠程式載入。可以改用 browser_open 打開，再用 browser_read 讀。）"); + out.push_str("\n\n（頁面幾乎沒有文字，可能要登入或靠程式載入。可以改用 browser_open 打開，它會等頁面載入再回傳內容。）"); } Ok(out) } @@ -230,13 +230,29 @@ pub fn decode_entities(s: &str) -> String { out.replace("&", "&") } +/// Where `` and ``, never `
` (taking +/// `
` for `` cut every page with a header off there — articles came back as their +/// menus). +fn open_tag(lower: &str, tag: &str) -> Option { + let pat = format!("<{tag}"); + let mut from = 0; + while let Some(i) = lower[from..].find(&pat).map(|i| i + from) { + match lower[i + pat.len()..].chars().next() { + Some(c) if c == '>' || c == '/' || c.is_whitespace() => return Some(i), + None => return Some(i), + _ => from = i + pat.len(), + } + } + None +} + /// Visible text of an HTML page: drop scripts, styles and tags, keep line breaks for blocks. pub fn html_to_text(html: &str) -> String { let mut s = html.to_string(); for tag in ["script", "style", "noscript", "svg", "head"] { loop { let lower = s.to_ascii_lowercase(); - let Some(a) = lower.find(&format!("<{tag}")) else { break }; + let Some(a) = open_tag(&lower, tag) else { break }; let end_tag = format!(""); match lower[a..].find(&end_tag) { Some(b) => s.replace_range(a..a + b + end_tag.len(), " "), @@ -259,8 +275,10 @@ pub fn html_to_text(html: &str) -> String { '>' if in_tag => { in_tag = false; let t = tag.trim_start_matches('/').split(|c: char| c.is_whitespace()).next().unwrap_or("").to_ascii_lowercase(); - if ["p", "div", "br", "li", "h1", "h2", "h3", "h4", "tr", "section", "article", "header", "footer"].contains(&t.as_str()) { + if ["p", "div", "br", "li", "h1", "h2", "h3", "h4", "h5", "h6", "tr", "section", "article", "header", "footer", "main", "nav", "aside", "ul", "ol", "table", "blockquote", "pre", "dt", "dd", "figcaption"].contains(&t.as_str()) { out.push('\n'); + } else if t == "td" || t == "th" { + out.push(' '); } } _ if in_tag => tag.push(c), @@ -468,6 +486,19 @@ fn browser_read(max_chars: usize, cancel: &AtomicBool) -> Result mod tests { use super::*; + #[test] + fn a_page_with_a_header_keeps_its_article() { + // Taking
for cut pages off at their header (WordPress sites, news sites). + let html = "T

預售策略

先賣再錄

方案價格
©
"; + let t = html_to_text(html); + assert!(t.contains("首頁 課程") && t.contains("預售策略") && t.contains("先賣再錄") && t.contains("©"), "{t}"); + assert!(t.contains("方案 價格"), "{t}"); + // A page with no at all, only a
. + assert!(html_to_text("
選單

內文

").contains("內文")); + // is still the head. + assert!(!html_to_text("不要

要

").contains("不要")); + } + #[test] fn html_becomes_readable_text() { let html = "T

標題

第一段 文字 & 更多

中文
"; @@ -501,3 +532,4 @@ mod tests { assert!(explain("Executing JavaScript through AppleScript is turned off.", "Google Chrome").contains("允許來自 Apple 事件的 JavaScript")); } } +