AnyBot/src/browser.rs

1276 lines
77 KiB
Rust
Raw Normal View History

// AnyBot's browser, the full toolset. botbrowser.rs starts Chrome and keeps profiles; this file
// drives a tab through the Chrome DevTools Protocol like a person would (a Firefox tab through
// WebDriver BiDi: bidi.rs does each DevTools command the BiDi way, so the tools are the same):
// - real mouse and keyboard (Input.dispatchMouseEvent / insertText / dispatchKeyEvent), so
// React date pickers, dropdowns and menus that ignore scripted events still respond;
// - a numbered list of everything clickable (links with their addresses, buttons, fields,
// dropdowns, tabs, calendar cells — anything the page gives a pointer cursor);
// - after each action only what changed (new address, new text), not the whole page again;
// - tab screenshots (no screen-recording permission needed), tables as Markdown, find on page,
// waiting for text, tabs, back/forward/reload, file uploads, downloads, PDF, page dialogs and
// running a script on the page.
use serde_json::{json, Value};
use std::io::ErrorKind;
use std::net::{SocketAddr, TcpStream};
use std::time::{Duration, Instant};
use crate::botbrowser::{self, Kind};
const CALL_TIMEOUT_SECS: u64 = 30;
/// Longest element list in one answer.
const MAX_ELEMENTS: usize = 250;
/// One connection to a tab: DevTools (Chrome-family), or a tab of a Firefox driven over BiDi.
pub struct Tab {
link: Link,
next: u64,
}
enum Link {
Cdp(Box<tungstenite::WebSocket<TcpStream>>),
Bidi { port: u16, context: String },
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
}
/// The ordinary page tabs of a /json/list answer, most recently used first.
pub(crate) fn pages(list: &Value) -> Vec<Value> {
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
list.as_array().cloned().unwrap_or_default().into_iter().filter(|t| t["type"] == "page" && !t["url"].as_str().unwrap_or("").starts_with("devtools://")).collect()
}
/// The browser's port for a profile (starting it if needed), and which protocol it speaks.
pub fn port_of(profile: Option<&str>) -> Result<(String, u16, Kind), String> {
let prof = botbrowser::resolve(profile)?;
let (_, port, _) = botbrowser::ensure(&prof, None)?;
Ok((prof.clone(), port, botbrowser::kind_of(Some(&prof))))
}
impl Tab {
/// The tab this step works on: its job's latest own tab, else the latest one no other job
/// has (the list has the most recently used first).
pub fn current(profile: Option<&str>) -> Result<Tab, String> {
let (_, port, kind) = port_of(profile)?;
let list = botbrowser::pages_of(kind, port)?;
// This job's own latest tab (another bot's job may have one in front).
let t = botbrowser::pick_tab(&list).ok_or("AnyBot 瀏覽器沒有開著的分頁,請先用 browser_open 打開網址。")?;
botbrowser::claim(t["id"].as_str().unwrap_or(""));
if kind == Kind::Firefox {
let context = t["id"].as_str().unwrap_or("").to_string();
crate::bidi::touch(port, &context);
return Ok(Tab { link: Link::Bidi { port, context }, next: 0 });
}
Tab::connect(port, t)
}
fn connect(port: u16, t: &Value) -> Result<Tab, String> {
let ws_url = t["webSocketDebuggerUrl"].as_str().ok_or("這個分頁不能控制(可能已被別的工具連著)")?;
let addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
let stream = TcpStream::connect_timeout(&addr, Duration::from_secs(5)).map_err(|e| e.to_string())?;
stream.set_read_timeout(Some(Duration::from_secs(CALL_TIMEOUT_SECS))).ok();
let (ws, _) = tungstenite::client::client(ws_url, stream).map_err(|e| format!("連不上 AnyBot 瀏覽器:{e}"))?;
Ok(Tab { link: Link::Cdp(Box::new(ws)), next: 0 })
}
pub fn is_firefox(&self) -> bool {
matches!(self.link, Link::Bidi { .. })
}
/// One DevTools command; events that arrive meanwhile are skipped. (On Firefox, done the
/// WebDriver BiDi way, with an answer shaped the same.)
pub fn call(&mut self, method: &str, params: Value) -> Result<Value, String> {
let ws = match &mut self.link {
Link::Bidi { port, context } => return crate::bidi::cdp(*port, context, method, &params),
Link::Cdp(ws) => ws,
};
self.next += 1;
let id = self.next;
ws.send(tungstenite::Message::Text(json!({"id": id, "method": method, "params": params}).to_string().into())).map_err(|e| e.to_string())?;
let start = Instant::now();
loop {
if start.elapsed() > Duration::from_secs(CALL_TIMEOUT_SECS) {
return Err("瀏覽器太久沒有回應(頁面可能跳出了對話框:可以用 browser_dialog 處理)".into());
}
match ws.read() {
Ok(tungstenite::Message::Text(t)) => {
let v: Value = serde_json::from_str(t.as_str()).unwrap_or(Value::Null);
if v["id"] == id {
if let Some(e) = v.get("error") {
return Err(e["message"].as_str().unwrap_or("瀏覽器拒絕了這個動作").to_string());
}
return Ok(v["result"].clone());
}
}
Ok(_) => {}
Err(tungstenite::Error::Io(e)) if e.kind() == ErrorKind::WouldBlock || e.kind() == ErrorKind::TimedOut => return Err("瀏覽器太久沒有回應(頁面可能跳出了對話框:可以用 browser_dialog 處理)".into()),
Err(e) => return Err(e.to_string()),
}
}
}
/// Run JavaScript and get its value back.
pub fn eval(&mut self, expr: &str) -> Result<Value, String> {
let r = self.call("Runtime.evaluate", json!({"expression": expr, "returnByValue": true, "awaitPromise": true, "userGesture": true}))?;
if let Some(ex) = r.get("exceptionDetails") {
return Err(format!("頁面上的程式出錯:{}", ex["exception"]["description"].as_str().or(ex["text"].as_str()).unwrap_or("")));
}
Ok(r["result"]["value"].clone())
}
fn close(self) {
if let Link::Cdp(mut ws) = self.link {
let _ = ws.close(None);
}
}
}
// ---------- what the page offers ----------
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
/// Every element matching `sel`, the page's own and those inside open shadow roots (web
/// components: Chromium's settings pages, YouTube, many app frameworks), in the order they are
/// laid out: a host, then what its shadow root shows, then its own children. Plain
/// querySelectorAll stops at every shadow root, so those buttons had no number at all.
const DEEP_JS: &str = "const deepAll = (sel, root) => { const out = []; const walk = r => { for (const el of r.querySelectorAll('*')) { if (sel === '*' || el.matches(sel)) out.push(el); if (el.shadowRoot) walk(el.shadowRoot); } }; walk(root || document); return out; };";
/// Numbers everything clickable or fillable (data-anybot-ref) and lists it.
const ELEMENTS_JS: &str = r#"(() => {
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
%DEEP%
const vis = el => { const r = el.getBoundingClientRect(); if (r.width < 2 || r.height < 2) return false; const s = getComputedStyle(el); return s.visibility !== 'hidden' && s.display !== 'none' && +s.opacity > 0.05; };
const clean = s => (s || '').replace(/\s+/g, ' ').trim();
const name = el => clean(el.getAttribute('aria-label') || el.innerText || el.value || el.getAttribute('placeholder') || el.title || el.getAttribute('alt') || (el.querySelector('img[alt]') || {}).alt || '').slice(0, 80);
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
const label = el => { const a = el.getAttribute('aria-label'); if (a) return clean(a).slice(0, 60); if (el.labels && el.labels[0]) return clean(el.labels[0].innerText).slice(0, 60); const id = el.getAttribute('aria-labelledby'); if (id) { const home = el.getRootNode(); const l = (home.getElementById ? home : document).getElementById(id); if (l) return clean(l.innerText).slice(0, 60); } return clean(el.getAttribute('placeholder') || el.name || el.title || ''); };
const base = 'a[href],button,input:not([type=hidden]),select,textarea,summary,[contenteditable=""],[contenteditable=true],[role=button],[role=link],[role=tab],[role=menuitem],[role=menuitemcheckbox],[role=option],[role=checkbox],[role=radio],[role=combobox],[role=switch],[role=treeitem],[role=gridcell],[onclick],[tabindex]:not([tabindex="-1"])';
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
const all = deepAll('*');
const order = new Map(all.map((el, i) => [el, i]));
const set = new Set(all.filter(el => el.matches(base)));
// Anything the page shows a pointer cursor for (calendar days, cards, custom buttons…).
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
for (const el of all) {
if (set.size > 4000) break;
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
if (!el.matches('div,span,li,td,th,p,img,svg,i,label')) continue;
if (getComputedStyle(el).cursor === 'pointer' && !(el.parentElement && getComputedStyle(el.parentElement).cursor === 'pointer')) set.add(el);
}
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
all.forEach(e => e.hasAttribute('data-anybot-ref') && e.removeAttribute('data-anybot-ref'));
const inView = el => { const r = el.getBoundingClientRect(); return r.bottom > 0 && r.top < innerHeight && r.right > 0 && r.left < innerWidth; };
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
const ordered = [...set].filter(el => vis(el) && !el.closest('[aria-hidden=true]')).sort((a, b) => order.get(a) - order.get(b));
const out = []; const seen = new Set(); let n = 0;
for (const el of ordered) {
const tag = el.tagName.toLowerCase(); const role = el.getAttribute('role') || '';
let kind, text = name(el), extra = '';
if (tag === 'a') { kind = '連結'; extra = el.href ? '→ ' + el.href : ''; }
else if (tag === 'select') { kind = '下拉選單'; text = label(el); extra = [...el.options].map(o => (o.selected ? '✓' : '') + clean(o.text)).slice(0, 20).join(' / '); }
else if (tag === 'input' || tag === 'textarea') {
const t = (el.type || 'text').toLowerCase();
if (t === 'checkbox' || t === 'radio') { kind = t === 'checkbox' ? '勾選框' : '選項'; text = label(el) || text; extra = el.checked ? '已勾' : '未勾'; }
else if (t === 'submit' || t === 'button' || t === 'reset') kind = '按鈕';
else if (t === 'file') { kind = '上傳檔案'; text = label(el) || '選擇檔案'; }
else { kind = '輸入框'; text = label(el); extra = t === 'password' ? (el.value ? '(已填)' : '(密碼)') : (el.value ? '目前:' + clean(el.value).slice(0, 40) : ''); if (el.readOnly) extra += '(唯讀,點它會打開選擇器)'; }
}
else if (el.isContentEditable) { kind = '輸入框'; text = label(el) || clean(el.innerText).slice(0, 40); }
else if (tag === 'button' || role === 'button' || tag === 'summary') kind = '按鈕';
else if (role === 'tab') { kind = '分頁標籤'; if (el.getAttribute('aria-selected') === 'true') extra = '(目前)'; }
else if (role === 'menuitem' || role === 'menuitemcheckbox' || role === 'option' || role === 'treeitem') { kind = '選單項目'; if (el.getAttribute('aria-selected') === 'true' || el.getAttribute('aria-checked') === 'true') extra = '(已選)'; }
else if (role === 'combobox') { kind = '下拉選單'; extra = clean(el.innerText).slice(0, 40); }
else if (role === 'link') kind = '連結';
else if (role === 'checkbox' || role === 'switch' || role === 'radio') { kind = '勾選框'; extra = el.getAttribute('aria-checked') === 'true' ? '已勾' : '未勾'; }
else if (role === 'gridcell' || tag === 'td') { kind = '格子'; const full = el.getAttribute('title') || el.getAttribute('aria-label') || el.getAttribute('data-date') || (el.closest('[title]') || {}).title || ''; if (full && full !== text) text = text ? text + '(' + full + ')' : full; }
else { kind = '可點的項目'; const cell = el.closest('td[title],[role=gridcell][aria-label],[data-date]'); if (cell) { kind = '格子'; const full = cell.getAttribute('title') || cell.getAttribute('aria-label') || cell.getAttribute('data-date'); if (full && full !== text) text = text ? text + '(' + full + ')' : full; } }
if (!text && !extra) continue;
if (el.disabled || el.getAttribute('aria-disabled') === 'true') extra += '(停用)';
const key = kind + '|' + text + '|' + extra;
if (seen.has(key)) continue;
seen.add(key);
n += 1; el.setAttribute('data-anybot-ref', String(n));
out.push('[' + n + '] ' + kind + '「' + text + '」' + (extra ? ' ' + extra : '') + (inView(el) ? '' : ' (要捲動)'));
if (n >= %MAX%) break;
}
return JSON.stringify({t: document.title, u: location.href, list: out, more: ordered.length > n});
})()"#;
/// The numbered list of what can be clicked or filled.
pub fn elements(tab: &mut Tab) -> Result<(String, String, Vec<String>, bool), String> {
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
let v = tab.eval(&ELEMENTS_JS.replace("%DEEP%", DEEP_JS).replace("%MAX%", &MAX_ELEMENTS.to_string()))?;
let p: Value = v.as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or(Value::Null);
let list = p["list"].as_array().map(|a| a.iter().filter_map(|x| x.as_str().map(String::from)).collect()).unwrap_or_default();
Ok((p["t"].as_str().unwrap_or("").into(), p["u"].as_str().unwrap_or("").into(), list, p["more"].as_bool().unwrap_or(false)))
}
/// Draws a numbered box over every listed element that is on screen (removed after the shot).
const MARKS_JS: &str = r#"(() => {
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
%DEEP%
const old = document.getElementById('__anybot_marks'); if (old) old.remove();
const box = document.createElement('div'); box.id = '__anybot_marks';
box.style.cssText = 'position:fixed;inset:0;pointer-events:none;z-index:2147483647';
const colors = ['#e11d48', '#2563eb', '#16a34a', '#d97706', '#9333ea'];
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
for (const el of deepAll('[data-anybot-ref]')) {
const r = el.getBoundingClientRect();
if (r.bottom < 0 || r.top > innerHeight || r.right < 0 || r.left > innerWidth || r.width < 2) continue;
const n = el.getAttribute('data-anybot-ref'); const c = colors[n % colors.length];
const f = document.createElement('div');
f.style.cssText = `position:fixed;left:${r.left}px;top:${r.top}px;width:${r.width}px;height:${r.height}px;border:1.5px solid ${c};border-radius:3px;box-sizing:border-box`;
const tag = document.createElement('div');
tag.textContent = n;
tag.style.cssText = `position:fixed;left:${Math.max(0, r.left - 1)}px;top:${Math.max(0, r.top - 15)}px;background:${c};color:#fff;font:bold 11px/14px -apple-system,Arial;padding:0 3px;border-radius:3px`;
box.appendChild(f); box.appendChild(tag);
}
document.documentElement.appendChild(box);
return 1;
})()"#;
const SNAPSHOT_JS: &str = "JSON.stringify({t: document.title, u: location.href, x: document.body ? document.body.innerText : ''})";
fn snapshot(tab: &mut Tab) -> (String, String, Vec<String>) {
let v = tab.eval(SNAPSHOT_JS).unwrap_or(Value::Null);
let p: Value = v.as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or(Value::Null);
let lines = p["x"].as_str().unwrap_or("").lines().map(str::trim).filter(|l| !l.is_empty()).map(String::from).collect();
(p["t"].as_str().unwrap_or("").into(), p["u"].as_str().unwrap_or("").into(), lines)
}
/// What an action changed: the address, and the lines of text that are new.
fn changes(before: &(String, String, Vec<String>), after: &(String, String, Vec<String>)) -> String {
let mut out = vec![];
if before.1 != after.1 {
out.push(format!("網址變成:{}", after.1));
}
if before.0 != after.0 {
out.push(format!("標題:{}", after.0));
}
let old: std::collections::HashSet<&String> = before.2.iter().collect();
let new: Vec<&String> = after.2.iter().filter(|l| !old.contains(l)).collect();
let gone = before.2.iter().filter(|l| !after.2.contains(l)).count();
if new.is_empty() && gone == 0 && before.1 == after.1 {
out.push("頁面看起來沒有變化(可能要等一下、換一個元素,或用 browser_screenshot 看看)。".into());
} else if !new.is_empty() {
let shown: Vec<String> = new.iter().take(40).map(|l| l.chars().take(160).collect()).collect();
out.push(format!("新出現的文字:\n{}{}", shown.join("\n"), if new.len() > 40 { format!("\n…還有 {} 行(用 browser_read 看全部)", new.len() - 40) } else { String::new() }));
}
out.join("\n")
}
// ---------- finding an element ----------
/// The element a bot means: its number, else visible text / label / placeholder.
fn find_js(r#ref: &str, text: &str) -> String {
let rf = serde_json::to_string(r#ref.trim().trim_matches(['[', ']'])).unwrap_or_default();
let tx = serde_json::to_string(text.trim()).unwrap_or_default();
format!(
r#"(() => {{
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
{deep}
const rf = {rf}, tx = {tx};
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
if (rf) {{ const e = deepAll('[data-anybot-ref="' + rf + '"]')[0]; if (e) return e; }}
if (!tx) return null;
const t = s => (s || '').replace(/\s+/g, ' ').trim();
const vis = e => {{ const r = e.getBoundingClientRect(); return r.width > 1 && r.height > 1; }};
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
const all = deepAll('a,button,[role],input,select,textarea,label,summary,li,td,th,span,div,p,img').filter(vis);
const hit = e => [t(e.innerText), t(e.getAttribute('aria-label')), t(e.getAttribute('placeholder')), t(e.title), t(e.value), e.labels && e.labels[0] ? t(e.labels[0].innerText) : ''];
// The control the words name, not the box around it: 「Customer name:」 is a <p> holding a
// label and its input, 「Submit order」 a <p> holding the button (its middle is empty page).
const ctl = 'a,button,input,select,textarea,summary,[role=button],[role=link],[role=checkbox],[role=radio],[role=switch],[role=tab],[role=menuitem],[role=option],[role=textbox],[role=combobox],[contenteditable=""],[contenteditable=true]';
const isCtl = e => e.matches(ctl);
const inner = e => {{
if (isCtl(e)) return e;
if (e.tagName === 'LABEL' && e.control && vis(e.control)) return e.control;
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
const c = deepAll(ctl, e).filter(vis);
return c.length === 1 ? c[0] : e;
}};
const shortest = l => l.slice().sort((a, b) => t(a.innerText).length - t(b.innerText).length)[0];
const pick = l => l.find(isCtl) || l.map(inner).find(isCtl) || shortest(l) || null;
const exact = all.filter(e => hit(e).includes(tx));
return exact.length ? pick(exact) : pick(all.filter(e => hit(e).some(h => h.includes(tx))));
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
}})()"#,
deep = DEEP_JS
)
}
/// Bring the element into view and get its centre (CSS pixels in the viewport).
fn locate(tab: &mut Tab, r#ref: &str, text: &str) -> Result<(f64, f64, String), String> {
let js = format!(
"(() => {{ const el = {}; if (!el) return null; el.scrollIntoView({{block: 'center', inline: 'center'}}); const r = el.getBoundingClientRect(); return JSON.stringify({{x: r.left + r.width / 2, y: r.top + r.height / 2, tag: el.tagName, type: el.type || '', ro: !!el.readOnly}}); }})()",
find_js(r#ref, text)
);
let v = tab.eval(&js)?;
let what = if !r#ref.trim().is_empty() { format!("[{}]", r#ref.trim().trim_matches(['[', ']'])) } else { format!("「{}」", text.trim()) };
let p: Value = v.as_str().and_then(|s| serde_json::from_str(s).ok()).ok_or_else(|| format!("這一頁找不到 {what}。用 browser_read 或 browser_elements 看最新的清單(編號每次讀取會重排)。"))?;
Ok((p["x"].as_f64().unwrap_or(0.0), p["y"].as_f64().unwrap_or(0.0), format!("{what}{}", if p["tag"] == "INPUT" && p["ro"] == true { "(唯讀欄位)" } else { "" })))
}
fn mouse(tab: &mut Tab, kind: &str, x: f64, y: f64, button: &str, count: i64) -> Result<(), String> {
tab.call("Input.dispatchMouseEvent", json!({"type": kind, "x": x, "y": y, "button": button, "buttons": if kind == "mousePressed" { 1 } else { 0 }, "clickCount": count})).map(|_| ())
}
/// A real click (the page can't tell it from a person's).
fn click_at(tab: &mut Tab, x: f64, y: f64, button: &str, count: i64) -> Result<(), String> {
mouse(tab, "mouseMoved", x, y, "none", 0)?;
for n in 1..=count {
mouse(tab, "mousePressed", x, y, button, n)?;
mouse(tab, "mouseReleased", x, y, button, n)?;
}
Ok(())
}
/// Key names the bot may use → (key, code, keyCode, text).
fn key_info(k: &str) -> Option<(&'static str, &'static str, i64, &'static str)> {
Some(match k.to_lowercase().as_str() {
"enter" | "return" => ("Enter", "Enter", 13, "\r"),
"tab" => ("Tab", "Tab", 9, ""),
"escape" | "esc" => ("Escape", "Escape", 27, ""),
"backspace" => ("Backspace", "Backspace", 8, ""),
"delete" => ("Delete", "Delete", 46, ""),
"space" => (" ", "Space", 32, " "),
"arrowdown" | "down" => ("ArrowDown", "ArrowDown", 40, ""),
"arrowup" | "up" => ("ArrowUp", "ArrowUp", 38, ""),
"arrowleft" | "left" => ("ArrowLeft", "ArrowLeft", 37, ""),
"arrowright" | "right" => ("ArrowRight", "ArrowRight", 39, ""),
"pagedown" => ("PageDown", "PageDown", 34, ""),
"pageup" => ("PageUp", "PageUp", 33, ""),
"home" => ("Home", "Home", 36, ""),
"end" => ("End", "End", 35, ""),
"a" => ("a", "KeyA", 65, "a"),
"c" => ("c", "KeyC", 67, "c"),
"v" => ("v", "KeyV", 86, "v"),
"x" => ("x", "KeyX", 88, "x"),
"f" => ("f", "KeyF", 70, "f"),
_ => return None,
})
}
/// Press a key or combination ("Enter", "ctrl+a", "shift+tab").
fn press(tab: &mut Tab, combo: &str) -> Result<(), String> {
let parts: Vec<&str> = combo.split('+').map(str::trim).filter(|s| !s.is_empty()).collect();
let (key, mods) = parts.split_last().ok_or("請給按鍵,例如 Enter")?;
let mut m = 0;
for x in mods {
m |= match x.to_lowercase().as_str() {
"alt" | "option" => 1,
"ctrl" | "control" => 2,
"cmd" | "meta" | "command" => 4,
"shift" => 8,
other => return Err(format!("不認得的按鍵「{other}」")),
};
}
let (k, code, kc, text) = key_info(key).ok_or_else(|| format!("不認得的按鍵「{key}」,可以用 Enter、Tab、Escape、Backspace、方向鍵、PageDown、Home、End,或 ctrl+a 這類組合"))?;
let text = if m & (1 | 2 | 4) != 0 { "" } else { text };
tab.call("Input.dispatchKeyEvent", json!({"type": if text.is_empty() { "rawKeyDown" } else { "keyDown" }, "key": k, "code": code, "windowsVirtualKeyCode": kc, "modifiers": m, "text": text}))?;
tab.call("Input.dispatchKeyEvent", json!({"type": "keyUp", "key": k, "code": code, "windowsVirtualKeyCode": kc, "modifiers": m}))?;
Ok(())
}
fn settle(ms: u64) {
std::thread::sleep(Duration::from_millis(ms));
}
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
/// What the page shows, cheaply: load state, address and how much text.
const SIG_JS: &str = "document.readyState+'|'+location.href+'|'+(document.body?document.body.innerText.length:0)";
fn page_sig(tab: &mut Tab) -> String {
tab.eval(SIG_JS).ok().and_then(|v| v.as_str().map(String::from)).unwrap_or_default()
}
/// Wait for an action's effect, not a fixed time: back as soon as the page has changed and holds
/// still (loaded, same address and text twice in a row), or when nothing has started to change
/// within `quiet_ms`; never longer than `max_ms`. (Fixed sleeps read slow pages too early and
/// kept the bot waiting on fast ones.)
pub fn settle_after(tab: &mut Tab, before: &str, quiet_ms: u64, max_ms: u64) {
let start = Instant::now();
let mut last = String::new();
let mut changed = false;
loop {
settle(150);
let now = page_sig(tab);
changed |= now != before;
let elapsed = start.elapsed().as_millis() as u64;
let still = now.starts_with("complete|") && now == last;
if (still && (changed || elapsed >= quiet_ms)) || elapsed >= max_ms {
return;
}
last = now;
}
}
/// Wait until the tab in front has loaded a real page (not the blank tab it starts as) and holds
/// still, at most `max_ms`. Says whether it did. Connects afresh each look: a navigation can swap
/// the page behind the tab.
pub fn wait_loaded(profile: Option<&str>, max_ms: u64) -> bool {
let start = Instant::now();
let mut last = String::new();
while (start.elapsed().as_millis() as u64) < max_ms {
let now = Tab::current(profile).map(|mut t| {
let sig = page_sig(&mut t);
t.close();
sig
});
let now = now.unwrap_or_default();
if now.starts_with("complete|") && !now.contains("|about:blank|") && now == last {
return true;
}
last = now;
settle(200);
}
false
}
// ---------- the tools ----------
fn s(args: &Value, k: &str) -> String {
args.get(k).map(|v| v.as_str().map(String::from).unwrap_or_else(|| if v.is_null() { String::new() } else { v.to_string() })).unwrap_or_default().trim().to_string()
}
pub const NAMES: [&str; 18] = [
"browser_elements",
"browser_click",
"browser_hover",
"browser_type",
"browser_press",
"browser_select",
"browser_scroll",
"browser_wait",
"browser_find",
"browser_table",
"browser_screenshot",
"browser_tabs",
"browser_back",
"browser_upload",
"browser_download",
"browser_pdf",
"browser_dialog",
"browser_eval",
];
pub fn is_tool(name: &str) -> bool {
NAMES.contains(&name)
}
pub fn label(name: &str) -> &'static str {
match name {
"browser_elements" => "列出網頁上的按鈕",
"browser_click" => "在瀏覽器點擊",
"browser_hover" => "在瀏覽器停在上面",
"browser_type" => "在瀏覽器輸入",
"browser_press" => "在瀏覽器按鍵",
"browser_select" => "在瀏覽器選擇",
"browser_scroll" => "捲動瀏覽器",
"browser_wait" => "等網頁載入",
"browser_find" => "在網頁上找字",
"browser_table" => "讀網頁表格",
"browser_screenshot" => "瀏覽器截圖",
"browser_tabs" => "瀏覽器分頁",
"browser_back" => "瀏覽器上一頁",
"browser_upload" => "上傳檔案到網頁",
"browser_download" => "從網頁下載",
"browser_pdf" => "把網頁存成 PDF",
"browser_dialog" => "處理網頁對話框",
"browser_eval" => "在網頁執行程式",
_ => "瀏覽器",
}
}
pub fn describe(name: &str, args: &str) -> String {
let v: Value = serde_json::from_str(args).unwrap_or(Value::Null);
let target = if !s(&v, "ref").is_empty() { format!("[{}]", s(&v, "ref")) } else { s(&v, "text") };
match name {
"browser_type" => format!("{target} ← {}", s(&v, "value").chars().take(40).collect::<String>()),
"browser_select" => format!("{target} → {}", s(&v, "option")),
"browser_press" => s(&v, "key"),
"browser_scroll" => if target.is_empty() { s(&v, "direction") } else { target },
"browser_wait" => if s(&v, "text").is_empty() { format!("{} 秒", s(&v, "seconds")) } else { format!("等「{}」", s(&v, "text")) },
"browser_find" => format!("「{}」", s(&v, "text")),
"browser_tabs" => format!("{} {}", s(&v, "action"), s(&v, "index")),
"browser_back" => s(&v, "direction"),
"browser_upload" => s(&v, "path"),
"browser_eval" => s(&v, "script").chars().take(60).collect(),
_ => target,
}
}
/// The one browser tool's name (H2: every browser action rides on it, like `computer`).
pub const TOOL: &str = "browser";
/// action -> the internal per-action tool it runs as. The internal names stay everywhere
/// downstream (approvals, oplog, the work log, remote limbs): the model only ever sees `browser`.
pub fn action_tool(action: &str) -> Option<&'static str> {
Some(match action {
"open" => "browser_open",
"read" => "browser_read",
"profiles" => "browser_profiles",
"elements" => "browser_elements",
"find" => "browser_find",
"table" => "browser_table",
"screenshot" => "browser_screenshot",
"click" => "browser_click",
"hover" => "browser_hover",
"type" => "browser_type",
"press" => "browser_press",
"select" => "browser_select",
"scroll" => "browser_scroll",
"wait" => "browser_wait",
"tabs" => "browser_tabs",
"back" => "browser_back",
"upload" => "browser_upload",
"download" => "browser_download",
"pdf" => "browser_pdf",
"dialog" => "browser_dialog",
"eval" => "browser_eval",
"fill" => "browser_fill_login", // the kind picks login/card/contact; see expand_call
_ => return None,
})
}
/// The one browser tool. One schema, trained-prior shapes (open/read/click/type…); the old
/// eighteen schemas' guidance lives in the action list below.
pub fn tool(profile: &Value) -> Value {
json!({
"name": TOOL,
"description": "Drive AnyBot's browser: one tool, picked by action. For anything on a web page use this, never screen coordinates. Start with open (it answers like read) or read, then act by the numbers (ref) or visible text the last result showed. Never act on a page from memory — use the numbers the latest result showed.
actions:
- open {url}: open a link in a new tab and report the page (title, address, text, numbered controls)
- read {max_chars}: read the current tab the same way (numbers for click/type/select)
- elements: just the numbered clickable/fillable controls (cheaper than read)
- find {text}: like Cmd-F: every place the text appears with words around it
- table {index, all, max_rows}: read the page's tables as Markdown (reports, product lists)
- screenshot {full, marks}: the page as a picture with numbered boxes on every control; needs no screen permission
- click {ref|text, double, button}: real mouse click (menus, date pickers, dropdowns, calendar days); buying/paying/sending/deleting needs the person's OK first
- hover {ref|text}: move over a control (opens hover menus) and report what appeared
- type {ref|text, value, append, submit}: real keystrokes into a field (clears first unless append). Never type passwords here: use fill
- press {key}: Enter, Tab, Escape, ctrl+a, … (optionally on ref/text first)
- select {ref|text, option}: choose a dropdown option (custom dropdowns included)
- scroll {direction: down|up|top|bottom} (on ref/text brings it into view; loads endless lists)
- wait {text | seconds}: until text appears (≤20 s) or a few seconds
- tabs {action: list|switch|new|close, index, url}: the tab strip
- back {direction: back|forward|reload}
- upload {ref|text, path}: put a file from this computer into a file chooser
- download {ref|text}: click a download/export button, save to Downloads/AnyBot, return the path
- pdf: save the page as a PDF in Downloads/AnyBot
- dialog {accept, text}: answer a pop-up dialog the page opened
- eval {script}: short read-only JavaScript for data the other actions can't reach; it can't send requests, navigate or submit (site APIs get accounts flagged)
- fill {kind: login|card|contact, id, site, submit}: sign in / pay card / address from the person's 保險庫 — you never see the secret; the person approves every use (login_id/card_id/contact_id come from vault_list; default login: the current site's)",
"parameters": {"type": "object", "properties": {
"action": {"type": "string", "enum": ["open","read","elements","find","table","screenshot","click","hover","type","press","select","scroll","wait","tabs","back","upload","download","pdf","dialog","eval","profiles","fill"]},
"url": {"type": "string"},
"max_chars": {"type": "integer", "description": "read: longest text to return (default 20000)"},
"text": {"type": "string", "description": "find/wait: the text; click/type/…: the control's visible text"},
"ref": {"type": "string", "description": "The element's number from the latest read/elements/screenshot result"},
"index": {"type": "integer", "description": "table: which table; tabs: which tab"},
"all": {"type": "boolean", "description": "table: every table"},
"max_rows": {"type": "integer"},
"full": {"type": "boolean", "description": "screenshot: the whole page"},
"marks": {"type": "boolean", "description": "screenshot: false for no number boxes"},
"double": {"type": "boolean", "description": "click: double-click"},
"button": {"type": "string", "enum": ["left", "right", "middle"]},
"value": {"type": "string", "description": "type: the text"},
"append": {"type": "boolean", "description": "type: keep what's in the field"},
"submit": {"type": "boolean", "description": "type/fill: press Enter / sign in afterwards"},
"key": {"type": "string", "description": "press: Enter, Tab, Escape, ctrl+a…"},
"option": {"type": "string", "description": "select: the option's visible text"},
"direction": {"type": "string", "enum": ["down", "up", "top", "bottom", "back", "forward", "reload"], "description": "scroll: down/up/top/bottom; back: back/forward/reload"},
"seconds": {"type": "number"},
"path": {"type": "string", "description": "upload: a file on this computer"},
"accept": {"type": "boolean", "description": "dialog"},
"script": {"type": "string", "description": "eval: read-only JavaScript"},
"kind": {"type": "string", "enum": ["login", "card", "contact"], "description": "fill: what to fill"},
"id": {"type": "string", "description": "fill: the vault item's id from vault_list"},
"site": {"type": "string", "description": "fill login: domain (default: the current tab's)"},
"name": {"type": "string", "description": "profiles: name for add/use"},
"profile": profile
}, "required": ["action"]}
})
}
/// Expand one `browser` call into the internal per-action tool at the model-output boundary,
/// so every guard downstream (approvals incl. the vault's, oplog, replay, the remote link, the
/// work log) keeps its stable per-action name.
pub fn expand_call(c: &mut crate::model::ToolCall) {
if c.name != TOOL {
return;
}
let mut v = crate::tools::args_of(&c.args);
let Some(action) = v["action"].as_str().and_then(action_tool) else { return };
let mut name = action.to_string();
if let Some(o) = v.as_object_mut() {
o.remove("action");
if name == "browser_fill_login" {
// fill {kind, id} names the exact vault tool and its id field.
let kind = o.remove("kind").and_then(|k| k.as_str().map(|s| s.to_string())).unwrap_or_else(|| "login".into());
let id = o.remove("id");
name = match kind.as_str() {
"card" => {
if let Some(id) = id {
o.insert("card_id".to_string(), id);
}
"browser_fill_card"
}
"contact" => {
if let Some(id) = id {
o.insert("contact_id".to_string(), id);
}
"browser_fill_contact"
}
_ => {
if let Some(id) = id {
o.insert("login_id".to_string(), id);
}
"browser_fill_login"
}
}
.to_string();
}
}
c.name = name;
c.args = v.to_string();
}
fn downloads_dir() -> std::path::PathBuf {
let d = dirs::download_dir().unwrap_or_else(std::env::temp_dir).join("AnyBot");
let _ = std::fs::create_dir_all(&d);
d
}
/// Run one browser tool. Returns (text for the bot, a JPEG as base64 when it took a picture).
pub fn run(name: &str, args: &Value) -> Result<(String, Option<String>), String> {
let profile = Some(s(args, "profile")).filter(|p| !p.is_empty());
let profile = profile.as_deref();
let (rf, text) = (s(args, "ref"), s(args, "text"));
match name {
"browser_tabs" => return tabs(args, profile).map(|t| (t, None)),
"browser_wait" if s(args, "text").is_empty() => {
let secs = args["seconds"].as_f64().unwrap_or(2.0).clamp(0.2, 30.0);
settle((secs * 1000.0) as u64);
return Ok((format!("已等 {secs} 秒。"), None));
}
_ => {}
}
let mut tab = Tab::current(profile)?;
let r = (|| -> Result<(String, Option<String>), String> {
match name {
"browser_elements" => {
let (title, url, list, more) = elements(&mut tab)?;
Ok((format!("標題:{title}\n網址:{url}\n\n{}{}", list.join("\n"), if more { "\n…還有更多(捲動後再列一次)" } else { "" }), None))
}
"browser_click" | "browser_hover" => {
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
let sig = page_sig(&mut tab);
let before = snapshot(&mut tab);
let (x, y, what) = locate(&mut tab, &rf, &text)?;
settle(150);
if name == "browser_hover" {
// Travel onto it (a jump straight there can skip the hover handlers).
mouse(&mut tab, "mouseMoved", x - 40.0, y - 20.0, "none", 0)?;
settle(60);
mouse(&mut tab, "mouseMoved", x - 10.0, y - 4.0, "none", 0)?;
settle(60);
mouse(&mut tab, "mouseMoved", x, y, "none", 0)?;
// Chrome may not update hover for a window that isn't in front: fire the
// hover events on the element too.
tab.eval(&format!("(() => {{ const el = {}; if (!el) return 0; const r = el.getBoundingClientRect(); const o = {{bubbles: true, cancelable: true, view: window, clientX: r.left + r.width / 2, clientY: r.top + r.height / 2}}; for (const t of ['pointerover', 'pointerenter', 'mouseover', 'mouseenter', 'pointermove', 'mousemove']) el.dispatchEvent(new (t.startsWith('pointer') ? PointerEvent : MouseEvent)(t, o)); return 1; }})()", find_js(&rf, &text)))?;
} else {
let button = match s(args, "button").as_str() {
"right" => "right",
"middle" => "middle",
_ => "left",
};
click_at(&mut tab, x, y, button, if args["double"].as_bool().unwrap_or(false) { 2 } else { 1 })?;
}
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
settle_after(&mut tab, &sig, 900, 4000);
let after = snapshot(&mut tab);
Ok((format!("已{} {what}。\n{}", if name == "browser_hover" { "停在" } else { "點" }, changes(&before, &after)), None))
}
"browser_type" => {
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
let sig = page_sig(&mut tab);
let before = snapshot(&mut tab);
let (x, y, what) = locate(&mut tab, &rf, &text)?;
click_at(&mut tab, x, y, "left", 1)?;
settle(150);
if !args["append"].as_bool().unwrap_or(false) {
// Select what's there so typing replaces it (works in inputs and editors).
tab.eval("(() => { const e = document.activeElement; if (e && 'select' in e && typeof e.select === 'function') e.select(); else document.execCommand('selectAll'); return 1; })()")?;
press(&mut tab, "Backspace")?;
}
tab.call("Input.insertText", json!({"text": s(args, "value")}))?;
if args["submit"].as_bool().unwrap_or(false) {
settle(150);
press(&mut tab, "Enter")?;
}
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
settle_after(&mut tab, &sig, 700, 4000);
let after = snapshot(&mut tab);
let now: String = tab.eval("(() => { const e = document.activeElement; return e && 'value' in e ? (e.type === 'password' ? '' : String(e.value)) : ''; })()")?.as_str().unwrap_or("").chars().take(80).collect();
Ok((format!("已在 {what} 輸入。{}\n{}", if now.is_empty() { String::new() } else { format!("欄位現在是「{now}」。") }, changes(&before, &after)), None))
}
"browser_press" => {
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
let sig = page_sig(&mut tab);
let before = snapshot(&mut tab);
if !rf.is_empty() || !text.is_empty() {
let (x, y, _) = locate(&mut tab, &rf, &text)?;
click_at(&mut tab, x, y, "left", 1)?;
settle(150);
}
press(&mut tab, &s(args, "key"))?;
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
settle_after(&mut tab, &sig, 700, 4000);
let after = snapshot(&mut tab);
Ok((format!("已按 {}。\n{}", s(args, "key"), changes(&before, &after)), None))
}
"browser_select" => {
let option = s(args, "option");
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
let sig = page_sig(&mut tab);
let before = snapshot(&mut tab);
let js = format!(
r#"(() => {{ const el = {}; if (!el) return 'missing'; if (el.tagName !== 'SELECT') return 'custom';
const want = {}.toLowerCase(); const o = [...el.options].find(o => o.text.trim().toLowerCase() === want || o.value.toLowerCase() === want) || [...el.options].find(o => o.text.trim().toLowerCase().includes(want));
if (!o) return 'no-option:' + [...el.options].map(o => o.text.trim()).join(' / ');
Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'value').set.call(el, o.value);
el.dispatchEvent(new Event('input', {{bubbles: true}})); el.dispatchEvent(new Event('change', {{bubbles: true}})); return 'ok'; }})()"#,
find_js(&rf, &text),
serde_json::to_string(&option).unwrap_or_default()
);
let r = tab.eval(&js)?.as_str().unwrap_or("").to_string();
if r == "missing" {
return Err("這一頁找不到那個下拉選單。用 browser_elements 看最新的清單。".into());
}
if let Some(opts) = r.strip_prefix("no-option:") {
return Err(format!("沒有「{option}」這個選項。可選:{opts}"));
}
if r == "custom" {
// A custom dropdown: open it with a real click, then click the option.
let (x, y, _) = locate(&mut tab, &rf, &text)?;
click_at(&mut tab, x, y, "left", 1)?;
settle(700);
let (ox, oy, _) = locate(&mut tab, "", &option).map_err(|_| format!("已打開選單,但找不到「{option}」這個選項。用 browser_elements 看選單裡有什麼。"))?;
click_at(&mut tab, ox, oy, "left", 1)?;
}
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
settle_after(&mut tab, &sig, 700, 4000);
let after = snapshot(&mut tab);
Ok((format!("已選「{option}」。\n{}", changes(&before, &after)), None))
}
"browser_scroll" => {
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
let sig = page_sig(&mut tab);
let before = snapshot(&mut tab);
if !rf.is_empty() || !text.is_empty() {
let (_, _, what) = locate(&mut tab, &rf, &text)?;
settle(500);
let after = snapshot(&mut tab);
return Ok((format!("已捲到 {what}。\n{}", changes(&before, &after)), None));
}
let js = match s(args, "direction").as_str() {
"up" => "window.scrollBy(0, -innerHeight * 0.85)",
"top" => "window.scrollTo(0, 0)",
"bottom" => "window.scrollTo(0, document.documentElement.scrollHeight)",
_ => "window.scrollBy(0, innerHeight * 0.85)",
};
// Pages that scroll an inner box: scroll the biggest scrollable element too.
tab.eval(&format!("(() => {{ {js}; const boxes = [...document.querySelectorAll('*')].filter(e => e.scrollHeight > e.clientHeight + 40 && /(auto|scroll)/.test(getComputedStyle(e).overflowY)).sort((a, b) => b.clientHeight - a.clientHeight); if (boxes[0]) {{ const b = boxes[0]; const d = '{}'; if (d === 'top') b.scrollTop = 0; else if (d === 'bottom') b.scrollTop = b.scrollHeight; else b.scrollTop += (d === 'up' ? -1 : 1) * b.clientHeight * 0.85; }} return 1; }})()", s(args, "direction")))?;
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
settle_after(&mut tab, &sig, 500, 2500);
let after = snapshot(&mut tab);
let pos = tab.eval("JSON.stringify([Math.round(scrollY), document.documentElement.scrollHeight, innerHeight])")?;
Ok((format!("已捲動(位置 {})。\n{}", pos.as_str().unwrap_or(""), changes(&before, &after)), None))
}
"browser_wait" => {
let want = s(args, "text");
let limit = args["seconds"].as_f64().unwrap_or(20.0).clamp(1.0, 60.0);
let start = Instant::now();
while start.elapsed().as_secs_f64() < limit {
let (_, _, lines) = snapshot(&mut tab);
if lines.iter().any(|l| l.contains(&want)) {
return Ok((format!("「{want}」出現了({:.1} 秒)。", start.elapsed().as_secs_f64()), None));
}
settle(500);
}
Err(format!("等了 {limit} 秒,「{want}」沒有出現。"))
}
"browser_find" => {
let want = s(args, "text");
let (_, url, lines) = snapshot(&mut tab);
let hits: Vec<String> = lines.iter().enumerate().filter(|(_, l)| l.to_lowercase().contains(&want.to_lowercase())).take(30).map(|(i, _)| {
let ctx: Vec<&str> = lines[i.saturating_sub(2)..(i + 3).min(lines.len())].iter().map(String::as_str).collect();
format!("• {}", ctx.join(" | ").chars().take(300).collect::<String>())
}).collect();
if hits.is_empty() {
return Ok((format!("這一頁({url})找不到「{want}」。"), None));
}
Ok((format!("「{want}」出現 {} 處:\n{}", hits.len(), hits.join("\n")), None))
}
"browser_table" => table(&mut tab, args).map(|t| (t, None)),
"browser_screenshot" => {
let mut params = json!({"format": "jpeg", "quality": 75});
// Numbered boxes on every clickable thing in view (the default for a plain
// screenshot): look once, then click by number.
let marks = args["marks"].as_bool().unwrap_or(rf.is_empty() && text.is_empty());
let mut listed = vec![];
if marks {
let (_, _, list, _) = elements(&mut tab)?;
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
tab.eval(&MARKS_JS.replace("%DEEP%", DEEP_JS))?;
let in_view: std::collections::HashSet<String> = tab.eval(&format!("(() => {{ {DEEP_JS} return JSON.stringify(deepAll('[data-anybot-ref]').filter(e => {{ const r = e.getBoundingClientRect(); return r.bottom > 0 && r.top < innerHeight && r.right > 0 && r.left < innerWidth && r.width > 1; }}).map(e => e.getAttribute('data-anybot-ref'))); }})()"))?.as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or_default();
listed = list.into_iter().filter(|l| l.strip_prefix('[').and_then(|r| r.split(']').next()).map(|n| in_view.contains(n)).unwrap_or(false)).collect();
settle(120);
}
if !rf.is_empty() || !text.is_empty() {
let v = tab.eval(&format!("(() => {{ const el = {}; if (!el) return null; el.scrollIntoView({{block: 'center'}}); const r = el.getBoundingClientRect(); return JSON.stringify([r.left + scrollX, r.top + scrollY, r.width, r.height]); }})()", find_js(&rf, &text)))?;
let r: Vec<f64> = v.as_str().and_then(|s| serde_json::from_str(s).ok()).ok_or("找不到那個元素")?;
settle(300);
params["clip"] = json!({"x": r[0], "y": r[1], "width": r[2].max(1.0), "height": r[3].max(1.0), "scale": 1});
params["captureBeyondViewport"] = json!(true);
} else if args["full"].as_bool().unwrap_or(false) {
let m = tab.call("Page.getLayoutMetrics", json!({}))?;
let (w, h) = (m["cssContentSize"]["width"].as_f64().unwrap_or(1280.0), m["cssContentSize"]["height"].as_f64().unwrap_or(2000.0).min(12000.0));
params["clip"] = json!({"x": 0, "y": 0, "width": w, "height": h, "scale": 1});
params["captureBeyondViewport"] = json!(true);
}
let r = tab.call("Page.captureScreenshot", params);
if marks {
let _ = tab.eval("(() => { const o = document.getElementById('__anybot_marks'); if (o) o.remove(); return 1; })()");
}
let r = r?;
let b64 = r["data"].as_str().ok_or("截圖失敗")?.to_string();
// Keep pictures a sensible size for the model.
let b64 = shrink(&b64).unwrap_or(b64);
let (title, url, _) = snapshot(&mut tab);
let mut text_out = format!("AnyBot 瀏覽器分頁的截圖:{title}\n網址:{url}");
if marks && !listed.is_empty() {
text_out.push_str(&format!("\n圖上紅框裡的數字就是元素編號,用 browser_click 的 ref 點它:\n{}", listed.join("\n")));
}
Ok((text_out, Some(b64)))
}
"browser_back" => {
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
let sig = page_sig(&mut tab);
let before = snapshot(&mut tab);
match s(args, "direction").as_str() {
"forward" => tab.eval("history.forward()").map(|_| ())?,
"reload" => tab.call("Page.reload", json!({})).map(|_| ())?,
_ => tab.eval("history.back()").map(|_| ())?,
}
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
settle_after(&mut tab, &sig, 900, 5000);
let after = snapshot(&mut tab);
Ok((format!("好了。\n{}", changes(&before, &after)), None))
}
"browser_upload" => {
let path = s(args, "path");
// The browser opens the file from its own disk: Bot 的電腦's when it runs there.
let in_box = botbrowser::using_box();
let full = if in_box { crate::cloudbox::box_path(&path) } else { crate::tools::expand_home(&path).display().to_string() };
let there = if in_box { crate::cloudbox::box_has_file(&full) } else { std::path::Path::new(&full).is_file() };
if !there {
return Err(format!("這台電腦上沒有「{path}」這個檔案"));
}
let js = format!("(() => {{ let el = {}; if (el && el.tagName !== 'INPUT') el = el.querySelector('input[type=file]') || el.closest('label')?.querySelector('input[type=file]'); return el || document.querySelector('input[type=file]'); }})()", find_js(&rf, &text));
let r = tab.call("Runtime.evaluate", json!({"expression": js, "returnByValue": false}))?;
let obj = r["result"]["objectId"].as_str().ok_or("這一頁找不到上傳檔案的欄位")?.to_string();
let before = snapshot(&mut tab);
tab.call("DOM.setFileInputFiles", json!({"files": [full], "objectId": obj}))?;
tab.eval(&format!("(() => {{ const el = {js}; if (el) el.dispatchEvent(new Event('change', {{bubbles: true}})); return 1; }})()"))?;
settle(1000);
// What the field holds now (a page that shows nothing about it left the bot
// checking with five more steps), and what the page did with it.
let held = tab.eval(&format!("(() => {{ const el = {js}; return el && el.files ? Array.from(el.files).map(f => f.name + '(' + f.size + ' 位元組)').join('、') : ''; }})()")).ok().and_then(|v| v.as_str().map(String::from)).unwrap_or_default();
if held.is_empty() {
return Err(format!("放了 {full},但上傳欄位還是空的(這一頁可能要先按「上傳」之類的按鈕,或欄位不是這一個)。"));
}
let after = snapshot(&mut tab);
let seen = changes(&before, &after);
let seen = if seen.contains("沒有變化") { String::new() } else { format!("\n{seen}") };
Ok((format!("已把 {full} 放進上傳欄位,欄位裡現在是:{held}。{seen}"), None))
}
"browser_download" => {
// The file lands on the browser's own disk: Bot 的電腦's when it runs there.
let in_box = botbrowser::using_box();
let dir = if in_box { crate::cloudbox::BOX_DOWNLOADS.to_string() } else { downloads_dir().display().to_string() };
if in_box {
crate::cloudbox::box_mkdir(&dir)?;
}
let clicked_at = std::time::SystemTime::now() - Duration::from_secs(1);
let set = tab.call("Browser.setDownloadBehavior", json!({"behavior": "allow", "downloadPath": dir, "eventsEnabled": false})).or_else(|_| tab.call("Page.setDownloadBehavior", json!({"behavior": "allow", "downloadPath": dir})));
// A Firefox that can't be told where saves to its own Downloads folder: watch that.
let dir = match set {
Ok(_) => dir,
Err(_) if tab.is_firefox() && !in_box => dirs::download_dir().unwrap_or_else(std::env::temp_dir).display().to_string(),
Err(e) => return Err(e),
};
let (x, y, what) = locate(&mut tab, &rf, &text)?;
click_at(&mut tab, x, y, "left", 1)?;
let start = Instant::now();
while start.elapsed().as_secs() < 60 {
settle(700);
// A new file, or one written again since the click (same name replaced).
let now: Vec<String> = if in_box {
crate::cloudbox::box_new_files(&dir, clicked_at.duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0))
} else {
std::fs::read_dir(&dir).map(|r| r.flatten().filter(|e| e.metadata().and_then(|m| m.modified()).map(|t| t >= clicked_at).unwrap_or(false)).map(|e| e.path().display().to_string()).collect()).unwrap_or_default()
};
if let Some(f) = now.iter().find(|p| !(p.ends_with(".crdownload") || p.ends_with(".tmp") || p.ends_with(".part"))) {
return Ok((format!("已點 {what},下載好了:{f}(可以用 read_file 讀它)"), None));
}
}
Err(format!("已點 {what},但 60 秒內沒有下載到檔案(有些網站要先選格式或日期,或在新分頁產生報表)。"))
}
"browser_pdf" => {
let r = tab.call("Page.printToPDF", json!({"printBackground": true}))?;
use base64::Engine;
let bytes = base64::engine::general_purpose::STANDARD.decode(r["data"].as_str().unwrap_or("")).map_err(|e| e.to_string())?;
let (title, _, _) = snapshot(&mut tab);
let safe: String = title.chars().filter(|c| !"/\\:*?\"<>|".contains(*c)).take(60).collect();
let name = format!("{}-{}.pdf", if safe.trim().is_empty() { "網頁".into() } else { safe }, chrono::Utc::now().with_timezone(&crate::routines::person_tz()).format("%Y%m%d-%H%M%S"));
// Saved where the bot can open it: on Bot 的電腦 when that's the browser's computer.
let path = if botbrowser::using_box() {
let p = format!("{}/{name}", crate::cloudbox::BOX_DOWNLOADS);
crate::cloudbox::box_write(&p, &bytes)?;
p
} else {
let p = downloads_dir().join(name);
std::fs::write(&p, bytes).map_err(|e| e.to_string())?;
p.display().to_string()
};
Ok((format!("已存成 PDF:{path}"), None))
}
"browser_dialog" => {
let mut p = json!({"accept": args["accept"].as_bool().unwrap_or(true)});
if !s(args, "text").is_empty() {
p["promptText"] = json!(s(args, "text"));
}
tab.call("Page.handleJavaScriptDialog", p).map_err(|_| "現在沒有對話框".to_string())?;
Ok(("已處理對話框。".into(), None))
}
"browser_eval" => {
let script = s(args, "script");
if let Some(bad) = eval_refusal(&script) {
return Err(bad);
}
let v = tab.eval(&script)?;
let out = if v.is_string() { v.as_str().unwrap_or("").to_string() } else { serde_json::to_string_pretty(&v).unwrap_or_default() };
Ok((out.chars().take(20000).collect(), None))
}
other => Err(format!("沒有「{other}」這個工具")),
}
})();
tab.close();
r
}
/// browser_eval only reads the page: no requests to the site's servers, no navigation, no
/// submitting. Calling a site's private APIs gets accounts flagged (Shopee answered with a captcha).
pub fn eval_refusal(script: &str) -> Option<String> {
let l = script.to_lowercase().replace(' ', "");
let banned = ["fetch(", "xmlhttprequest", "sendbeacon", "websocket(", "eventsource(", ".submit(", "location=", "location.href=", "location.assign", "location.replace", "window.open(", "document.cookie=", "localstorage.setitem", "import(", "navigator.serviceworker", "window.fetch="];
banned.iter().find(|b| l.contains(*b)).map(|b| {
format!("browser_eval 只能讀取頁面上已經有的東西,不能發送請求或改變網站(不允許 {b})。請像人一樣用 browser_click / browser_type / browser_select 操作頁面;直接呼叫網站的內部 API 會被當成機器人(例如跳出拼圖驗證)。")
})
}
/// Re-encode a screenshot no wider than 1280 px.
fn shrink(b64: &str) -> Option<String> {
use base64::Engine;
let bytes = base64::engine::general_purpose::STANDARD.decode(b64).ok()?;
let img = image::load_from_memory(&bytes).ok()?;
let img = if img.width() > 1280 { img.resize(1280, u32::MAX, image::imageops::FilterType::Triangle) } else { img };
let img = if img.height() > 4000 { img.crop_imm(0, 0, img.width(), 4000) } else { img };
let mut out = std::io::Cursor::new(vec![]);
image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, 72).encode_image(&img.to_rgb8()).ok()?;
Some(base64::engine::general_purpose::STANDARD.encode(out.into_inner()))
}
/// A job is done with this browser: keep the page it ended on, close the rest (the list is
/// latest-used first). A browser that isn't open stays closed. Not offered to the model; the
/// loop calls it (browser_tabs action "tidy") when a job ends.
fn tidy(profile: Option<&str>) -> Result<String, String> {
let job = botbrowser::job();
// (Its tabs are nobody's from here on, whatever happens below.)
let mine = if job.is_empty() { vec![] } else { botbrowser::release(&job) };
let Some(port) = botbrowser::running_port(profile) else { return Ok("瀏覽器沒有開著。".into()) };
let kind = botbrowser::kind_of(profile);
let list = botbrowser::pages_of(kind, port)?;
let close: Vec<&Value> = if job.is_empty() {
// (An older brain that doesn't say which job: the old way, all but the front page.)
list.iter().skip(1).collect()
} else {
// The job's own tabs but its latest; other jobs' and the person's are left alone.
list.iter().filter(|t| mine.iter().any(|m| t["id"].as_str() == Some(m.as_str()))).skip(1).collect()
};
let closed = close.iter().filter(|t| botbrowser::close_tab(kind, port, t["id"].as_str().unwrap_or("")).is_ok()).count();
Ok(format!("關了 {closed} 個用完的分頁。"))
}
fn tabs(args: &Value, profile: Option<&str>) -> Result<String, String> {
if s(args, "action") == "tidy" {
return tidy(profile);
}
let (_, port, kind) = port_of(profile)?;
// Another job's tabs aren't listed, switched to or closed.
let list: Vec<Value> = botbrowser::pages_of(kind, port)?.into_iter().filter(|t| !botbrowser::others_tab(t["id"].as_str().unwrap_or(""))).collect();
let current = botbrowser::pick_tab(&list).and_then(|t| t["id"].as_str()).unwrap_or("").to_string();
let idx = args["index"].as_u64().map(|i| i as usize);
let pick = |i: Option<usize>| -> Result<&Value, String> { i.and_then(|i| list.get(i.wrapping_sub(1))).ok_or_else(|| "請給分頁編號(先用 action=list 看)".to_string()) };
match s(args, "action").as_str() {
"switch" => {
let t = pick(idx)?;
botbrowser::activate_tab(kind, port, t["id"].as_str().unwrap_or(""))?;
botbrowser::claim(t["id"].as_str().unwrap_or(""));
Ok(format!("已切到分頁 {}:{}({})", idx.unwrap_or(0), t["title"].as_str().unwrap_or(""), t["url"].as_str().unwrap_or("")))
}
"close" => {
let t = pick(idx)?;
botbrowser::close_tab(kind, port, t["id"].as_str().unwrap_or(""))?;
Ok(format!("已關閉分頁:{}", t["title"].as_str().unwrap_or("")))
}
"new" => {
let url = crate::web::normalize_url(&s(args, "url")).unwrap_or_else(|_| "about:blank".into());
let made = botbrowser::new_tab(kind, port, &url)?;
botbrowser::claim(&made);
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
let loaded = wait_loaded(profile, 10_000);
Ok(format!("已開新分頁:{url}{}", if loaded { "" } else { "(還在載入,可以用 browser_wait 等它)" }))
}
_ => Ok(if list.is_empty() {
"沒有開著的分頁。".into()
} else {
// Untitled pages (about:blank, bare pages) show their url, so the model never sees an empty line.
list.iter().enumerate().map(|(i, t)| {
let url = t["url"].as_str().unwrap_or("");
let title = t["title"].as_str().unwrap_or("");
let title = if title.is_empty() { url } else { title };
format!("{}. {}{}\n {}", i + 1, title, if t["id"].as_str() == Some(current.as_str()) { "(目前)" } else { "" }, url)
}).collect::<Vec<_>>().join("\n")
}),
}
}
/// HTML tables and ARIA grids as Markdown.
fn table(tab: &mut Tab, args: &Value) -> Result<String, String> {
let js = r#"(() => {
const clean = s => (s || '').replace(/\s+/g, ' ').trim().replace(/\|/g, '/');
const vis = el => { const r = el.getBoundingClientRect(); return r.width > 1 && r.height > 1; };
const tables = [];
for (const t of document.querySelectorAll('table')) {
if (!vis(t)) continue;
const rows = [...t.querySelectorAll('tr')].map(r => [...r.children].filter(c => c.tagName === 'TD' || c.tagName === 'TH').map(c => clean(c.innerText))).filter(r => r.some(x => x));
if (rows.length) tables.push({caption: clean((t.caption || {}).innerText || t.getAttribute('aria-label') || ''), rows});
}
for (const g of document.querySelectorAll('[role=grid],[role=table],[role=treegrid]')) {
if (!vis(g) || g.tagName === 'TABLE') continue;
const rows = [...g.querySelectorAll('[role=row]')].map(r => [...r.querySelectorAll('[role=columnheader],[role=cell],[role=gridcell],[role=rowheader]')].map(c => clean(c.innerText))).filter(r => r.some(x => x));
if (rows.length) tables.push({caption: clean(g.getAttribute('aria-label') || ''), rows});
}
// Fixed-header tables split into a header table and a body table (common in admin pages):
// glue a lone header row onto the next table with the same width.
const out = [];
for (let i = 0; i < tables.length; i++) {
const t = tables[i], n = tables[i + 1];
if (t.rows.length === 1 && n && n.rows[0] && n.rows[0].length === t.rows[0].length) { out.push({caption: t.caption || n.caption, rows: t.rows.concat(n.rows)}); i++; }
else out.push(t);
}
return JSON.stringify(out);
})()"#;
let v = tab.eval(js)?;
let tables: Vec<Value> = v.as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or_default();
if tables.is_empty() {
return Ok("這一頁沒有表格。數字如果在圖表裡,用 browser_screenshot 看,或用 browser_read 讀文字。".into());
}
let max_rows = args["max_rows"].as_u64().unwrap_or(200) as usize;
let pick: Vec<(usize, &Value)> = match (args["index"].as_u64(), args["all"].as_bool().unwrap_or(false)) {
(Some(i), _) => tables.iter().enumerate().filter(|(j, _)| *j + 1 == i as usize).collect(),
(None, true) => tables.iter().enumerate().collect(),
(None, false) => tables.iter().enumerate().max_by_key(|(_, t)| t["rows"].as_array().map(|r| r.len()).unwrap_or(0)).into_iter().collect(),
};
let mut out = format!("這一頁有 {} 個表格。\n", tables.len());
for (i, t) in pick {
let rows: Vec<Vec<String>> = t["rows"].as_array().map(|rs| rs.iter().map(|r| r.as_array().map(|c| c.iter().map(|x| x.as_str().unwrap_or("").to_string()).collect()).unwrap_or_default()).collect()).unwrap_or_default();
let width = rows.iter().map(|r| r.len()).max().unwrap_or(0);
out.push_str(&format!("\n### 表格 {}{}({} 列)\n", i + 1, if t["caption"].as_str().unwrap_or("").is_empty() { String::new() } else { format!(":{}", t["caption"].as_str().unwrap_or("")) }, rows.len().saturating_sub(1)));
for (k, r) in rows.iter().take(max_rows + 1).enumerate() {
let mut cells = r.clone();
cells.resize(width, String::new());
out.push_str(&format!("| {} |\n", cells.join(" | ")));
if k == 0 {
out.push_str(&format!("|{}\n", "---|".repeat(width)));
}
}
if rows.len() > max_rows + 1 {
out.push_str(&format!("…還有 {} 列(可以捲動或換頁再讀)\n", rows.len() - max_rows - 1));
}
}
Ok(out)
}
#[cfg(test)]
mod tests {
#[test]
fn the_one_tool_maps_every_action() {
let mut c = |args: &str| {
let mut call = crate::model::ToolCall { id: "x".into(), name: TOOL.into(), args: args.into(), status: crate::model::ToolStatus::Done };
expand_call(&mut call);
(call.name, call.args)
};
let (n, a) = c(r#"{"action":"click","ref":"12","double":true}"#);
assert_eq!(n, "browser_click");
assert!(a.contains(r#""ref":"12""#) && a.contains("double") && !a.contains("action"), "{a}");
let (n, a) = c(r#"{"action":"fill","kind":"card","id":"7"}"#);
assert_eq!(n, "browser_fill_card");
assert!(a.contains(r#""card_id":"7""#) && !a.contains("kind"), "{a}");
let (n, _) = c(r#"{"action":"fill"}"#);
assert_eq!(n, "browser_fill_login");
let (n, _) = c(r#"{"action":"nope"}"#);
assert_eq!(n, TOOL, "an unknown action stays put (the runner explains it)");
// Every internal browser tool is reachable from the one schema.
let mut mapped: Vec<&str> = tool(&json!({"type": "string"}))["parameters"]["properties"]["action"]["enum"]
.as_array()
.unwrap()
.iter()
.map(|a| action_tool(a.as_str().unwrap()).unwrap())
.collect();
mapped.extend(["browser_fill_card", "browser_fill_contact"]);
for n in NAMES {
assert!(mapped.contains(&n), "internal tool {n} is unreachable from browser");
}
}
use super::*;
#[test]
fn keys_and_changes() {
assert_eq!(key_info("Enter").unwrap().0, "Enter");
assert!(key_info("F13").is_none());
let a = ("t".to_string(), "u1".to_string(), vec!["a".to_string(), "b".to_string()]);
let b = ("t".to_string(), "u2".to_string(), vec!["a".to_string(), "c".to_string()]);
let c = changes(&a, &b);
assert!(c.contains("網址變成:u2") && c.contains("新出現的文字:\nc"));
assert!(changes(&a, &a).contains("沒有變化"));
}
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
#[test]
fn the_front_page_tab_is_used() {
let list = json!([
{"type": "service_worker", "url": "https://x", "webSocketDebuggerUrl": "ws://a"},
{"type": "page", "url": "devtools://devtools", "webSocketDebuggerUrl": "ws://b"},
{"type": "page", "url": "https://www.threads.net/", "title": "Threads", "webSocketDebuggerUrl": "ws://c"},
{"type": "page", "url": "https://example.com/", "webSocketDebuggerUrl": "ws://d"}
]);
let p = pages(&list);
assert_eq!(p.len(), 2);
assert_eq!(p[0]["webSocketDebuggerUrl"], "ws://c");
}
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
/// Controls inside web components (open shadow roots, two levels deep) get numbers and can be
/// clicked and typed into, like Chromium's settings pages and many app frameworks. Headless:
/// `ANYBOT_DATA_DIR=<scratch> ANYBOT_BROWSER_ENGINE=firefox ANYBOT_FIREFOX_HEADLESS=1 cargo test
/// numbers_controls_inside_web_components -- --ignored --nocapture` (or Chrome, with a window).
#[test]
#[ignore]
fn numbers_controls_inside_web_components() {
use std::io::{Read, Write};
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = l.local_addr().unwrap().port();
std::thread::spawn(move || {
for mut c in l.incoming().flatten() {
let mut buf = [0u8; 4096];
let _ = c.read(&mut buf);
let body = r#"<html><head><meta charset=utf-8><title>元件</title></head><body>
<h1>設定</h1><my-panel></my-panel><p id=out>還沒按</p>
<script>
customElements.define('my-toggle', class extends HTMLElement { constructor() { super(); const r = this.attachShadow({mode: 'open'});
r.innerHTML = '<div role=switch tabindex=0 aria-label="自動更新" aria-checked=false style="width:40px;height:20px;background:#ccc"></div>';
const s = r.firstChild; s.addEventListener('click', () => { s.setAttribute('aria-checked', 'true'); document.getElementById('out').textContent = '自動更新:開'; }); } });
customElements.define('my-panel', class extends HTMLElement { constructor() { super(); const r = this.attachShadow({mode: 'open'});
r.innerHTML = '<input id=n placeholder="名稱"> <button id=go>開始</button> <my-toggle></my-toggle>';
r.getElementById('go').addEventListener('click', () => { document.getElementById('out').textContent = '按了開始:' + r.getElementById('n').value; }); } });
</script></body></html>"#;
let _ = write!(c, "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len());
let _ = c.write_all(body.as_bytes());
}
});
crate::botbrowser::add("shadow-test").unwrap();
let p = |v: Value| {
let mut v = v;
v["profile"] = json!("shadow-test");
v
};
let go = || -> Result<(), String> {
crate::botbrowser::open(&format!("http://127.0.0.1:{port}/"), Some("shadow-test"))?;
let (els, _) = run("browser_elements", &p(json!({})))?;
println!("{els}");
for want in ["「名稱」", "「開始」", "「自動更新」"] {
assert!(els.contains(want), "{want} missing: {els}");
}
let n = els.lines().find(|l| l.contains("「開始」")).and_then(|l| l.trim_start().strip_prefix('[')).and_then(|r| r.split(']').next()).unwrap().to_string();
run("browser_type", &p(json!({"text": "名稱", "value": "小明"})))?;
let (r, _) = run("browser_click", &p(json!({"ref": n})))?;
assert!(r.contains("按了開始:小明"), "{r}");
let (r, _) = run("browser_click", &p(json!({"text": "自動更新"})))?;
assert!(r.contains("自動更新:開"), "{r}");
Ok(())
};
let r = go();
let _ = crate::botbrowser::remove("shadow-test");
r.unwrap();
}
/// A dashboard like Shopee's affiliate pages: a React-style date picker that only reacts to
/// real clicks, a custom dropdown, a table, a hover menu, an export. Run with ANYBOT_DATA_DIR
/// set to a scratch folder; opens a browser window for a moment.
#[test]
#[ignore]
fn drives_a_dashboard() {
use std::io::{Read, Write};
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = l.local_addr().unwrap().port();
std::thread::spawn(move || {
for mut c in l.incoming().flatten() {
let mut buf = [0u8; 4096];
let n = c.read(&mut buf).unwrap_or(0);
let req = String::from_utf8_lossy(&buf[..n]).to_string();
let (ctype, body): (&str, Vec<u8>) = if req.starts_with("GET /export") {
("text/csv\r\nContent-Disposition: attachment; filename=\"report.csv\"", "商品,佣金\nA,10\n".as_bytes().to_vec())
} else {
("text/html; charset=utf-8", r#"<html><head><title>成效</title><style>.day{cursor:pointer;display:inline-block;padding:4px}</style></head><body>
<div id=range class=pick tabindex=0 aria-label="數據周期">2026-11-03</div>
<div id=cal style="display:none"><span class=day>近7天</span><span class=day>近30天</span></div>
<div id=menu>商品分析</div><div id=sub style="display:none">分潤率排行</div>
<div id=dd role=combobox aria-label="排序">預設</div><ul id=opts role=listbox style="display:none"><li role=option>佣金由高到低</li><li role=option>銷量</li></ul>
<input id=q placeholder="搜尋商品">
<div role=table aria-label="商品"><div role=row><span role=columnheader>商品</span><span role=columnheader>分潤率</span></div><div role=row><span role=cell>手機殼</span><span role=cell>8%</span></div><div role=row><span role=cell>乳液</span><span role=cell>12%</span></div></div>
<p id=out>點擊 0</p><a href="/export">匯出報表</a>
<script>
// Only trusted (real) clicks open the calendar, like many React pickers.
range.addEventListener('mousedown', e => { if (e.isTrusted) cal.style.display = 'block'; });
for (const d of document.querySelectorAll('.day')) d.addEventListener('click', e => { if (!e.isTrusted) return; range.innerText = d.innerText; out.innerText = d.innerText === '近30天' ? '點擊 60,訂單 3' : '點擊 9'; cal.style.display = 'none'; });
menu.addEventListener('mouseover', () => sub.style.display = 'block');
dd.addEventListener('click', () => opts.style.display = 'block');
for (const o of opts.children) o.addEventListener('click', () => { dd.innerText = o.innerText; opts.style.display = 'none'; });
q.addEventListener('keydown', e => { if (e.key === 'Enter') out.innerText = '搜尋:' + q.value; });
</script></body></html>"#.as_bytes().to_vec())
};
let _ = write!(c, "HTTP/1.1 200 OK\r\nContent-Type: {ctype}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len());
let _ = c.write_all(&body);
}
});
crate::botbrowser::add("drive-test").unwrap();
let p = |v: Value| { let mut v = v; v["profile"] = json!("drive-test"); v };
let go = || -> Result<(), String> {
crate::botbrowser::open(&format!("http://127.0.0.1:{port}/"), Some("drive-test"))?;
let (els, _) = run("browser_elements", &p(json!({})))?;
assert!(els.contains("「數據周期」"), "{els}");
run("browser_hover", &p(json!({"text": "商品分析"})))?;
let (shown, _) = run("browser_eval", &p(json!({"script": "getComputedStyle(document.getElementById('sub')).display"})))?;
assert_eq!(shown.trim(), "block");
// Real click opens the picker; a calendar day (only a pointer cursor) is listed and clickable.
let (r, _) = run("browser_click", &p(json!({"text": "數據周期"})))?;
assert!(r.contains("近30天"), "{r}");
let (r, _) = run("browser_click", &p(json!({"text": "近30天"})))?;
assert!(r.contains("點擊 60,訂單 3"), "{r}");
let (r, _) = run("browser_select", &p(json!({"text": "排序", "option": "佣金由高到低"})))?;
assert!(r.contains("佣金由高到低"), "{r}");
let (r, _) = run("browser_type", &p(json!({"text": "搜尋商品", "value": "手機殼", "submit": true})))?;
assert!(r.contains("搜尋:手機殼"), "{r}");
let (t, _) = run("browser_table", &p(json!({})))?;
assert!(t.contains("| 商品 | 分潤率 |") && t.contains("| 乳液 | 12% |"), "{t}");
let (f, _) = run("browser_find", &p(json!({"text": "手機殼"})))?;
assert!(f.contains("搜尋:手機殼") && f.contains("8%"), "{f}");
let (_, img) = run("browser_screenshot", &p(json!({})))?;
assert!(img.map(|b| b.len() > 1000).unwrap_or(false));
let (tabs, _) = run("browser_tabs", &p(json!({"action": "list"})))?;
assert!(tabs.contains("成效"), "{tabs}");
let (d, _) = run("browser_download", &p(json!({"text": "匯出報表"})))?;
assert!(d.contains("report") && d.contains(".csv"), "{d}");
let (v, _) = run("browser_eval", &p(json!({"script": "document.querySelectorAll('[role=row]').length"})))?;
assert_eq!(v.trim(), "3");
assert!(run("browser_eval", &p(json!({"script": "fetch('/api/x')"}))).is_err());
let (t, img) = run("browser_screenshot", &p(json!({})))?;
assert!(img.is_some() && t.contains("紅框") && t.contains("[1]"), "{t}");
let (gone, _) = run("browser_eval", &p(json!({"script": "!document.getElementById('__anybot_marks')"})))?;
assert_eq!(gone.trim(), "true");
Ok(())
};
let r = go();
let _ = crate::botbrowser::remove("drive-test");
r.unwrap();
}
}