AnyBot/src/botbrowser.rs

625 lines
28 KiB
Rust
Raw Normal View History

// AnyBot's own browser window, like Grok Bot's own browser: a separate Chrome-family profile that
// AnyBot starts and talks to directly (Chrome's DevTools protocol on a local port), so reading
// and opening pages needs no browser setting ("允許來自 Apple 事件的 JavaScript") and no macOS
// "control this app" prompt. The person logs in to sites in this window once; the profile keeps it.
//
// Profiles (設定檔): the person's own Chrome profiles can't be used (macOS refuses other apps
// Chrome's data folder, and Chrome refuses remote control of its normal folder), so AnyBot's
// browser keeps its own: 「預設」 plus any the person or the bot adds (工作, 私人…). Each is its own
// browser folder and window with its own logins; switching picks which one the bot uses.
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
use serde_json::Value;
use std::path::PathBuf;
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
/// Local port for talking to the bot's browser (estimated; any free local port works).
pub const PORT: u16 = 9333;
/// How long to wait for the browser to come up (estimated).
const START_TIMEOUT_SECS: u64 = 20;
/// The first profile: the browser folder made before profiles existed (its logins stay).
pub const DEFAULT_PROFILE: &str = "預設";
/// The person's own browser (read through accessibility, like Grok Bot's Computer Use helper):
/// whichever Chrome profile they have open there is the one read.
pub const PERSON_PROFILE: &str = "你的瀏覽器";
/// Other ways people name their own browser.
const PERSON_ALIASES: [&str; 6] = ["你的 chrome", "我的 chrome", "我的瀏覽器", "你的chrome", "我的chrome", "my chrome"];
/// Longest profile name (estimated).
pub const PROFILE_NAME_MAX_CHARS: usize = 30;
/// File Chrome writes into its folder with the port it chose (`--remote-debugging-port=0`).
const ACTIVE_PORT_FILE: &str = "DevToolsActivePort";
/// Chrome-family browsers that can be started as AnyBot's own window.
#[cfg(target_os = "macos")]
const CANDIDATES: &[(&str, &str)] = &[
("Google Chrome", "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"),
("Brave Browser", "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"),
("Microsoft Edge", "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"),
("Chromium", "/Applications/Chromium.app/Contents/MacOS/Chromium"),
];
/// Linux: the usual install places (distribution packages, the vendors' .deb/.rpm, Snap, Flatpak).
#[cfg(not(target_os = "macos"))]
const CANDIDATES: &[(&str, &str)] = &[
("Google Chrome", "/usr/bin/google-chrome-stable"),
("Google Chrome", "/usr/bin/google-chrome"),
("Google Chrome", "/opt/google/chrome/chrome"),
("Brave Browser", "/usr/bin/brave-browser"),
("Brave Browser", "/opt/brave.com/brave/brave"),
("Microsoft Edge", "/usr/bin/microsoft-edge"),
("Chromium", "/usr/bin/chromium"),
("Chromium", "/usr/bin/chromium-browser"),
("Chromium", "/snap/bin/chromium"),
("Chromium", "/var/lib/flatpak/exports/bin/org.chromium.Chromium"),
];
pub fn engine() -> Option<(&'static str, &'static str)> {
CANDIDATES.iter().copied().find(|(_, p)| std::path::Path::new(p).exists())
}
pub fn available() -> bool {
if using_box() {
return true;
}
engine().is_some()
}
fn data_dir() -> PathBuf {
crate::model::State::path().parent().map(PathBuf::from).unwrap_or_else(std::env::temp_dir)
}
/// The 「預設」 profile lives with AnyBot's data (logins stay between runs).
pub fn profile_dir() -> PathBuf {
data_dir().join("browser")
}
fn profiles_root() -> PathBuf {
data_dir().join("browser-profiles")
}
fn dir_for(name: &str) -> PathBuf {
if name == DEFAULT_PROFILE { profile_dir() } else { profiles_root().join(name) }
}
/// All profiles: 「預設」 first, then the added ones by name.
pub fn profiles() -> Vec<String> {
let mut extra: Vec<String> = std::fs::read_dir(profiles_root())
.map(|it| it.flatten().filter(|e| e.path().is_dir()).filter_map(|e| e.file_name().to_str().map(str::to_string)).collect())
.unwrap_or_default();
extra.sort();
let mut all = vec![DEFAULT_PROFILE.to_string()];
all.extend(extra.into_iter().filter(|n| n != DEFAULT_PROFILE));
all
}
fn current_file() -> PathBuf {
data_dir().join("browser-profile.txt")
}
/// Every choice: AnyBot's own profiles, then the person's own browser.
pub fn choices() -> Vec<String> {
let mut all = profiles();
// Reading the person's own browser goes through macOS accessibility; not on Linux.
if cfg!(target_os = "macos") {
all.push(PERSON_PROFILE.to_string());
}
all
}
/// The profile the bot uses when it isn't told which.
pub fn current() -> String {
let want = std::fs::read_to_string(current_file()).unwrap_or_default().trim().to_string();
match_profile(&choices(), &want).unwrap_or_else(|| DEFAULT_PROFILE.to_string())
}
pub fn set_current(name: &str) -> Result<String, String> {
let name = resolve(Some(name))?;
std::fs::write(current_file(), &name).map_err(|e| format!("存不了設定檔選擇:{e}"))?;
Ok(name)
}
/// Find a profile by name, ignoring case and spaces at the ends.
pub fn match_profile(list: &[String], want: &str) -> Option<String> {
let w = want.trim().to_lowercase();
if w.is_empty() {
return None;
}
if PERSON_ALIASES.contains(&w.as_str()) && list.iter().any(|n| n == PERSON_PROFILE) {
return Some(PERSON_PROFILE.to_string());
}
list.iter().find(|n| n.to_lowercase() == w).cloned()
}
/// Which profile a tool call means: the one named, or the current one.
pub fn resolve(requested: Option<&str>) -> Result<String, String> {
if using_box() {
return Ok(DEFAULT_PROFILE.to_string());
}
match requested.map(str::trim).filter(|s| !s.is_empty()) {
None => Ok(current()),
Some(r) => {
let all = choices();
match_profile(&all, r).ok_or_else(|| format!("AnyBot 瀏覽器沒有叫「{r}」的設定檔。現有:{}。要新的可以用 browser_profiles 新增(action: add)。", all.join("、")))
}
}
}
/// A usable profile name: not empty, not too long, no slashes or dots-only names.
pub fn check_name(name: &str) -> Result<String, String> {
let n = name.trim();
if n.is_empty() {
return Err("請給設定檔名稱".into());
}
if n.chars().count() > PROFILE_NAME_MAX_CHARS {
return Err(format!("設定檔名稱最多 {PROFILE_NAME_MAX_CHARS} 個字"));
}
if n == PERSON_PROFILE || PERSON_ALIASES.contains(&n.to_lowercase().as_str()) {
return Err(format!("「{n}」是留給你自己瀏覽器的名字,請換一個"));
}
if n.contains('/') || n.contains(':') || n.starts_with('.') || n.chars().any(char::is_control) {
return Err("設定檔名稱不能有 / 或 : 也不能用 . 開頭".into());
}
Ok(n.to_string())
}
/// Make a new empty profile (its own folder, so its own logins).
pub fn add(name: &str) -> Result<String, String> {
let n = check_name(name)?;
if let Some(have) = match_profile(&profiles(), &n) {
return Ok(have);
}
std::fs::create_dir_all(profiles_root().join(&n)).map_err(|e| format!("建不了設定檔:{e}"))?;
Ok(n)
}
/// Remove an added profile and its logins (the person's choice, from Settings). 「預設」 and the
/// person's own browser can't be removed, and an open window must be closed first.
pub fn remove(name: &str) -> Result<(), String> {
let all = profiles();
let n = match_profile(&all, name).ok_or_else(|| format!("沒有叫「{name}」的設定檔"))?;
if n == DEFAULT_PROFILE || n == PERSON_PROFILE {
return Err(format!("「{n}」不能移除"));
}
if live_port(&n).is_some() {
return Err(format!("「{n}」的視窗還開著,請先關掉它再移除。"));
}
let dir = profiles_root().join(&n);
// Only ever a folder directly inside AnyBot's browser-profiles folder.
if dir.parent() != Some(profiles_root().as_path()) {
return Err("設定檔位置不對,沒有移除".into());
}
std::fs::remove_dir_all(&dir).map_err(|e| format!("移除不了「{n}」:{e}"))?;
if current() == n || !choices().contains(&current()) {
let _ = std::fs::write(current_file(), DEFAULT_PROFILE);
}
Ok(())
}
pub(crate) fn local() -> ureq::Agent {
ureq::Agent::config_builder().timeout_global(Some(Duration::from_secs(5))).http_status_as_error(false).build().into()
}
/// The fixed port of 「預設」 (`ANYBOT_BROWSER_PORT` overrides it, for testing).
fn default_port() -> u16 {
std::env::var("ANYBOT_BROWSER_PORT").ok().and_then(|p| p.parse().ok()).unwrap_or(PORT)
}
fn endpoint(port: u16, path: &str) -> String {
format!("http://127.0.0.1:{port}{path}")
}
fn alive(port: u16) -> bool {
port != 0 && local().get(&endpoint(port, "/json/version")).call().map(|r| r.status().as_u16() == 200).unwrap_or(false)
}
/// First line of Chrome's DevToolsActivePort file.
pub fn parse_active_port(text: &str) -> Option<u16> {
text.lines().next()?.trim().parse().ok().filter(|p| *p != 0)
}
/// The port a running window of this profile answers on, if any.
fn live_port(name: &str) -> Option<u16> {
let file = std::fs::read_to_string(dir_for(name).join(ACTIVE_PORT_FILE)).ok().and_then(|t| parse_active_port(&t));
let fixed = (name == DEFAULT_PROFILE).then(default_port);
[fixed, file].into_iter().flatten().find(|p| alive(*p))
}
/// Percent-encode a URL so it can ride in `/json/new?<url>`.
pub fn encode_component(s: &str) -> String {
s.bytes()
.map(|b| match b {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' | b':' | b'/' => (b as char).to_string(),
_ => format!("%{b:02X}"),
})
.collect()
}
thread_local! {
/// Browser steps on this thread go to Bot 的電腦's Chromium (its box), not this computer's.
static IN_BOX: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
}
/// Run `f` with the browser steps on Bot 的電腦's own Chromium.
pub fn in_box<T>(f: impl FnOnce() -> T) -> T {
IN_BOX.with(|b| b.set(true));
let r = f();
IN_BOX.with(|b| b.set(false));
r
}
pub fn using_box() -> bool {
IN_BOX.with(|b| b.get())
}
/// The debugging port of a bot browser that is already open (None when it isn't; never starts one).
pub fn running_port(profile: Option<&str>) -> Option<u16> {
if using_box() {
return crate::cloudbox::browser_running().then(crate::cloudbox::browser_port).and_then(Result::ok);
}
live_port(&resolve(profile).ok()?)
}
// ---------- each job its own tabs ----------
// Bots share a browser (the box's Chromium has one profile for all of them). Every step used to
// act on the tab in front, so two jobs at once read and clicked on each other's pages, and one
// finishing closed the other's. A job now owns the tabs it opens or lands on (a popup it clicked
// open too): its steps use the latest of those, and another job's tabs are never used, listed
// or closed by it.
thread_local! {
/// The job whose browser step runs on this thread (empty: not a bot's job).
static JOB: std::cell::RefCell<String> = const { std::cell::RefCell::new(String::new()) };
}
/// (tab id, job) for the tabs running jobs own.
static OWNERS: std::sync::Mutex<Vec<(String, String)>> = std::sync::Mutex::new(Vec::new());
/// Run `f` as a step of job `job` (on its own tabs).
pub fn as_job<T>(job: &str, f: impl FnOnce() -> T) -> T {
let before = JOB.with(|j| std::mem::replace(&mut *j.borrow_mut(), job.to_string()));
let r = f();
JOB.with(|j| *j.borrow_mut() = before);
r
}
pub fn job() -> String {
JOB.with(|j| j.borrow().clone())
}
fn owner(tab: &str) -> Option<String> {
OWNERS.lock().unwrap_or_else(|e| e.into_inner()).iter().find(|(t, _)| t == tab).map(|(_, j)| j.clone())
}
/// Another running job owns this tab.
pub fn others_tab(tab: &str) -> bool {
owner(tab).is_some_and(|j| j != job())
}
/// This step's job takes the tab, when it is nobody's.
pub fn claim(tab: &str) {
let me = job();
if me.is_empty() || tab.is_empty() {
return;
}
let mut o = OWNERS.lock().unwrap_or_else(|e| e.into_inner());
if !o.iter().any(|(t, _)| t == tab) {
o.push((tab.to_string(), me));
}
}
/// The job is over: its tabs are nobody's again (returned).
pub fn release(job: &str) -> Vec<String> {
let mut o = OWNERS.lock().unwrap_or_else(|e| e.into_inner());
let mine = o.iter().filter(|(_, j)| j == job).map(|(t, _)| t.clone()).collect();
o.retain(|(_, j)| j != job);
mine
}
/// The tab of a page list (latest-used first) this step works on: its job's latest own tab, else
/// the latest one no other job owns (a job's first step).
pub fn pick_tab(list: &[Value]) -> Option<&Value> {
let me = job();
let id = |t: &Value| t["id"].as_str().unwrap_or("").to_string();
if !me.is_empty() {
if let Some(t) = list.iter().find(|t| owner(&id(t)).as_deref() == Some(me.as_str())) {
return Some(t);
}
}
list.iter().find(|t| !others_tab(&id(t)))
}
/// The open page tabs' ids (none when the browser isn't open; it isn't started for this).
pub fn tab_ids(profile: Option<&str>) -> Vec<String> {
let Some(port) = running_port(profile) else { return vec![] };
let list: Value = local().get(&endpoint(port, "/json/list")).call().ok().and_then(|r| r.into_body().read_json().ok()).unwrap_or_default();
list.as_array().map(|a| a.iter().filter(|t| t["type"] == "page").filter_map(|t| t["id"].as_str().map(String::from)).collect()).unwrap_or_default()
}
/// Tabs that appeared during this step (a popup a click opened) are its job's.
pub fn adopt_new(profile: Option<&str>, before: &[String]) {
if job().is_empty() {
return;
}
for id in tab_ids(profile) {
if !before.contains(&id) {
claim(&id);
}
}
}
/// Start this profile's window if it isn't running. Returns (browser name, port, just started).
pub fn ensure(profile: &str, first_url: Option<&str>) -> Result<(&'static str, u16, bool), String> {
if using_box() {
let fresh = !crate::cloudbox::browser_running();
let port = crate::cloudbox::browser_port()?;
if fresh {
if let Some(u) = first_url {
let _ = local().put(&endpoint(port, &format!("/json/new?{}", encode_component(u)))).send_empty();
}
}
return Ok(("它桌面上的 Chromium", port, fresh && first_url.is_some()));
}
let Some((name, bin)) = engine() else { return Err("這台電腦沒有 Chrome、Brave、Edge 或 Chromium,沒辦法開 AnyBot 自己的瀏覽器。".into()) };
if profile == PERSON_PROFILE {
return Err("「你的瀏覽器」是你自己的視窗,AnyBot 不會另外開。".into());
}
if let Some(p) = live_port(profile) {
return Ok((name, p, false));
}
let dir = dir_for(profile);
let _ = std::fs::create_dir_all(&dir);
let _ = std::fs::remove_file(dir.join(ACTIVE_PORT_FILE));
// 「預設」 keeps its fixed port; other profiles let Chrome pick a free one and read it back.
let launch_port = if profile == DEFAULT_PROFILE { default_port() } else { 0 };
// Defense in depth: this becomes a Chrome launch argument. Every caller normalizes to a
// full URL first; a bare "-…" (a Chrome flag) must never get here from a tool argument.
let url = match first_url {
Some(u) if u.starts_with("http://") || u.starts_with("https://") => u,
Some(_) => return Err("第一個網址必須是 http(s) 網址".into()),
None => "about:blank",
};
let mut c = Command::new(bin);
c.arg(format!("--user-data-dir={}", dir.display()))
.arg(format!("--remote-debugging-port={launch_port}"))
.arg("--remote-allow-origins=http://127.0.0.1")
.arg("--no-first-run")
.arg("--no-default-browser-check")
.arg(url)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null());
#[cfg(unix)]
{
use std::os::unix::process::CommandExt;
c.process_group(0);
}
c.spawn().map_err(|e| format!("打不開 {name}:{e}"))?;
let start = Instant::now();
while start.elapsed() < Duration::from_secs(START_TIMEOUT_SECS) {
if let Some(p) = live_port(profile) {
return Ok((name, p, true));
}
std::thread::sleep(Duration::from_millis(250));
}
Err(format!("{name} 沒有及時打開(AnyBot 瀏覽器,設定檔「{profile}」)。請再試一次。"))
}
/// Open a link in a new tab of the bot's browser (in the named profile, which becomes current).
pub fn open(url: &str, profile: Option<&str>) -> Result<String, String> {
let prof = resolve(profile)?;
let _ = set_current(&prof);
let (name, port, fresh) = ensure(&prof, Some(url))?;
if !fresh {
let r = local().put(&endpoint(port, &format!("/json/new?{}", encode_component(url)))).send_empty().map_err(|e| e.to_string())?;
if r.status().as_u16() >= 400 {
return Err(format!("{name} 沒辦法開新分頁({})", r.status()));
}
// The new tab is this job's (its next steps work there, not on another job's tab).
let made: Value = r.into_body().read_json().unwrap_or_default();
claim(made["id"].as_str().unwrap_or(""));
close_old_tabs(port);
}
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
// Back when the page has loaded (not after a fixed 3 s: a slow page was still blank then),
// with what it shows — the same as browser_read, one step fewer for every page.
let loaded = crate::browser::wait_loaded(Some(&prof), OPEN_WAIT_MS);
let page = read(PAGE_CHARS_ON_OPEN, Some(&prof)).unwrap_or_default();
let head = format!("已在 AnyBot 瀏覽器({name},設定檔「{prof}」)打開 {url}{}", if loaded { "。" } else { ",還在載入(可以用 browser_wait 等它,再 browser_read)。" });
Ok(if page.is_empty() { head } else { format!("{head}\n\n{page}") })
}
/// Most tabs the bot's browser keeps. Every browser_open is a new tab and nothing closed them:
/// Bot 的電腦 once had 19 Threads pages open and its whole 2 GB in use (estimated).
const MAX_TABS: usize = 6;
/// Close the tabs past MAX_TABS that were used longest ago (Chrome lists the latest first).
fn close_old_tabs(port: u16) {
let Ok(list) = local().get(&endpoint(port, "/json/list")).call().map_err(|e| e.to_string()).and_then(|r| r.into_body().read_json::<Value>().map_err(|e| e.to_string())) else { return };
for id in old_tabs(&list, |id| !others_tab(id)) {
let _ = local().get(&endpoint(port, &format!("/json/close/{id}"))).call();
}
}
/// The page tabs past MAX_TABS in a /json/list answer (latest-used first), among those `ours`
/// says may be closed (never another job's).
fn old_tabs(list: &Value, ours: impl Fn(&str) -> bool) -> Vec<String> {
list.as_array().map(|a| a.iter().filter(|t| t["type"] == "page").filter_map(|t| t["id"].as_str()).filter(|id| ours(id)).skip(MAX_TABS).map(String::from).collect()).unwrap_or_default()
}
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
/// Longest wait for a page browser_open opened (estimated).
const OPEN_WAIT_MS: u64 = 12_000;
/// Text of the page handed back by browser_open (browser_read gives more when asked).
const PAGE_CHARS_ON_OPEN: usize = 12_000;
/// Bring a profile's window up (Settings button): start it, or bring its front tab forward.
pub fn show(profile: &str) -> Result<(), String> {
let (_, port, fresh) = ensure(profile, None)?;
if fresh {
return Ok(());
}
let list: Value = local().get(&endpoint(port, "/json/list")).call().map_err(|e| e.to_string())?.into_body().read_json().map_err(|e| e.to_string())?;
let id = list.as_array().and_then(|a| a.iter().find(|t| t["type"] == "page")).and_then(|t| t["id"].as_str()).map(str::to_string);
match id {
Some(id) => local().get(&endpoint(port, &format!("/json/activate/{id}"))).call().map(|_| ()).map_err(|e| e.to_string()),
None => local().put(&endpoint(port, "/json/new?about:blank")).send_empty().map(|_| ()).map_err(|e| e.to_string()),
}
}
/// The browser_profiles tool: list, add or switch profiles.
pub fn profiles_tool(action: &str, name: &str) -> Result<String, String> {
let list_text = || {
let cur = current();
choices().iter().map(|p| if *p == cur { format!("{p}(使用中)") } else { p.clone() }).collect::<Vec<_>>().join("、")
};
match action.trim() {
"" | "list" => Ok(format!("AnyBot 瀏覽器的設定檔:{}。每個設定檔有自己的登入。「{PERSON_PROFILE}」是使用者平常的瀏覽器:使用者在那裡切到哪個 Chrome 設定檔,就讀哪個。", list_text())),
"add" | "create" => {
let n = add(name)?;
set_current(&n)?;
Ok(format!("已新增並切換到設定檔「{n}」(全新的,還沒登入任何網站)。用 browser_open 打開網站後,請使用者在那個視窗登入一次。現在:{}", list_text()))
}
"use" | "switch" => {
let n = set_current(name)?;
let note = if n == PERSON_PROFILE { "(使用者平常的瀏覽器:請使用者在那裡切到想要的 Chrome 設定檔)" } else { "" };
Ok(format!("已切換到「{n}」{note},之後的 browser_open / browser_read 都用它。現在:{}", list_text()))
}
other => Err(format!("不懂「{other}」,action 請用 list、add 或 use")),
}
}
/// Run JavaScript in the current tab of the bot's browser. Returns (address, result).
pub fn eval_current(profile: Option<&str>, expression: &str) -> Result<(String, Value), String> {
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
let mut tab = crate::browser::Tab::current(profile)?;
let url = tab.eval("location.href")?.as_str().unwrap_or("").to_string();
let v = tab.eval(expression)?;
Ok((url, v))
}
const READ_EXPR: &str = "JSON.stringify({t:document.title,u:location.href,x:document.body?document.body.innerText:''})";
/// Title, address and visible text of the current tab of the bot's browser.
pub fn read(max_chars: usize, profile: Option<&str>) -> Result<String, String> {
let prof = resolve(profile)?;
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
let (name, _, _) = ensure(&prof, None)?;
// One connection to the tab for the text and the numbered elements.
let mut tab = crate::browser::Tab::current(Some(&prof))?;
let v = tab.eval(READ_EXPR)?;
let page: Value = v.as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or(Value::Null);
let text: String = page["x"].as_str().unwrap_or("").lines().map(str::trim).filter(|l| !l.is_empty()).collect::<Vec<_>>().join("\n");
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
let title = page["t"].as_str().unwrap_or("").to_string();
let url = page["u"].as_str().unwrap_or("").to_string();
let mut clipped: String = text.chars().take(max_chars).collect();
if text.chars().count() > max_chars {
clipped.push_str("\n…(後面還有,已截斷)");
}
let login_hint = {
let l = clipped.to_lowercase();
(l.contains("log in") || l.contains("sign in") || clipped.contains("登入")) && clipped.chars().count() < 1500
};
let mut out = format!("瀏覽器:AnyBot 瀏覽器({name},設定檔「{prof}」)\n標題:{title}\n網址:{url}\n\n{clipped}");
// The links, buttons and fields, numbered for browser_click / browser_type / browser_select.
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
if let Ok((_, _, list, _)) = crate::browser::elements(&mut tab) {
if !list.is_empty() {
out.push_str("\n\n可以操作的元素(用 browser_click、browser_type、browser_select 等工具的 ref;連結的網址在 → 後面,不要自己猜網址;看不懂時用 browser_screenshot):\n");
out.push_str(&list.join("\n"));
}
}
if login_hint {
out.push_str("\n\n(這頁看起來要登入。請使用者在 AnyBot 瀏覽器視窗登入一次,之後就會記住。)");
}
// A blank tab says nothing about a site (a bot read one eleven times and called the person
// logged out of Threads without ever opening it).
if url.is_empty() || url == "about:blank" || url.starts_with("chrome://newtab") {
out.push_str("\n\n(這個分頁是空白的,不代表任何網站的狀態。要看哪個網站(例如有沒有登入),就用 browser_open 打開它。)");
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
/// Live check against a running Chrome on ANYBOT_BROWSER_PORT (run by hand with --ignored).
#[test]
#[ignore]
fn reads_a_real_page() {
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
// Opening hands back the loaded page itself (no browser_read needed).
let r = open("https://example.com", None).unwrap();
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
assert!(r.contains("example.com") && r.contains("Example Domain") && r.contains("[1] 連結"), "{r}");
let page = read(2000, None).unwrap();
assert!(page.contains("Example Domain"), "{page}");
println!("{page}");
}
#[test]
fn only_the_tabs_used_longest_ago_close() {
let mut list: Vec<Value> = (0..9).map(|i| serde_json::json!({"type": "page", "id": format!("p{i}")})).collect();
list.insert(2, serde_json::json!({"type": "browser_ui", "id": "ui"}));
list.insert(4, serde_json::json!({"type": "service_worker", "id": "sw"}));
assert_eq!(old_tabs(&Value::Array(list.clone()), |_| true), vec!["p6", "p7", "p8"]);
// Another job's tabs don't count and are never closed.
assert_eq!(old_tabs(&Value::Array(list.clone()), |id| id != "p0" && id != "p7"), vec!["p8"]);
list.truncate(5);
assert!(old_tabs(&Value::Array(list), |_| true).is_empty());
}
#[test]
fn each_job_works_on_its_own_tabs() {
let page = |id: &str| serde_json::json!({"type": "page", "id": id});
// Latest-used first: B's tab is in front, A's further back.
let list = vec![page("tab-b"), page("tab-a"), page("tab-free")];
as_job("job-a", || claim("tab-a"));
as_job("job-b", || claim("tab-b"));
assert_eq!(as_job("job-a", || pick_tab(&list).map(|t| t["id"].clone())), Some("tab-a".into()));
assert_eq!(as_job("job-b", || pick_tab(&list).map(|t| t["id"].clone())), Some("tab-b".into()));
// A new job starts on the latest tab nobody has; another job's tabs are off limits.
assert_eq!(as_job("job-c", || pick_tab(&list).map(|t| t["id"].clone())), Some("tab-free".into()));
assert!(as_job("job-c", || others_tab("tab-a") && !others_tab("tab-free")));
assert!(as_job("job-a", || !others_tab("tab-a")));
// A claimed tab stays its first owner's.
as_job("job-b", || claim("tab-a"));
assert_eq!(owner("tab-a").as_deref(), Some("job-a"));
// Outside a job nothing is claimed.
claim("tab-free");
assert_eq!(owner("tab-free"), None);
assert_eq!(release("job-a"), vec!["tab-a".to_string()]);
assert_eq!(release("job-b"), vec!["tab-b".to_string()]);
assert_eq!(owner("tab-a"), None);
}
#[test]
fn urls_are_encoded_for_new_tabs() {
assert_eq!(encode_component("https://www.threads.net/@me?x=1&y=2"), "https://www.threads.net/%40me%3Fx%3D1%26y%3D2");
}
#[test]
fn profiles_are_found_by_name() {
let list = vec![DEFAULT_PROFILE.to_string(), "工作".to_string(), "Personal".to_string()];
assert_eq!(match_profile(&list, " personal ").as_deref(), Some("Personal"));
assert_eq!(match_profile(&list, "工作").as_deref(), Some("工作"));
assert_eq!(match_profile(&list, "私人"), None);
assert_eq!(match_profile(&list, ""), None);
let with_person = vec![DEFAULT_PROFILE.to_string(), PERSON_PROFILE.to_string()];
assert_eq!(match_profile(&with_person, "我的 Chrome").as_deref(), Some(PERSON_PROFILE));
assert_eq!(match_profile(&list, "我的 Chrome"), None);
assert!(check_name("我的 Chrome").is_err());
assert!(check_name(PERSON_PROFILE).is_err());
}
#[test]
fn profile_names_are_checked() {
assert_eq!(check_name(" 工作 ").unwrap(), "工作");
assert!(check_name("").is_err());
assert!(check_name("a/b").is_err());
assert!(check_name("..").is_err());
assert!(check_name(&"字".repeat(PROFILE_NAME_MAX_CHARS + 1)).is_err());
assert!(check_name(&"字".repeat(PROFILE_NAME_MAX_CHARS)).is_ok());
}
#[test]
fn chosen_port_is_read_back() {
assert_eq!(parse_active_port("53211\n/devtools/browser/abc\n"), Some(53211));
assert_eq!(parse_active_port("0\n"), None);
assert_eq!(parse_active_port(""), None);
}
}