AnyBot/src/botbrowser.rs

1021 lines
48 KiB
Rust
Raw Normal View History

// AnyBot's own browser window, like Grok Bot's own browser: a separate Chrome-family profile that
// AnyBot starts and talks to directly (Chrome's DevTools protocol on a local port), so reading
// and opening pages needs no browser setting ("允許來自 Apple 事件的 JavaScript") and no macOS
// "control this app" prompt. The person logs in to sites in this window once; the profile keeps it.
//
// Profiles (設定檔): the person's own Chrome profiles can't be used (macOS refuses other apps
// Chrome's data folder, and Chrome refuses remote control of its normal folder), so AnyBot's
// browser keeps its own: 「預設」 plus any the person or the bot adds (工作, 私人…). Each is its own
// browser folder and window with its own logins; switching picks which one the bot uses.
//
// Firefox works too (through WebDriver BiDi, bidi.rs): it runs AnyBot's own profiles on a
// computer without a Chrome-family browser, and the person's own Firefox profiles are choices
// as well (「阿金(Firefox)」, with their logins). Firefox can't be taken over once it's open, so
// AnyBot opens such a profile itself (the person closes its everyday window first).
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
use serde_json::Value;
use std::path::PathBuf;
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
/// Local port for talking to the bot's browser (estimated; any free local port works).
pub const PORT: u16 = 9333;
/// How long to wait for the browser to come up (estimated).
const START_TIMEOUT_SECS: u64 = 20;
/// The first profile: the browser folder made before profiles existed (its logins stay).
pub const DEFAULT_PROFILE: &str = "預設";
/// The person's own browser (read through accessibility, like Grok Bot's Computer Use helper):
/// whichever Chrome profile they have open there is the one read.
pub const PERSON_PROFILE: &str = "你的瀏覽器";
/// Other ways people name their own browser.
const PERSON_ALIASES: [&str; 6] = ["你的 chrome", "我的 chrome", "我的瀏覽器", "你的chrome", "我的chrome", "my chrome"];
/// Longest profile name (estimated).
pub const PROFILE_NAME_MAX_CHARS: usize = 30;
/// File Chrome writes into its folder with the port it chose (`--remote-debugging-port=0`).
const ACTIVE_PORT_FILE: &str = "DevToolsActivePort";
/// Chrome-family browsers that can be started as AnyBot's own window.
#[cfg(target_os = "macos")]
const CANDIDATES: &[(&str, &str)] = &[
("Google Chrome", "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"),
("Brave Browser", "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"),
("Microsoft Edge", "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"),
("Chromium", "/Applications/Chromium.app/Contents/MacOS/Chromium"),
];
/// Linux: the usual install places (distribution packages, the vendors' .deb/.rpm, Snap, Flatpak).
#[cfg(not(target_os = "macos"))]
const CANDIDATES: &[(&str, &str)] = &[
("Google Chrome", "/usr/bin/google-chrome-stable"),
("Google Chrome", "/usr/bin/google-chrome"),
("Google Chrome", "/opt/google/chrome/chrome"),
("Brave Browser", "/usr/bin/brave-browser"),
("Brave Browser", "/opt/brave.com/brave/brave"),
("Microsoft Edge", "/usr/bin/microsoft-edge"),
("Chromium", "/usr/bin/chromium"),
("Chromium", "/usr/bin/chromium-browser"),
("Chromium", "/snap/bin/chromium"),
("Chromium", "/var/lib/flatpak/exports/bin/org.chromium.Chromium"),
];
pub fn engine() -> Option<(&'static str, &'static str)> {
CANDIDATES.iter().copied().find(|(_, p)| std::path::Path::new(p).exists())
}
/// Firefox: the usual install places (Snap first: Ubuntu's /usr/bin/firefox only hands over to it).
#[cfg(target_os = "macos")]
const FIREFOX: &[&str] = &["/Applications/Firefox.app/Contents/MacOS/firefox", "/Applications/Firefox Developer Edition.app/Contents/MacOS/firefox", "/Applications/Firefox Nightly.app/Contents/MacOS/firefox"];
#[cfg(not(target_os = "macos"))]
const FIREFOX: &[&str] = &["/snap/bin/firefox", "/usr/bin/firefox", "/usr/lib/firefox/firefox", "/opt/firefox/firefox", "/var/lib/flatpak/exports/bin/org.mozilla.firefox"];
pub fn firefox() -> Option<&'static str> {
FIREFOX.iter().copied().find(|p| std::path::Path::new(p).exists())
}
/// Which protocol a profile's browser speaks.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Kind {
/// Chrome, Brave, Edge, Chromium (and the box's Chromium): the DevTools protocol.
Chrome,
/// Firefox: WebDriver BiDi.
Firefox,
}
/// The browser that runs AnyBot's own profiles: a Chrome-family one when there is one, else
/// Firefox. (ANYBOT_BROWSER_ENGINE=firefox picks Firefox anyway, for testing.)
fn own_kind() -> Option<Kind> {
let want_firefox = std::env::var("ANYBOT_BROWSER_ENGINE").map(|e| e.eq_ignore_ascii_case("firefox")).unwrap_or(false);
match (engine().is_some(), firefox().is_some()) {
(_, true) if want_firefox => Some(Kind::Firefox),
(true, _) => Some(Kind::Chrome),
(false, true) => Some(Kind::Firefox),
(false, false) => None,
}
}
pub fn available() -> bool {
if using_box() {
return true;
}
own_kind().is_some()
}
/// How the person's own Firefox profiles are listed: 「阿金(Firefox)」.
pub const FIREFOX_MARK: &str = "(Firefox)";
/// Where Firefox keeps its profiles on this computer (packaged as Snap or Flatpak too).
fn firefox_roots() -> Vec<PathBuf> {
let Some(home) = dirs::home_dir() else { return vec![] };
if cfg!(target_os = "macos") {
vec![home.join("Library/Application Support/Firefox")]
} else {
vec![home.join(".mozilla/firefox"), home.join(".config/mozilla/firefox"), home.join("snap/firefox/common/.mozilla/firefox"), home.join(".var/app/org.mozilla.firefox/.mozilla/firefox")]
}
}
/// profiles.ini's profiles: (name, folder).
fn ini_profiles(root: &std::path::Path, ini: &str) -> Vec<(String, PathBuf)> {
let mut out = vec![];
let (mut name, mut path, mut relative, mut in_profile) = (String::new(), String::new(), true, false);
let mut flush = |name: &mut String, path: &mut String, relative: bool| {
if !name.is_empty() && !path.is_empty() {
out.push((std::mem::take(name), if relative { root.join(&*path) } else { PathBuf::from(&*path) }));
}
name.clear();
path.clear();
};
for line in ini.lines().map(str::trim) {
if line.starts_with('[') {
flush(&mut name, &mut path, relative);
in_profile = line.starts_with("[Profile");
relative = true;
} else if in_profile {
match line.split_once('=') {
Some(("Name", v)) => name = v.to_string(),
Some(("Path", v)) => path = v.to_string(),
Some(("IsRelative", v)) => relative = v != "0",
_ => {}
}
}
}
flush(&mut name, &mut path, relative);
out
}
/// The newer profile switcher's profiles (Firefox 138+): their names live only in
/// 「Profile Groups/<id>.sqlite」. Read from a copy (Firefox keeps the file open and busy).
fn group_profiles(root: &std::path::Path) -> Vec<(String, PathBuf)> {
let Ok(files) = std::fs::read_dir(root.join("Profile Groups")) else { return vec![] };
let mut out = vec![];
for db in files.flatten().map(|e| e.path()).filter(|p| p.extension().and_then(|e| e.to_str()) == Some("sqlite")) {
let tmp = std::env::temp_dir().join(format!("anybot-ff-{}-{}", std::process::id(), crate::httpd::random_hex(4)));
if std::fs::create_dir_all(&tmp).is_err() {
continue;
}
let copy = tmp.join("groups.sqlite");
let read = || -> rusqlite::Result<Vec<(String, String)>> {
let c = rusqlite::Connection::open(&copy)?;
let mut q = c.prepare("SELECT path, name FROM Profiles")?;
let rows = q.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))?;
rows.collect()
};
let copied = std::fs::copy(&db, &copy).is_ok();
// Recent changes may still be in the write-ahead log beside it.
let _ = std::fs::copy(db.with_extension("sqlite-wal"), tmp.join("groups.sqlite-wal"));
for (path, name) in if copied { read().unwrap_or_default() } else { vec![] } {
let p = PathBuf::from(&path);
out.push((name, if p.is_absolute() { p } else { root.join(p) }));
}
let _ = std::fs::remove_dir_all(&tmp);
}
out
}
/// The person's own Firefox profiles (name, folder), as Firefox lists them. Looked up at most
/// every few seconds (the tool list asks for it at every step).
pub fn firefox_profiles() -> Vec<(String, PathBuf)> {
type Found = Vec<(String, PathBuf)>;
static CACHE: std::sync::Mutex<Option<(Instant, Found)>> = std::sync::Mutex::new(None);
let mut c = CACHE.lock().unwrap_or_else(|e| e.into_inner());
if let Some((at, list)) = c.as_ref() {
if at.elapsed() < Duration::from_secs(10) {
return list.clone();
}
}
let mut list: Vec<(String, PathBuf)> = vec![];
for root in firefox_roots().iter().filter(|r| r.is_dir()) {
let ini = std::fs::read_to_string(root.join("profiles.ini")).unwrap_or_default();
// A name from the profile switcher wins over profiles.ini's (often just "default").
for (name, dir) in group_profiles(root).into_iter().chain(ini_profiles(root, &ini)) {
if !dir.is_dir() || list.iter().any(|(_, d)| *d == dir) {
continue;
}
let name = name.trim().to_string();
let mut unique = name.clone();
let mut n = 2;
while list.iter().any(|(m, _)| *m == unique) {
unique = format!("{name} {n}");
n += 1;
}
list.push((unique, dir));
}
}
*c = Some((Instant::now(), list.clone()));
list
}
/// A Snap or Flatpak Firefox only reaches its own corner of the home folder: AnyBot's own
/// Firefox profiles live there.
fn firefox_own_root() -> PathBuf {
let bin = firefox().unwrap_or("");
let home = dirs::home_dir().unwrap_or_default();
if bin.starts_with("/snap/") || (bin == "/usr/bin/firefox" && std::path::Path::new("/snap/bin/firefox").exists()) {
home.join("snap/firefox/common/AnyBot")
} else if bin.contains("flatpak") {
home.join(".var/app/org.mozilla.firefox/AnyBot")
} else {
data_dir().join("firefox-profiles")
}
}
/// Where a profile runs: its browser and its folder.
fn place(name: &str) -> Option<(Kind, PathBuf)> {
if let Some(own) = name.strip_suffix(FIREFOX_MARK) {
return firefox_profiles().into_iter().find(|(n, _)| n == own).map(|(_, d)| (Kind::Firefox, d));
}
match own_kind()? {
Kind::Firefox => Some((Kind::Firefox, firefox_own_root().join(name))),
Kind::Chrome => Some((Kind::Chrome, dir_for(name))),
}
}
/// The protocol of a profile's browser (the box's Chromium is Chrome).
pub fn kind_of(profile: Option<&str>) -> Kind {
if using_box() {
return Kind::Chrome;
}
resolve(profile).ok().and_then(|n| place(&n)).map(|(k, _)| k).unwrap_or(Kind::Chrome)
}
/// Is this Firefox profile open (by anyone)? Firefox holds a lock in it while it runs.
fn firefox_in_use(dir: &std::path::Path) -> bool {
#[cfg(target_os = "linux")]
{
// "lock" → "host:+pid" while it runs (left behind after a crash: then the pid is gone).
std::fs::read_link(dir.join("lock")).ok().and_then(|t| t.to_string_lossy().rsplit('+').next().and_then(|p| p.parse::<u32>().ok())).is_some_and(|pid| std::path::Path::new(&format!("/proc/{pid}")).exists())
}
#[cfg(target_os = "macos")]
{
Command::new("lsof").arg("-t").arg(dir.join(".parentlock")).output().map(|o| !o.stdout.is_empty()).unwrap_or(false)
}
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
{
let _ = dir;
false
}
}
/// The port AnyBot can drive this Firefox profile on (it was started for AnyBot), if it's open.
fn firefox_port(dir: &std::path::Path) -> Option<u16> {
let v: Value = serde_json::from_str(&std::fs::read_to_string(dir.join(FIREFOX_PORT_FILE)).ok()?).ok()?;
let port = u16::try_from(v["ws_port"].as_u64()?).ok().filter(|p| *p != 0)?;
// The file outlives a crash: only a running profile with that port open counts.
let addr = std::net::SocketAddr::from(([127, 0, 0, 1], port));
(firefox_in_use(dir) && std::net::TcpStream::connect_timeout(&addr, Duration::from_millis(500)).is_ok()).then_some(port)
}
/// File Firefox writes into its profile with the BiDi port it chose (`--remote-debugging-port 0`).
const FIREFOX_PORT_FILE: &str = "WebDriverBiDiServer.json";
fn data_dir() -> PathBuf {
crate::model::State::path().parent().map(PathBuf::from).unwrap_or_else(std::env::temp_dir)
}
/// The 「預設」 profile lives with AnyBot's data (logins stay between runs).
pub fn profile_dir() -> PathBuf {
data_dir().join("browser")
}
fn profiles_root() -> PathBuf {
data_dir().join("browser-profiles")
}
fn dir_for(name: &str) -> PathBuf {
if name == DEFAULT_PROFILE { profile_dir() } else { profiles_root().join(name) }
}
/// All profiles: 「預設」 first, then the added ones by name.
pub fn profiles() -> Vec<String> {
let mut extra: Vec<String> = std::fs::read_dir(profiles_root())
.map(|it| it.flatten().filter(|e| e.path().is_dir()).filter_map(|e| e.file_name().to_str().map(str::to_string)).collect())
.unwrap_or_default();
extra.sort();
let mut all = vec![DEFAULT_PROFILE.to_string()];
all.extend(extra.into_iter().filter(|n| n != DEFAULT_PROFILE));
all
}
fn current_file() -> PathBuf {
data_dir().join("browser-profile.txt")
}
/// Every choice: AnyBot's own profiles, the person's own Firefox profiles, then the person's
/// own browser.
pub fn choices() -> Vec<String> {
let mut all = profiles();
if firefox().is_some() {
all.extend(firefox_profiles().into_iter().map(|(n, _)| format!("{n}{FIREFOX_MARK}")));
}
// Reading the person's own browser goes through macOS accessibility; not on Linux.
if cfg!(target_os = "macos") {
all.push(PERSON_PROFILE.to_string());
}
all
}
/// The profile the bot uses when it isn't told which.
pub fn current() -> String {
let want = std::fs::read_to_string(current_file()).unwrap_or_default().trim().to_string();
match_profile(&choices(), &want).unwrap_or_else(|| DEFAULT_PROFILE.to_string())
}
pub fn set_current(name: &str) -> Result<String, String> {
let name = resolve(Some(name))?;
std::fs::write(current_file(), &name).map_err(|e| format!("存不了設定檔選擇:{e}"))?;
Ok(name)
}
/// Find a profile by name, ignoring case and spaces at the ends.
pub fn match_profile(list: &[String], want: &str) -> Option<String> {
let w = want.trim().to_lowercase();
if w.is_empty() {
return None;
}
if PERSON_ALIASES.contains(&w.as_str()) && list.iter().any(|n| n == PERSON_PROFILE) {
return Some(PERSON_PROFILE.to_string());
}
if let Some(n) = list.iter().find(|n| n.to_lowercase() == w) {
return Some(n.clone());
}
// A Firefox profile by its own name (「阿金」, 「Firefox 阿金」 for 「阿金(Firefox)」), when
// only one has it.
let bare = |s: &str| s.to_lowercase().replace(&FIREFOX_MARK.to_lowercase(), "").replace("(firefox)", "").replace("firefox", "").replace([':', ':'], "").trim().to_string();
let b = bare(&w);
let hits: Vec<&String> = list.iter().filter(|n| n.ends_with(FIREFOX_MARK) && !b.is_empty() && bare(n) == b).collect();
(hits.len() == 1).then(|| hits[0].clone())
}
/// Which profile a tool call means: the one named, or the current one.
pub fn resolve(requested: Option<&str>) -> Result<String, String> {
if using_box() {
return Ok(DEFAULT_PROFILE.to_string());
}
match requested.map(str::trim).filter(|s| !s.is_empty()) {
None => Ok(current()),
Some(r) => {
let all = choices();
match_profile(&all, r).ok_or_else(|| format!("AnyBot 瀏覽器沒有叫「{r}」的設定檔。現有:{}。要新的可以用 browser_profiles 新增(action: add)。", all.join("、")))
}
}
}
/// A usable profile name: not empty, not too long, no slashes or dots-only names.
pub fn check_name(name: &str) -> Result<String, String> {
let n = name.trim();
if n.is_empty() {
return Err("請給設定檔名稱".into());
}
if n.chars().count() > PROFILE_NAME_MAX_CHARS {
return Err(format!("設定檔名稱最多 {PROFILE_NAME_MAX_CHARS} 個字"));
}
if n == PERSON_PROFILE || PERSON_ALIASES.contains(&n.to_lowercase().as_str()) {
return Err(format!("「{n}」是留給你自己瀏覽器的名字,請換一個"));
}
if n.contains('/') || n.contains(':') || n.starts_with('.') || n.chars().any(char::is_control) {
return Err("設定檔名稱不能有 / 或 : 也不能用 . 開頭".into());
}
if n.to_lowercase().contains("firefox") {
return Err("名字裡有 Firefox 的是你自己的 Firefox 設定檔,請換一個名字".into());
}
Ok(n.to_string())
}
/// Make a new empty profile (its own folder, so its own logins).
pub fn add(name: &str) -> Result<String, String> {
let n = check_name(name)?;
if let Some(have) = match_profile(&profiles(), &n) {
return Ok(have);
}
std::fs::create_dir_all(profiles_root().join(&n)).map_err(|e| format!("建不了設定檔:{e}"))?;
Ok(n)
}
/// Remove an added profile and its logins (the person's choice, from Settings). 「預設」 and the
/// person's own browser can't be removed, and an open window must be closed first.
pub fn remove(name: &str) -> Result<(), String> {
let all = profiles();
let n = match_profile(&all, name).ok_or_else(|| format!("沒有叫「{name}」的設定檔"))?;
if n == DEFAULT_PROFILE || n == PERSON_PROFILE {
return Err(format!("「{n}」不能移除"));
}
if n.ends_with(FIREFOX_MARK) {
return Err(format!("「{n}」是你自己的 Firefox 設定檔,AnyBot 不會移除它(要刪請在 Firefox 裡刪)"));
}
if live_port(&n).is_some() {
return Err(format!("「{n}」的視窗還開著,請先關掉它再移除。"));
}
let dir = profiles_root().join(&n);
// Only ever a folder directly inside AnyBot's browser-profiles folder.
if dir.parent() != Some(profiles_root().as_path()) {
return Err("設定檔位置不對,沒有移除".into());
}
std::fs::remove_dir_all(&dir).map_err(|e| format!("移除不了「{n}」:{e}"))?;
if current() == n || !choices().contains(&current()) {
let _ = std::fs::write(current_file(), DEFAULT_PROFILE);
}
Ok(())
}
pub(crate) fn local() -> ureq::Agent {
ureq::Agent::config_builder().timeout_global(Some(Duration::from_secs(5))).http_status_as_error(false).build().into()
}
/// The fixed port of 「預設」 (`ANYBOT_BROWSER_PORT` overrides it, for testing).
fn default_port() -> u16 {
std::env::var("ANYBOT_BROWSER_PORT").ok().and_then(|p| p.parse().ok()).unwrap_or(PORT)
}
fn endpoint(port: u16, path: &str) -> String {
format!("http://127.0.0.1:{port}{path}")
}
fn alive(port: u16) -> bool {
port != 0 && local().get(&endpoint(port, "/json/version")).call().map(|r| r.status().as_u16() == 200).unwrap_or(false)
}
/// First line of Chrome's DevToolsActivePort file.
pub fn parse_active_port(text: &str) -> Option<u16> {
text.lines().next()?.trim().parse().ok().filter(|p| *p != 0)
}
/// The port a running window of this profile answers on, if any.
fn live_port(name: &str) -> Option<u16> {
if let Some((Kind::Firefox, dir)) = place(name) {
return firefox_port(&dir);
}
let file = std::fs::read_to_string(dir_for(name).join(ACTIVE_PORT_FILE)).ok().and_then(|t| parse_active_port(&t));
let fixed = (name == DEFAULT_PROFILE).then(default_port);
[fixed, file].into_iter().flatten().find(|p| alive(*p))
}
/// Percent-encode a URL so it can ride in `/json/new?<url>`.
pub fn encode_component(s: &str) -> String {
s.bytes()
.map(|b| match b {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' | b':' | b'/' => (b as char).to_string(),
_ => format!("%{b:02X}"),
})
.collect()
}
thread_local! {
/// Browser steps on this thread go to Bot 的電腦's Chromium (its box), not this computer's.
static IN_BOX: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
}
/// Run `f` with the browser steps on Bot 的電腦's own Chromium.
pub fn in_box<T>(f: impl FnOnce() -> T) -> T {
IN_BOX.with(|b| b.set(true));
let r = f();
IN_BOX.with(|b| b.set(false));
r
}
pub fn using_box() -> bool {
IN_BOX.with(|b| b.get())
}
/// The debugging port of a bot browser that is already open (None when it isn't; never starts one).
pub fn running_port(profile: Option<&str>) -> Option<u16> {
if using_box() {
return crate::cloudbox::browser_running().then(crate::cloudbox::browser_port).and_then(Result::ok);
}
live_port(&resolve(profile).ok()?)
}
// ---------- each job its own tabs ----------
// Bots share a browser (the box's Chromium has one profile for all of them). Every step used to
// act on the tab in front, so two jobs at once read and clicked on each other's pages, and one
// finishing closed the other's. A job now owns the tabs it opens or lands on (a popup it clicked
// open too): its steps use the latest of those, and another job's tabs are never used, listed
// or closed by it.
thread_local! {
/// The job whose browser step runs on this thread (empty: not a bot's job).
static JOB: std::cell::RefCell<String> = const { std::cell::RefCell::new(String::new()) };
}
/// (tab id, job) for the tabs running jobs own.
static OWNERS: std::sync::Mutex<Vec<(String, String)>> = std::sync::Mutex::new(Vec::new());
/// Run `f` as a step of job `job` (on its own tabs).
pub fn as_job<T>(job: &str, f: impl FnOnce() -> T) -> T {
let before = JOB.with(|j| std::mem::replace(&mut *j.borrow_mut(), job.to_string()));
let r = f();
JOB.with(|j| *j.borrow_mut() = before);
r
}
pub fn job() -> String {
JOB.with(|j| j.borrow().clone())
}
fn owner(tab: &str) -> Option<String> {
OWNERS.lock().unwrap_or_else(|e| e.into_inner()).iter().find(|(t, _)| t == tab).map(|(_, j)| j.clone())
}
/// Another running job owns this tab.
pub fn others_tab(tab: &str) -> bool {
owner(tab).is_some_and(|j| j != job())
}
/// This step's job takes the tab, when it is nobody's.
pub fn claim(tab: &str) {
let me = job();
if me.is_empty() || tab.is_empty() {
return;
}
let mut o = OWNERS.lock().unwrap_or_else(|e| e.into_inner());
if !o.iter().any(|(t, _)| t == tab) {
o.push((tab.to_string(), me));
}
}
/// The job is over: its tabs are nobody's again (returned).
pub fn release(job: &str) -> Vec<String> {
let mut o = OWNERS.lock().unwrap_or_else(|e| e.into_inner());
let mine = o.iter().filter(|(_, j)| j == job).map(|(t, _)| t.clone()).collect();
o.retain(|(_, j)| j != job);
mine
}
/// The tab of a page list (latest-used first) this step works on: its job's latest own tab, else
/// the latest one no other job owns (a job's first step).
pub fn pick_tab(list: &[Value]) -> Option<&Value> {
let me = job();
let id = |t: &Value| t["id"].as_str().unwrap_or("").to_string();
if !me.is_empty() {
if let Some(t) = list.iter().find(|t| owner(&id(t)).as_deref() == Some(me.as_str())) {
return Some(t);
}
}
list.iter().find(|t| !others_tab(&id(t)))
}
/// The open page tabs' ids (none when the browser isn't open; it isn't started for this).
pub fn tab_ids(profile: Option<&str>) -> Vec<String> {
let Some(port) = running_port(profile) else { return vec![] };
pages_of(kind_of(profile), port).unwrap_or_default().iter().filter_map(|t| t["id"].as_str().map(String::from)).collect()
}
// ---------- tabs, whichever the browser ----------
/// The open page tabs ({id, url, title}), the one in use first.
pub fn pages_of(kind: Kind, port: u16) -> Result<Vec<Value>, String> {
match kind {
Kind::Firefox => crate::bidi::pages(port),
Kind::Chrome => {
let v: Value = local().get(&endpoint(port, "/json/list")).call().map_err(|e| e.to_string())?.into_body().read_json().map_err(|e| e.to_string())?;
Ok(crate::browser::pages(&v))
}
}
}
/// Open a tab at `url`. Returns its id.
pub fn new_tab(kind: Kind, port: u16, url: &str) -> Result<String, String> {
match kind {
Kind::Firefox => crate::bidi::new_tab(port, url),
Kind::Chrome => {
let r = local().put(&endpoint(port, &format!("/json/new?{}", encode_component(url)))).send_empty().map_err(|e| e.to_string())?;
if r.status().as_u16() >= 400 {
return Err(format!("瀏覽器沒辦法開新分頁({})", r.status()));
}
let made: Value = r.into_body().read_json().unwrap_or_default();
Ok(made["id"].as_str().unwrap_or("").to_string())
}
}
}
pub fn activate_tab(kind: Kind, port: u16, id: &str) -> Result<(), String> {
match kind {
Kind::Firefox => crate::bidi::activate(port, id),
Kind::Chrome => local().get(&endpoint(port, &format!("/json/activate/{id}"))).call().map(|_| ()).map_err(|e| e.to_string()),
}
}
pub fn close_tab(kind: Kind, port: u16, id: &str) -> Result<(), String> {
match kind {
Kind::Firefox => crate::bidi::close(port, id),
Kind::Chrome => local().get(&endpoint(port, &format!("/json/close/{id}"))).call().map(|_| ()).map_err(|e| e.to_string()),
}
}
/// Tabs that appeared during this step (a popup a click opened) are its job's.
pub fn adopt_new(profile: Option<&str>, before: &[String]) {
if job().is_empty() {
return;
}
for id in tab_ids(profile) {
if !before.contains(&id) {
claim(&id);
}
}
}
/// Start this profile's window if it isn't running. Returns (browser name, port, just started).
pub fn ensure(profile: &str, first_url: Option<&str>) -> Result<(&'static str, u16, bool), String> {
if using_box() {
let fresh = !crate::cloudbox::browser_running();
let port = crate::cloudbox::browser_port()?;
if fresh {
if let Some(u) = first_url {
let _ = local().put(&endpoint(port, &format!("/json/new?{}", encode_component(u)))).send_empty();
}
}
return Ok(("它桌面上的 Chromium", port, fresh && first_url.is_some()));
}
if profile == PERSON_PROFILE {
return Err("「你的瀏覽器」是你自己的視窗,AnyBot 不會另外開。".into());
}
if let Some((Kind::Firefox, dir)) = place(profile) {
return ensure_firefox(profile, &dir, first_url);
}
let Some((name, bin)) = engine() else { return Err("這台電腦沒有 Chrome、Brave、Edge、Chromium 或 Firefox,沒辦法開 AnyBot 自己的瀏覽器。".into()) };
if let Some(p) = live_port(profile) {
return Ok((name, p, false));
}
let dir = dir_for(profile);
let _ = std::fs::create_dir_all(&dir);
let _ = std::fs::remove_file(dir.join(ACTIVE_PORT_FILE));
// 「預設」 keeps its fixed port; other profiles let Chrome pick a free one and read it back.
let launch_port = if profile == DEFAULT_PROFILE { default_port() } else { 0 };
// Defense in depth: this becomes a Chrome launch argument. Every caller normalizes to a
// full URL first; a bare "-…" (a Chrome flag) must never get here from a tool argument.
let url = match first_url {
Some(u) if u.starts_with("http://") || u.starts_with("https://") => u,
Some(_) => return Err("第一個網址必須是 http(s) 網址".into()),
None => "about:blank",
};
let mut c = Command::new(bin);
c.arg(format!("--user-data-dir={}", dir.display()))
.arg(format!("--remote-debugging-port={launch_port}"))
.arg("--remote-allow-origins=http://127.0.0.1")
.arg("--no-first-run")
.arg("--no-default-browser-check")
.arg(url)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null());
#[cfg(unix)]
{
use std::os::unix::process::CommandExt;
c.process_group(0);
}
c.spawn().map_err(|e| format!("打不開 {name}:{e}"))?;
let start = Instant::now();
while start.elapsed() < Duration::from_secs(START_TIMEOUT_SECS) {
if let Some(p) = live_port(profile) {
return Ok((name, p, true));
}
std::thread::sleep(Duration::from_millis(250));
}
Err(format!("{name} 沒有及時打開(AnyBot 瀏覽器,設定檔「{profile}」)。請再試一次。"))
}
/// Start a Firefox profile so AnyBot can drive it (WebDriver BiDi on a port Firefox picks).
fn ensure_firefox(profile: &str, dir: &std::path::Path, first_url: Option<&str>) -> Result<(&'static str, u16, bool), String> {
let bin = firefox().ok_or("這台電腦沒有 Firefox")?;
if let Some(p) = firefox_port(dir) {
return Ok(("Firefox", p, false));
}
// Open the everyday way: Firefox can't be taken over then, and a second copy of the same
// profile won't start.
if firefox_in_use(dir) {
return Err(format!("「{profile}」的 Firefox 視窗已經開著,但不是 AnyBot 打開的,AnyBot 沒辦法操作它。請使用者先關掉這個設定檔的 Firefox 視窗(登入都會留著),再讓 AnyBot 打開它;之後要用這個設定檔時,從 AnyBot 打開(設定 › 瀏覽器設定檔 › 開啟視窗)。"));
}
let _ = std::fs::create_dir_all(dir);
let _ = std::fs::remove_file(dir.join(FIREFOX_PORT_FILE));
// Defense in depth, as for Chrome: never a bare "-…" (a Firefox flag) from a tool argument.
let url = match first_url {
Some(u) if u.starts_with("http://") || u.starts_with("https://") => u,
Some(_) => return Err("第一個網址必須是 http(s) 網址".into()),
None => "about:blank",
};
let mut c = Command::new(bin);
c.arg("--new-instance").arg("--profile").arg(dir).arg("--remote-debugging-port").arg("0");
if std::env::var_os("ANYBOT_FIREFOX_HEADLESS").is_some() {
c.arg("--headless");
}
c.arg(url).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null());
#[cfg(unix)]
{
use std::os::unix::process::CommandExt;
c.process_group(0);
}
c.spawn().map_err(|e| format!("打不開 Firefox:{e}"))?;
let start = Instant::now();
while start.elapsed() < Duration::from_secs(START_TIMEOUT_SECS) {
if let Some(p) = firefox_port(dir) {
return Ok(("Firefox", p, true));
}
std::thread::sleep(Duration::from_millis(250));
}
Err(format!("Firefox 沒有及時打開(設定檔「{profile}」)。請再試一次。"))
}
/// Open a link in a new tab of the bot's browser (in the named profile, which becomes current).
pub fn open(url: &str, profile: Option<&str>) -> Result<String, String> {
let prof = resolve(profile)?;
let _ = set_current(&prof);
let (name, port, fresh) = ensure(&prof, Some(url))?;
if !fresh {
let kind = kind_of(Some(&prof));
let id = new_tab(kind, port, url).map_err(|e| format!("{name} 沒辦法開新分頁:{e}"))?;
// The new tab is this job's (its next steps work there, not on another job's tab).
claim(&id);
close_old_tabs(kind, port);
}
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
// Back when the page has loaded (not after a fixed 3 s: a slow page was still blank then),
// with what it shows — the same as browser_read, one step fewer for every page.
let loaded = crate::browser::wait_loaded(Some(&prof), OPEN_WAIT_MS);
let page = read(PAGE_CHARS_ON_OPEN, Some(&prof)).unwrap_or_default();
let head = format!("已在 AnyBot 瀏覽器({name},設定檔「{prof}」)打開 {url}{}", if loaded { "。" } else { ",還在載入(可以用 browser_wait 等它,再 browser_read)。" });
Ok(if page.is_empty() { head } else { format!("{head}\n\n{page}") })
}
/// Most tabs the bot's browser keeps. Every browser_open is a new tab and nothing closed them:
/// Bot 的電腦 once had 19 Threads pages open and its whole 2 GB in use (estimated).
const MAX_TABS: usize = 6;
/// Close the tabs past MAX_TABS that were used longest ago (the list has the latest first).
fn close_old_tabs(kind: Kind, port: u16) {
let Ok(list) = pages_of(kind, port) else { return };
for id in old_tabs(&Value::Array(list), |id| !others_tab(id)) {
let _ = close_tab(kind, port, &id);
}
}
/// The page tabs past MAX_TABS in a /json/list answer (latest-used first), among those `ours`
/// says may be closed (never another job's).
fn old_tabs(list: &Value, ours: impl Fn(&str) -> bool) -> Vec<String> {
list.as_array().map(|a| a.iter().filter(|t| t["type"] == "page").filter_map(|t| t["id"].as_str()).filter(|id| ours(id)).skip(MAX_TABS).map(String::from).collect()).unwrap_or_default()
}
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
/// Longest wait for a page browser_open opened (estimated).
const OPEN_WAIT_MS: u64 = 12_000;
/// Text of the page handed back by browser_open (browser_read gives more when asked).
const PAGE_CHARS_ON_OPEN: usize = 12_000;
/// Bring a profile's window up (Settings button): start it, or bring its front tab forward.
pub fn show(profile: &str) -> Result<(), String> {
let (_, port, fresh) = ensure(profile, None)?;
if fresh {
return Ok(());
}
let kind = kind_of(Some(profile));
let list = pages_of(kind, port)?;
match list.first().and_then(|t| t["id"].as_str()) {
Some(id) => activate_tab(kind, port, id),
None => new_tab(kind, port, "about:blank").map(|_| ()),
}
}
/// The browser_profiles tool: list, add or switch profiles.
pub fn profiles_tool(action: &str, name: &str) -> Result<String, String> {
let list_text = || {
let cur = current();
choices().iter().map(|p| if *p == cur { format!("{p}(使用中)") } else { p.clone() }).collect::<Vec<_>>().join("、")
};
match action.trim() {
"" | "list" => Ok(format!(
"AnyBot 瀏覽器的設定檔:{}。每個設定檔有自己的登入。{}{}",
list_text(),
if choices().iter().any(|c| c.ends_with(FIREFOX_MARK)) { format!("名字後面有「{FIREFOX_MARK}」的是使用者自己的 Firefox 設定檔(用使用者的登入),AnyBot 會用可以操作的方式打開它;如果使用者已經用一般方式開著那個設定檔,要先請使用者關掉那個視窗。") } else { String::new() },
if choices().iter().any(|c| c == PERSON_PROFILE) { format!("「{PERSON_PROFILE}」是使用者平常的瀏覽器:使用者在那裡切到哪個 Chrome 設定檔,就讀哪個。") } else { String::new() }
)),
"add" | "create" => {
let n = add(name)?;
set_current(&n)?;
Ok(format!("已新增並切換到設定檔「{n}」(全新的,還沒登入任何網站)。用 browser_open 打開網站後,請使用者在那個視窗登入一次。現在:{}", list_text()))
}
"use" | "switch" => {
let n = set_current(name)?;
let note = if n == PERSON_PROFILE { "(使用者平常的瀏覽器:請使用者在那裡切到想要的 Chrome 設定檔)" } else { "" };
Ok(format!("已切換到「{n}」{note},之後的 browser_open / browser_read 都用它。現在:{}", list_text()))
}
other => Err(format!("不懂「{other}」,action 請用 list、add 或 use")),
}
}
/// Run JavaScript in the current tab of the bot's browser. Returns (address, result).
pub fn eval_current(profile: Option<&str>, expression: &str) -> Result<(String, Value), String> {
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
let mut tab = crate::browser::Tab::current(profile)?;
let url = tab.eval("location.href")?.as_str().unwrap_or("").to_string();
let v = tab.eval(expression)?;
Ok((url, v))
}
const READ_EXPR: &str = "JSON.stringify({t:document.title,u:location.href,x:document.body?document.body.innerText:''})";
/// Title, address and visible text of the current tab of the bot's browser.
pub fn read(max_chars: usize, profile: Option<&str>) -> Result<String, String> {
let prof = resolve(profile)?;
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
let (name, _, _) = ensure(&prof, None)?;
// One connection to the tab for the text and the numbered elements.
let mut tab = crate::browser::Tab::current(Some(&prof))?;
let v = tab.eval(READ_EXPR)?;
let page: Value = v.as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or(Value::Null);
let text: String = page["x"].as_str().unwrap_or("").lines().map(str::trim).filter(|l| !l.is_empty()).collect::<Vec<_>>().join("\n");
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
let title = page["t"].as_str().unwrap_or("").to_string();
let url = page["u"].as_str().unwrap_or("").to_string();
let mut clipped: String = text.chars().take(max_chars).collect();
if text.chars().count() > max_chars {
clipped.push_str("\n…(後面還有,已截斷)");
}
let login_hint = {
let l = clipped.to_lowercase();
(l.contains("log in") || l.contains("sign in") || clipped.contains("登入")) && clipped.chars().count() < 1500
};
let mut out = format!("瀏覽器:AnyBot 瀏覽器({name},設定檔「{prof}」)\n標題:{title}\n網址:{url}\n\n{clipped}");
// The links, buttons and fields, numbered for browser_click / browser_type / browser_select.
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
if let Ok((_, _, list, _)) = crate::browser::elements(&mut tab) {
if !list.is_empty() {
out.push_str("\n\n可以操作的元素(用 browser_click、browser_type、browser_select 等工具的 ref;連結的網址在 → 後面,不要自己猜網址;看不懂時用 browser_screenshot):\n");
out.push_str(&list.join("\n"));
}
}
if login_hint {
out.push_str("\n\n(這頁看起來要登入。請使用者在 AnyBot 瀏覽器視窗登入一次,之後就會記住。)");
}
// A blank tab says nothing about a site (a bot read one eleven times and called the person
// logged out of Threads without ever opening it).
if url.is_empty() || url == "about:blank" || url.starts_with("chrome://newtab") {
out.push_str("\n\n(這個分頁是空白的,不代表任何網站的狀態。要看哪個網站(例如有沒有登入),就用 browser_open 打開它。)");
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
/// Live check against a running Chrome on ANYBOT_BROWSER_PORT (run by hand with --ignored).
#[test]
#[ignore]
fn reads_a_real_page() {
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
// Opening hands back the loaded page itself (no browser_read needed).
let r = open("https://example.com", None).unwrap();
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice Web - web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus. - browser_open waits for the page to load and returns it like browser_read (one step fewer per page); actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab. The bot's computer (box v6) - Numbers only on what can be seen: clipped to the window and the page's visible area, none under another app's window (a link scrolled under the toolbar was numbered on Back). - press_key with a typed character no longer swallows the keys after it (xdotool type --args 1). - The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys). - "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs. - One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone). The person's computers - Linux: the window system's front app when it has one, and the reason when there is none (was 「」); hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist. The loop - Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows): it was a plain shell on the brain itself, keys in reach, asked nothing. - The bot's own computer is re-checked each step; unreachable computers are failures, not safety blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys aren't counted as going in circles; steps aren't moved to a computer that can't do them. - One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the impossible server branches removed; duplicated prompt instructions merged. Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses on its box; remote E2E 28/28. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
assert!(r.contains("example.com") && r.contains("Example Domain") && r.contains("[1] 連結"), "{r}");
let page = read(2000, None).unwrap();
assert!(page.contains("Example Domain"), "{page}");
println!("{page}");
}
#[test]
fn only_the_tabs_used_longest_ago_close() {
let mut list: Vec<Value> = (0..9).map(|i| serde_json::json!({"type": "page", "id": format!("p{i}")})).collect();
list.insert(2, serde_json::json!({"type": "browser_ui", "id": "ui"}));
list.insert(4, serde_json::json!({"type": "service_worker", "id": "sw"}));
assert_eq!(old_tabs(&Value::Array(list.clone()), |_| true), vec!["p6", "p7", "p8"]);
// Another job's tabs don't count and are never closed.
assert_eq!(old_tabs(&Value::Array(list.clone()), |id| id != "p0" && id != "p7"), vec!["p8"]);
list.truncate(5);
assert!(old_tabs(&Value::Array(list), |_| true).is_empty());
}
#[test]
fn each_job_works_on_its_own_tabs() {
let page = |id: &str| serde_json::json!({"type": "page", "id": id});
// Latest-used first: B's tab is in front, A's further back.
let list = vec![page("tab-b"), page("tab-a"), page("tab-free")];
as_job("job-a", || claim("tab-a"));
as_job("job-b", || claim("tab-b"));
assert_eq!(as_job("job-a", || pick_tab(&list).map(|t| t["id"].clone())), Some("tab-a".into()));
assert_eq!(as_job("job-b", || pick_tab(&list).map(|t| t["id"].clone())), Some("tab-b".into()));
// A new job starts on the latest tab nobody has; another job's tabs are off limits.
assert_eq!(as_job("job-c", || pick_tab(&list).map(|t| t["id"].clone())), Some("tab-free".into()));
assert!(as_job("job-c", || others_tab("tab-a") && !others_tab("tab-free")));
assert!(as_job("job-a", || !others_tab("tab-a")));
// A claimed tab stays its first owner's.
as_job("job-b", || claim("tab-a"));
assert_eq!(owner("tab-a").as_deref(), Some("job-a"));
// Outside a job nothing is claimed.
claim("tab-free");
assert_eq!(owner("tab-free"), None);
assert_eq!(release("job-a"), vec!["tab-a".to_string()]);
assert_eq!(release("job-b"), vec!["tab-b".to_string()]);
assert_eq!(owner("tab-a"), None);
}
#[test]
fn urls_are_encoded_for_new_tabs() {
assert_eq!(encode_component("https://www.threads.net/@me?x=1&y=2"), "https://www.threads.net/%40me%3Fx%3D1%26y%3D2");
}
#[test]
fn profiles_are_found_by_name() {
let list = vec![DEFAULT_PROFILE.to_string(), "工作".to_string(), "Personal".to_string()];
assert_eq!(match_profile(&list, " personal ").as_deref(), Some("Personal"));
assert_eq!(match_profile(&list, "工作").as_deref(), Some("工作"));
assert_eq!(match_profile(&list, "私人"), None);
assert_eq!(match_profile(&list, ""), None);
let with_person = vec![DEFAULT_PROFILE.to_string(), PERSON_PROFILE.to_string()];
assert_eq!(match_profile(&with_person, "我的 Chrome").as_deref(), Some(PERSON_PROFILE));
assert_eq!(match_profile(&list, "我的 Chrome"), None);
assert!(check_name("我的 Chrome").is_err());
assert!(check_name(PERSON_PROFILE).is_err());
}
#[test]
fn profile_names_are_checked() {
assert_eq!(check_name(" 工作 ").unwrap(), "工作");
assert!(check_name("").is_err());
assert!(check_name("a/b").is_err());
assert!(check_name("..").is_err());
assert!(check_name(&"字".repeat(PROFILE_NAME_MAX_CHARS + 1)).is_err());
assert!(check_name(&"字".repeat(PROFILE_NAME_MAX_CHARS)).is_ok());
}
#[test]
fn the_persons_firefox_profiles_are_found_with_their_names() {
let root = std::env::temp_dir().join(format!("anybot-fxtest-{}", crate::httpd::random_hex(4)));
for d in ["lDxV7bhX.設定檔 1", "3F5mWbNN.設定檔 5", "abc.default-release", "/abs-not-here"] {
let _ = std::fs::create_dir_all(root.join(d.trim_start_matches('/')));
}
let ini = "[General]\nStartWithLastProfile=1\n\n[Profile1]\nName=default-release\nIsRelative=1\nPath=abc.default-release\n\n[Profile0]\nName=default\nIsRelative=1\nPath=3F5mWbNN.設定檔 5\nDefault=1\n\n[Install4F96D1932A9F858E]\nDefault=x\n";
let got = ini_profiles(&root, ini);
assert_eq!(got, vec![("default-release".to_string(), root.join("abc.default-release")), ("default".to_string(), root.join("3F5mWbNN.設定檔 5"))]);
// The newer profile switcher keeps its names in a SQLite file.
std::fs::create_dir_all(root.join("Profile Groups")).unwrap();
let c = rusqlite::Connection::open(root.join("Profile Groups/8d582846.sqlite")).unwrap();
c.execute_batch("CREATE TABLE Profiles (id INTEGER PRIMARY KEY, path TEXT, name TEXT, avatar TEXT); INSERT INTO Profiles (path, name, avatar) VALUES ('lDxV7bhX.設定檔 1', '女生', 'flower'), ('3F5mWbNN.設定檔 5', '阿金', 'hammer');").unwrap();
drop(c);
assert_eq!(group_profiles(&root), vec![("女生".to_string(), root.join("lDxV7bhX.設定檔 1")), ("阿金".to_string(), root.join("3F5mWbNN.設定檔 5"))]);
let _ = std::fs::remove_dir_all(&root);
}
/// The person's own Firefox profile, end to end (headless): listed by its name, refused while
/// it's open the everyday way, then opened by AnyBot and driven. HOME is a scratch folder (a
/// Firefox profile list is made there):
/// HOME=$T ANYBOT_DATA_DIR=$T/data ANYBOT_FIREFOX_HEADLESS=1 <test binary> --ignored a_persons_firefox_profile
#[test]
#[ignore]
fn a_persons_firefox_profile() {
use std::io::{Read, Write};
let bin = firefox().expect("Firefox is installed");
let root = firefox_roots()[0].clone();
assert!(root.starts_with(std::env::var("HOME").unwrap()) && !root.exists(), "run with HOME set to an empty scratch folder");
let dir = root.join("x1.阿金");
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(root.join("profiles.ini"), "[Profile0]\nName=阿金\nIsRelative=1\nPath=x1.阿金\n").unwrap();
assert!(choices().contains(&format!("阿金{FIREFOX_MARK}")), "{:?}", choices());
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = l.local_addr().unwrap().port();
std::thread::spawn(move || {
for mut c in l.incoming().flatten() {
let mut buf = [0u8; 2048];
let _ = c.read(&mut buf);
let body = "<html><head><title>阿金的頁面</title></head><body><button onclick=\"this.innerText='按到了'\">按我</button></body></html>";
let _ = write!(c, "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
}
});
let wait_for = |want: bool| {
let t = Instant::now();
while firefox_in_use(&dir) != want && t.elapsed() < Duration::from_secs(15) {
std::thread::sleep(Duration::from_millis(200));
}
assert_eq!(firefox_in_use(&dir), want);
};
// Open the everyday way (nobody can drive it): AnyBot asks for it to be closed first.
let mut everyday = Command::new(bin).args(["--headless", "--new-instance", "--profile"]).arg(&dir).arg("about:blank").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap();
wait_for(true);
let e = open(&format!("http://127.0.0.1:{port}/"), Some("阿金")).unwrap_err();
assert!(e.contains("先關掉"), "{e}");
let _ = everyday.kill();
let _ = everyday.wait();
wait_for(false);
// Now AnyBot opens it itself and drives it like any of its own.
let go = || -> Result<(), String> {
let r = open(&format!("http://127.0.0.1:{port}/"), Some("阿金"))?;
assert!(r.contains("Firefox") && r.contains("阿金(Firefox)") && r.contains("按我"), "{r}");
let (t, _) = crate::browser::run("browser_click", &serde_json::json!({"text": "按我", "profile": "阿金"}))?;
assert!(t.contains("按到了"), "{t}");
Ok(())
};
let r = go();
let _ = Command::new("pkill").arg("-f").arg(dir.display().to_string()).status();
r.unwrap();
}
#[test]
fn a_firefox_profile_answers_to_its_own_name() {
let list = vec![DEFAULT_PROFILE.to_string(), "工作".to_string(), format!("阿金{FIREFOX_MARK}"), format!("Sunny{FIREFOX_MARK}")];
for want in ["阿金", "阿金(Firefox)", "Firefox 阿金", "firefox:阿金", "阿金 (firefox)"] {
assert_eq!(match_profile(&list, want).as_deref(), Some("阿金(Firefox)"), "{want}");
}
assert_eq!(match_profile(&list, "sunny").as_deref(), Some("Sunny(Firefox)"));
// AnyBot's own profile of the same name wins; "Firefox" alone means none of them.
let both = vec!["阿金".to_string(), format!("阿金{FIREFOX_MARK}")];
assert_eq!(match_profile(&both, "阿金").as_deref(), Some("阿金"));
assert_eq!(match_profile(&list, "firefox"), None);
assert!(check_name("我的 Firefox").is_err());
}
#[test]
fn chosen_port_is_read_back() {
assert_eq!(parse_active_port("53211\n/devtools/browser/abc\n"), Some(53211));
assert_eq!(parse_active_port("0\n"), None);
assert_eq!(parse_active_port(""), None);
}
}