AnyBot/src/cloudbox.rs

2297 lines
106 KiB
Rust
Raw Normal View History

// 「Bot 的電腦」: AnyBot's own Linux computer that every bot shares, like Grok Bot's cloud "box".
//
// What the original does (Grok Bot 0.78.1, main-app.cjs dev-box control plane + local-exec-daemon):
// - The box is a container with an X11 desktop driven by xdotool (ComputerUseDriver XDOTOOL /
// NATIVE_X11), its size read with xrandr, watched through noVNC websockify on port 6080
// (`Ap.primaryNovncWebsockify:6080`).
// - Run states: ABSENT → 「建立中」, HIBERNATED → 「喚醒中」, STARTING → 「連線中」, RUNNING.
// - Health: GET /health with a 3 s timeout (QQn=3e3) that also says whether it's busy.
// - Store flush before a rebuild: POST /api/snapshotBoxStoreNow, 30 s (VQn=3e4).
// - Three rebuilds, one at a time (`isRecreateInFlight`):
// update (keep data, refused while busy): backing-up → creating → moving
// reset (wipe): cleaning-up → wiping → creating
// restore (force, from the durable store): cleaning-up → creating, then wait up to 90 s
// (zQn=9e4) polling every 2 s (HQn=2e3) for the copy-in to say "hydrated"
// - recreate-resume.json replays a rebuild cut short by a restart if it is under 10 minutes old
// ($Qn=10*6e4) and the box isn't healthy.
// Here the box is a local Docker container (or one on another Docker host, `ssh://me@server`),
// the durable store is a snapshot of the bot's home folder kept on this Mac, and xdotool runs
// through `docker exec`.
use base64::Engine;
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
use chrono::Datelike;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::path::PathBuf;
use std::process::{Command, Stdio};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant, SystemTime};
/// The box's usual name. Two AnyBots on one Docker host (a server's brain and a desktop AnyBot
/// left on the same machine) each took "anybot-box" for their own, and the desktop one kept
/// pausing the brain's box as "idle". ANYBOT_BOX_NAME gives an AnyBot a box of its own name.
pub const CONTAINER: &str = "anybot-box";
/// This AnyBot's box (ANYBOT_BOX_NAME, else "anybot-box").
pub fn container() -> &'static str {
static C: OnceLock<String> = OnceLock::new();
C.get_or_init(|| {
std::env::var("ANYBOT_BOX_NAME")
.ok()
.map(|n| n.trim().to_string())
.filter(|n| !n.is_empty() && n.len() <= 60 && n.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'))
.unwrap_or_else(|| CONTAINER.to_string())
})
}
/// A box of its own name takes over the usual one this AnyBot used before (same files: the
/// home folder is a volume, kept by name; a rename doesn't stop it).
fn adopt_usual_box() {
let me = container();
if me == CONTAINER || dk(&["inspect", "-f", "{{.Id}}", me]).is_ok() {
return;
}
if dk(&["inspect", "-f", "{{.Id}}", CONTAINER]).is_ok() && dk(&["rename", CONTAINER, me]).is_ok() {
log(&format!("Bot 的電腦改用自己的名字 {me}(原本叫 {CONTAINER})。"));
}
}
pub const IMAGE: &str = "anybot-box:1";
/// The bot's files live here and survive an update (the original's mounted sand-data tree).
pub const VOLUME: &str = "anybot-box-home";
pub const HOME: &str = "/home/bot";
pub const DISPLAY: &str = ":1";
/// noVNC inside the box (Ap.primaryNovncWebsockify).
pub const NOVNC_PORT: u16 = 6080;
/// Where noVNC is published on this Mac (estimated; any free port works).
pub const HOST_PORT: u16 = 16080;
/// Desktop size (estimated; the original reads it with xrandr).
pub const SCREEN: &str = "1280x800x24";
/// Health probe timeout (QQn).
pub const HEALTH_TIMEOUT_MS: u64 = 3_000;
/// Saving a snapshot ("flush the durable store"). The original waits 30 s (VQn); a home folder of
/// a GB with an open browser takes minutes to pack, and a cut-off snapshot is no snapshot.
pub const SNAPSHOT_SAVE_TIMEOUT_MS: u64 = 600_000;
/// Restore copy-in: longest wait and poll interval (zQn, HQn).
pub const COPY_IN_WAIT_MS: u64 = 90_000;
pub const COPY_IN_POLL_MS: u64 = 2_000;
/// A rebuild marker older than this is cleared instead of replayed ($Qn).
pub const RESUME_FRESH_MS: u64 = 600_000;
/// Boot: longest wait for the desktop to come up (estimated).
pub const BOOT_WAIT_MS: u64 = 60_000;
/// Quiet time before the daily snapshot is taken (nobody using the box for this long).
pub const QUIET_SECS: u64 = 900;
/// The box's files are saved to this computer about once a day (it no longer sleeps, which is
/// when they used to be saved).
pub const SNAPSHOT_EVERY_SECS: i64 = 24 * 3600;
/// Secret names: an upper-case letter, then A–Z, 0–9, _ (original rule); size limits estimated.
pub const SECRET_MAX_VALUE_BYTES: usize = 32_768;
pub const SECRET_MAX_COUNT: usize = 100;
/// xdotool timing, matching the Mac helper where it applies (CUActuator).
const CLICK_REPEAT_MS: u64 = 80;
const TYPE_DELAY_MS: u64 = 12;
const DOCKERFILE: &str = include_str!("../box/Dockerfile");
const START_SH: &str = include_str!("../box/start.sh");
const FLUXBOX_INIT: &str = include_str!("../box/fluxbox-init");
const OVERLAY: &str = include_str!("../box/overlay");
const XRESOURCES: &str = include_str!("../box/Xresources");
const MENU: &str = include_str!("../box/menu");
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
const A11Y: &str = include_str!("../box/a11y");
const BROWSER: &str = include_str!("../box/browser");
// ---------- state ----------
#[derive(Clone, Debug, PartialEq)]
pub enum Run {
/// Docker isn't installed or isn't running.
NoDocker(String),
/// Not made yet (ABSENT).
Absent,
/// Asleep (HIBERNATED): paused or stopped.
Asleep,
/// Being made, woken or connected (the transient states).
Starting(&'static str),
Running,
/// A rebuild is under way: (phase, detail).
Rebuilding(String, String),
Failed(String),
}
impl Run {
pub fn label(&self) -> String {
match self {
Run::NoDocker(_) => "沒有 Docker".into(),
Run::Absent => "尚未建立".into(),
Run::Asleep => "休眠中".into(),
Run::Starting(s) => (*s).into(),
Run::Running => "執行中".into(),
Run::Rebuilding(p, _) => phase_label(p).into(),
Run::Failed(_) => "發生問題".into(),
}
}
pub fn is_running(&self) -> bool {
*self == Run::Running
}
}
pub fn phase_label(p: &str) -> &'static str {
match p {
"backing-up" => "正在備份",
"creating" => "正在建立",
"moving" => "正在搬移資料",
"cleaning-up" => "正在拆除舊電腦",
"wiping" => "正在清除資料",
"restoring" => "正在從快照還原",
"done" => "完成",
"failed" => "失敗",
_ => "處理中",
}
}
#[derive(Clone, Serialize, Deserialize, Default)]
pub struct Config {
/// Empty = this Mac's Docker; otherwise a Docker host such as `ssh://me@server`.
#[serde(default)]
pub docker_host: String,
/// 排程更新: local "HH:MM" to update when a newer computer is available (None = off).
#[serde(default)]
pub auto_update_at: Option<String>,
/// The day a scheduled update last ran (so it runs once).
#[serde(default)]
pub auto_update_done: String,
/// 網路流量改走你的電腦 (new connections only).
#[serde(default)]
pub egress_via_mac: bool,
/// VNC password for the desktop viewer (generated on first use; the published port is
/// loopback-only, but the box's own programs can reach it, and they should need a key).
#[serde(default)]
pub vnc_password: String,
}
/// A scheduled update that was missed may still run for this long (the original: one hour).
pub const SCHEDULE_GRACE_MINUTES: i64 = 60;
struct Shared {
run: Run,
last_used: Instant,
rebuilding: bool,
log: Vec<String>,
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
/// The desktop's size in pixels (the bot's pictures are this, fitted; see box_scale).
size: Option<(u32, u32)>,
tunnel: Option<std::process::Child>,
host_port: u16,
}
fn shared() -> &'static Mutex<Shared> {
static S: OnceLock<Mutex<Shared>> = OnceLock::new();
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
S.get_or_init(|| Mutex::new(Shared { run: Run::Absent, last_used: Instant::now(), rebuilding: false, log: vec![], size: None, tunnel: None, host_port: HOST_PORT }))
}
fn config() -> &'static Mutex<Config> {
static C: OnceLock<Mutex<Config>> = OnceLock::new();
C.get_or_init(|| Mutex::new(load_config()))
}
/// The box's VNC password: made once, kept across restarts (the container is started with it,
/// the viewer opens with it). x11vnc ran with no password at all before — any program inside
/// the box (a page the bot opened, for instance) could watch and drive the desktop.
pub fn vnc_password() -> String {
{
let c = config().lock().unwrap();
if !c.vnc_password.is_empty() {
return c.vnc_password.clone();
}
}
let pw = crate::httpd::random_hex(10);
config().lock().unwrap().vnc_password.clone_from(&pw);
save_config(&config().lock().unwrap());
pw
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
/// The password the box's VNC server really has: the one it was made with (its VNC_PASSWORD), so a
/// viewer signs in even when this AnyBot's settings were made anew; else this AnyBot's own.
pub fn box_vnc_password() -> String {
dk_timeout(&["exec", container(), "printenv", "VNC_PASSWORD"], 5_000)
.ok()
.map(|b| String::from_utf8_lossy(&b).trim().to_string())
.filter(|p| !p.is_empty())
.unwrap_or_else(vnc_password)
}
static BUSY: AtomicUsize = AtomicUsize::new(0);
/// The person took control in the viewer: the bot's screen actions wait.
pub static HELD_BY_PERSON: AtomicBool = AtomicBool::new(false);
/// Marks the box as in use while alive (keeps it awake; an update waits for it).
pub struct Busy(bool);
impl Busy {
pub fn new(on: bool) -> Self {
if on {
BUSY.fetch_add(1, Ordering::SeqCst);
touch();
}
Busy(on)
}
}
impl Drop for Busy {
fn drop(&mut self) {
if self.0 {
BUSY.fetch_sub(1, Ordering::SeqCst);
touch();
}
}
}
pub fn is_busy() -> bool {
BUSY.load(Ordering::SeqCst) > 0
}
pub fn touch() {
shared().lock().unwrap().last_used = Instant::now();
}
pub fn run_state() -> Run {
shared().lock().unwrap().run.clone()
}
fn set_run(r: Run) {
shared().lock().unwrap().run = r;
}
pub fn log_lines() -> Vec<String> {
shared().lock().unwrap().log.clone()
}
fn log(line: &str) {
let mut s = shared().lock().unwrap();
s.log.push(format!("{} {line}", chrono::Local::now().format("%H:%M:%S")));
let n = s.log.len();
if n > 200 {
s.log.drain(..n - 200);
}
}
// ---------- files on this Mac ----------
fn data_dir() -> PathBuf {
crate::model::State::path().parent().map(|p| p.to_path_buf()).unwrap_or_else(std::env::temp_dir).join("box")
}
fn config_path() -> PathBuf {
data_dir().join("config.json")
}
fn load_config() -> Config {
std::fs::read_to_string(config_path()).ok().and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default()
}
pub fn docker_host() -> String {
config().lock().unwrap().docker_host.clone()
}
pub fn set_docker_host(h: &str) {
let h = h.trim().to_string();
let mut c = config().lock().unwrap();
if c.docker_host == h {
return;
}
c.docker_host = h;
let _ = std::fs::create_dir_all(data_dir());
let _ = std::fs::write(config_path(), serde_json::to_string_pretty(&*c).unwrap_or_default());
drop(c);
if let Some(mut t) = shared().lock().unwrap().tunnel.take() {
let _ = t.kill();
}
set_run(Run::Absent);
refresh();
}
fn save_config(c: &Config) {
let _ = std::fs::create_dir_all(data_dir());
let _ = std::fs::write(config_path(), serde_json::to_string_pretty(c).unwrap_or_default());
}
pub fn egress_via_mac() -> bool {
config().lock().unwrap().egress_via_mac
}
/// Turn routing through this Mac on or off. Only this Mac's Docker can reach it.
pub fn set_egress_via_mac(on: bool) -> Result<(), String> {
if on {
if !docker_host().is_empty() {
return Err("Bot 的電腦在別的主機上時,不能改走這台 Mac。".into());
}
crate::egress::start()?;
}
let mut c = config().lock().unwrap();
c.egress_via_mac = on;
save_config(&c);
Ok(())
}
/// Proxy settings for new commands in the box when routing through this Mac.
fn egress_env() -> String {
if !egress_via_mac() || !docker_host().is_empty() || !crate::policy::read_egress().allows() {
return String::new();
}
match crate::egress::start() {
Ok(port) => {
let url = format!("http://host.docker.internal:{port}");
["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"].iter().map(|k| format!(" -e {k}={url}")).collect::<String>() + " -e NO_PROXY=localhost,127.0.0.1 -e no_proxy=localhost,127.0.0.1"
}
Err(_) => String::new(),
}
}
pub fn auto_update_at() -> Option<String> {
config().lock().unwrap().auto_update_at.clone()
}
pub fn set_auto_update_at(at: Option<String>) {
let mut c = config().lock().unwrap();
c.auto_update_at = at;
save_config(&c);
}
/// The computer version baked into this AnyBot (the Dockerfile's label).
pub fn wanted_version() -> String {
DOCKERFILE.lines().find_map(|l| l.trim().strip_prefix("LABEL anybot.box.version=")).map(|v| v.trim_matches('"').to_string()).unwrap_or_default()
}
/// The running computer is older than the one this AnyBot carries.
pub fn update_available() -> bool {
match dk(&["inspect", "-f", "{{index .Config.Labels \"anybot.box.version\"}}", container()]) {
Ok(v) => v.trim() != wanted_version(),
Err(_) => false,
}
}
/// Is `now` inside the window [at, at + grace)? (`at` is "HH:MM").
pub fn in_schedule_window(at: &str, now: chrono::NaiveTime) -> bool {
let Ok(t) = chrono::NaiveTime::parse_from_str(at.trim(), "%H:%M") else { return false };
let mins = (now - t).num_minutes().rem_euclid(24 * 60);
mins < SCHEDULE_GRACE_MINUTES
}
fn maybe_scheduled_update() {
let (at, done) = {
let c = config().lock().unwrap();
(c.auto_update_at.clone(), c.auto_update_done.clone())
};
let Some(at) = at else { return };
let now = chrono::Local::now();
let today = now.format("%Y-%m-%d").to_string();
if done == today || !in_schedule_window(&at, now.time()) || is_busy() || !update_available() {
return;
}
log(&format!("排程更新({at})開始。"));
if rebuild(Rebuild::Update).is_ok() {
let mut c = config().lock().unwrap();
c.auto_update_done = today;
save_config(&c);
}
}
pub fn snapshot_path() -> PathBuf {
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
list_snapshots().pop().unwrap_or_else(|| snapshots_dir().join("snapshot.tar.gz"))
}
fn snapshots_dir() -> PathBuf {
data_dir().join("snapshots")
}
/// Every saved snapshot, oldest first. Names carry the time (`snapshot-YYYY-MM-DD-HHMMSS.tar.gz`)
/// so they sort by time and a copy keeps its stamp. The single old `snapshot.tar.gz` (from
/// before generations were kept) is moved in, stamped by its file date.
pub fn list_snapshots() -> Vec<PathBuf> {
let dir = snapshots_dir();
let _ = std::fs::create_dir_all(&dir);
let legacy = data_dir().join("snapshot.tar.gz");
if legacy.exists() {
let name = std::fs::metadata(&legacy)
.and_then(|m| m.modified())
.ok()
.map(|t| format!("snapshot-{}-legacy.tar.gz", chrono::DateTime::<chrono::Local>::from(t).format("%Y-%m-%d-%H%M%S")))
.unwrap_or_else(|| "snapshot-0000-00-00-000000-legacy.tar.gz".into());
let _ = std::fs::rename(&legacy, dir.join(name));
}
let mut v: Vec<PathBuf> = std::fs::read_dir(&dir)
.map(|rd| {
rd.flatten()
.map(|e| e.path())
.filter(|p| {
let n = p.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
n.starts_with("snapshot-") && n.ends_with(".tar.gz")
})
.collect()
})
.unwrap_or_default();
v.sort();
v
}
/// When a snapshot was saved (from its name; file dates don't survive copies).
fn snapshot_stamp(p: &std::path::Path) -> Option<chrono::DateTime<chrono::Local>> {
let stem = p.file_name()?.to_string_lossy().strip_suffix(".tar.gz")?.strip_prefix("snapshot-")?.trim_end_matches("-legacy").to_string();
chrono::NaiveDateTime::parse_from_str(&stem, "%Y-%m-%d-%H%M%S")
.ok()
.and_then(|t| chrono::TimeZone::from_local_datetime(&chrono::Local, &t).single())
.or_else(|| std::fs::metadata(p).ok().and_then(|m| m.modified().ok()).map(chrono::DateTime::<chrono::Local>::from))
}
/// When the last snapshot was saved.
pub fn snapshot_time() -> Option<chrono::DateTime<chrono::Local>> {
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
snapshot_stamp(&snapshot_path())
}
/// Snapshot generations: the last 7 days in full, then the newest one of each of the three
/// weeks before. The home folder holds every bot's logins — one bad snapshot overwriting the
/// only copy used to be one command away.
fn prune_snapshots() {
let list = list_snapshots();
if list.len() <= 7 {
return;
}
let now = chrono::Local::now();
let mut weeks_kept: Vec<(i32, u32)> = vec![];
let mut del = vec![];
for p in list.iter().rev() {
let Some(t) = snapshot_stamp(p) else { continue };
let age = (now - t).num_days();
if age <= 7 {
continue;
}
if age <= 28 {
let w = (t.iso_week().year(), t.iso_week().week());
if !weeks_kept.contains(&w) {
weeks_kept.push(w);
continue;
}
}
del.push(p.clone());
}
for p in del {
let _ = std::fs::remove_file(&p);
}
}
fn resume_path() -> PathBuf {
data_dir().join("recreate-resume.json")
}
/// Where docker and ssh live when the app is started from Finder (no shell PATH).
pub fn tool_path() -> String {
let base = std::env::var("PATH").unwrap_or_default();
let mut parts: Vec<String> = base.split(':').filter(|s| !s.is_empty()).map(String::from).collect();
let home = dirs::home_dir().map(|h| h.display().to_string()).unwrap_or_default();
for extra in ["/usr/local/bin", "/opt/homebrew/bin", "/usr/bin", "/bin", "/usr/sbin", "/sbin", &format!("{home}/.orbstack/bin"), "/Applications/Docker.app/Contents/Resources/bin"] {
if !parts.iter().any(|p| p == extra) {
parts.push(extra.to_string());
}
}
parts.join(":")
}
fn docker() -> Command {
let mut c = Command::new("docker");
c.env("PATH", tool_path());
let host = docker_host();
if !host.is_empty() {
c.env("DOCKER_HOST", host);
}
c.stdin(Stdio::null());
c
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
/// A program in the box with its stdin and stdout piped to this side (as the bot user).
pub fn exec_piped(args: &[&str]) -> Result<std::process::Child, String> {
let mut a = vec!["exec", "-i", "-u", "bot", container()];
a.extend_from_slice(args);
docker().args(&a).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().map_err(|e| format!("找不到 docker:{e}"))
}
/// The `docker …` prefix for shell strings (used by the command tools).
fn docker_prefix() -> String {
let host = docker_host();
if host.is_empty() {
"docker".into()
} else {
format!("DOCKER_HOST={} docker", crate::tools::sh_quote(&host))
}
}
/// Run docker; Ok(stdout) or Err(stderr).
fn dk(args: &[&str]) -> Result<String, String> {
// Never block forever on the daemon: a hung docker daemon used to pile up a new stuck
// thread every 20 s from the refresh loop. (docker's own CLI has no default timeout.)
let out = dk_timeout(args, 120_000)?;
Ok(String::from_utf8_lossy(&out).trim().to_string())
}
fn dk_timeout(args: &[&str], ms: u64) -> Result<Vec<u8>, String> {
dk_with_input(args, None, ms)
}
/// Run docker with `input` on its stdin (if any): Ok(stdout) or Err(stderr), killed after `ms`
/// ("逾時"). Both pipes are drained while waiting, or a big screenshot fills a pipe and stalls
/// docker.
fn dk_with_input(args: &[&str], input: Option<&[u8]>, ms: u64) -> Result<Vec<u8>, String> {
use std::io::{Read, Write};
let mut child = docker()
.args(args)
.stdin(if input.is_some() { Stdio::piped() } else { Stdio::null() })
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|e| format!("找不到 docker:{e}"))?;
if let (Some(b), Some(mut si)) = (input, child.stdin.take()) {
si.write_all(b).map_err(|e| e.to_string())?;
}
let mut so = child.stdout.take().ok_or("沒有輸出")?;
let mut se = child.stderr.take().ok_or("沒有輸出")?;
let out_t = std::thread::spawn(move || {
let mut v = vec![];
let _ = so.read_to_end(&mut v);
v
});
let err_t = std::thread::spawn(move || {
let mut v = vec![];
let _ = se.read_to_end(&mut v);
v
});
let start = Instant::now();
let status = loop {
if let Ok(Some(st)) = child.try_wait() {
break st;
}
if start.elapsed() > Duration::from_millis(ms) {
let _ = child.kill();
let _ = child.wait();
return Err("逾時".into());
}
std::thread::sleep(Duration::from_millis(5));
};
let out = out_t.join().unwrap_or_default();
let err = err_t.join().unwrap_or_default();
if status.success() {
Ok(out)
} else {
Err(String::from_utf8_lossy(&err).trim().to_string())
}
}
// ---------- status ----------
/// Ask Docker what state the box is in (doesn't change anything).
pub fn probe() -> Run {
if let Err(e) = dk(&["version", "--format", "{{.Server.Version}}"]) {
return Run::NoDocker(if e.contains("找不到") && !cfg!(target_os = "macos") {
"這台電腦沒有安裝 Docker。安裝 Docker Engine(Ubuntu / Debian:sudo apt install docker.io),再把自己加進 docker 群組(sudo usermod -aG docker $USER,然後重新登入),Bot 就會有自己的完整電腦。".into()
} else if e.contains("找不到") {
"這台 Mac 沒有安裝 Docker。安裝 OrbStack 或 Docker Desktop 後,Bot 就會有自己的完整電腦。".into()
} else if e.contains("permission denied") && !cfg!(target_os = "macos") {
"Docker 不讓 AnyBot 使用:請執行 sudo usermod -aG docker $USER,然後登出再登入。".into()
} else {
format!("Docker 沒有在執行:{}", e.lines().last().unwrap_or(""))
});
}
match dk(&["inspect", "-f", "{{.State.Status}}", container()]) {
Err(_) => Run::Absent,
Ok(s) => match s.as_str() {
"running" => {
// The health probe (3 s, like the original's GET /health).
if dk_timeout(&["exec", container(), "test", "-f", "/tmp/anybot-box.ready"], HEALTH_TIMEOUT_MS).is_ok() {
Run::Running
} else {
Run::Starting("連線中")
}
}
"paused" | "exited" | "created" => Run::Asleep,
"restarting" => Run::Starting("連線中"),
other => Run::Failed(format!("容器狀態:{other}")),
},
}
}
/// Refresh the shown state in the background (unless a rebuild or start owns it).
pub fn refresh() {
std::thread::spawn(|| {
let r = probe();
let mut s = shared().lock().unwrap();
if s.rebuilding || matches!(s.run, Run::Starting(_)) && !matches!(r, Run::Running) {
return;
}
s.run = r;
});
}
pub fn available() -> bool {
!matches!(run_state(), Run::NoDocker(_))
}
// ---------- image ----------
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
/// The box image is here and is the version this AnyBot carries (an old one would make a new
/// box without the newer parts: the numbered controls, the browser launcher).
fn image_exists() -> bool {
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
dk(&["image", "inspect", "-f", "{{index .Config.Labels \"anybot.box.version\"}}", IMAGE]).map(|v| v.trim() == wanted_version()).unwrap_or(false)
}
/// Build the box image from the files baked into AnyBot.
fn build_image(pull: bool) -> Result<(), String> {
let dir = std::env::temp_dir().join(format!("anybot-box-build-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
for (name, body) in [("Dockerfile", DOCKERFILE), ("start.sh", START_SH), ("fluxbox-init", FLUXBOX_INIT), ("overlay", OVERLAY), ("Xresources", XRESOURCES), ("menu", MENU), ("a11y", A11Y), ("browser", BROWSER)] {
std::fs::write(dir.join(name), body).map_err(|e| e.to_string())?;
}
log("正在建立電腦的映像檔(第一次要幾分鐘)…");
let mut args = vec!["build", "-q", "-t", IMAGE];
if pull {
args.push("--pull");
}
let d = dir.display().to_string();
args.push(&d);
// Builds take minutes (a cold one more than dk's short timeout allows).
let r = dk_timeout(&args, 20 * 60_000).map(|_| String::new());
let _ = std::fs::remove_dir_all(&dir);
r.map(|_| log("映像檔完成。")).map_err(|e| format!("建立映像檔失敗:{}", tail(&e, 600)))
}
fn tail(s: &str, n: usize) -> String {
let c: Vec<char> = s.chars().collect();
if c.len() <= n {
s.to_string()
} else {
c[c.len() - n..].iter().collect()
}
}
// ---------- lifecycle ----------
fn free_port(start: u16) -> u16 {
if !docker_host().is_empty() {
return start;
}
(start..start + 50).find(|p| std::net::TcpListener::bind(("127.0.0.1", *p)).is_ok()).unwrap_or(start)
}
fn create_container() -> Result<(), String> {
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
shared().lock().unwrap_or_else(|e| e.into_inner()).size = None;
if !image_exists() {
build_image(false)?;
}
let port = free_port(HOST_PORT);
let publish = format!("127.0.0.1:{port}:{NOVNC_PORT}");
let vol = format!("{VOLUME}:{HOME}");
let env = format!("SCREEN_SIZE={SCREEN}");
let vnc = format!("VNC_PASSWORD={}", vnc_password());
log("正在開一台新的電腦…");
// host.docker.internal → this Mac (for 網路流量改走你的電腦; Docker Desktop and OrbStack already have it).
dk(&["run", "-d", "--name", container(), "--hostname", "bot-computer", "--shm-size", "1g", "--pids-limit", "4096", "--memory", "2g", "--cpus", "2", "--add-host", "host.docker.internal:host-gateway", "-p", &publish, "-v", &vol, "-e", &env, "-e", &vnc, "--label", "anybot.box=1", IMAGE]).map_err(|e| format!("開不了電腦:{}", tail(&e, 400)))?;
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
ensure_holder();
Ok(())
}
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
fn holder_name() -> String {
format!("{}-home-holder", container())
}
/// Keep the home volume attached to a container. A box removed by an update is a moment with no
/// container using the volume — and `docker volume prune` (a command people run to reclaim
/// disk) deletes volumes that are attached to nothing, logins and all. One sleeping holder
/// keeps it always attached.
fn ensure_holder() {
if dk(&["volume", "inspect", "-f", "{{.Name}}", VOLUME]).is_err() {
return;
}
if dk(&["inspect", "-f", "{{.State.Running}}", &holder_name()]).map(|s| s == "true").unwrap_or(false) {
return;
}
let _ = dk(&["rm", "-f", &holder_name()]);
let vol = format!("{VOLUME}:{HOME}");
let _ = dk(&["run", "-d", "--name", &holder_name(), "--label", "anybot.box=1", "-v", &vol, IMAGE, "sleep", "infinity"]);
}
fn wait_ready(ms: u64) -> Result<(), String> {
let start = Instant::now();
while start.elapsed() < Duration::from_millis(ms) {
if dk(&["exec", container(), "test", "-f", "/tmp/anybot-box.ready"]).is_ok() {
return Ok(());
}
std::thread::sleep(Duration::from_millis(250));
}
Err("電腦開機逾時".into())
}
// ---------- the box's own browser ----------
// Browser steps on Bot 的電腦 drive the Chromium on its desktop (the person can watch it in the
// box's window). Its DevTools port stays inside the box; a local port reaches it through
// `docker exec` (no published port, so it also works with a Docker host elsewhere).
const BOX_CDP_PORT: u16 = 9222;
fn box_cdp_alive(local: u16) -> bool {
crate::botbrowser::local().get(&format!("http://127.0.0.1:{local}/json/version")).call().map(|r| r.status().as_u16() == 200).unwrap_or(false)
}
/// A local port that reaches the box's DevTools port.
fn box_cdp_forward() -> Result<u16, String> {
static PORT: Mutex<u16> = Mutex::new(0);
let mut p = PORT.lock().unwrap();
if *p != 0 {
return Ok(*p);
}
let l = std::net::TcpListener::bind("127.0.0.1:0").map_err(|e| e.to_string())?;
let port = l.local_addr().map_err(|e| e.to_string())?.port();
std::thread::spawn(move || {
for conn in l.incoming().flatten() {
std::thread::spawn(move || {
let script = format!("exec 3<>/dev/tcp/127.0.0.1/{BOX_CDP_PORT} || exit 1; cat <&3 & cat >&3; kill $! 2>/dev/null");
let Ok(mut child) = docker().args(["exec", "-i", container(), "bash", "-c", &script]).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()).spawn() else { return };
let (Some(mut cin), Some(mut cout)) = (child.stdin.take(), child.stdout.take()) else { return };
let (Ok(mut r), Ok(mut w)) = (conn.try_clone(), conn.try_clone()) else { return };
let up = std::thread::spawn(move || {
let _ = std::io::copy(&mut r, &mut cin);
});
let _ = std::io::copy(&mut cout, &mut w);
let _ = conn.shutdown(std::net::Shutdown::Both);
let _ = child.kill();
let _ = up.join();
let _ = child.wait();
});
}
});
*p = port;
Ok(port)
}
/// Is the box's Chromium answering already?
pub fn browser_running() -> bool {
box_cdp_forward().map(box_cdp_alive).unwrap_or(false)
}
/// The box's Chromium, started on its desktop when it isn't running. Gives the local port.
pub fn browser_port() -> Result<u16, String> {
ensure_running()?;
touch();
let port = box_cdp_forward()?;
if box_cdp_alive(port) {
return Ok(port);
}
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
// The box's one browser launcher (same profile and port as the person's menu entry, web
// pages exposed to accessibility).
dk_timeout(&["exec", "-u", "bot", container(), "anybot-browser"], 30_000).map_err(|e| format!("Bot 的電腦打不開瀏覽器:{e}"))?;
let start = Instant::now();
while start.elapsed() < Duration::from_secs(20) {
if box_cdp_alive(port) {
return Ok(port);
}
std::thread::sleep(Duration::from_millis(300));
}
Err("這台電腦的瀏覽器沒有及時打開,請再試一次。".into())
}
static START_LOCK: Mutex<()> = Mutex::new(());
/// Make sure the box is up, waking or creating it as needed. Blocks until it's ready.
pub fn ensure_running() -> Result<(), String> {
let _g = START_LOCK.lock().unwrap();
if shared().lock().unwrap().rebuilding {
return Err("Bot 的電腦正在更新,請等它完成再試。".into());
}
touch();
let r = probe();
let result = match r {
Run::Running => Ok(()),
Run::NoDocker(e) => Err(e),
Run::Absent => {
set_run(Run::Starting("建立中"));
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
create_container().and_then(|_| {
// A brand-new box on an empty volume (a wiped Docker, a new host): the saved
// home brings the files and every bot's logins back by themselves.
if snapshot_path().exists() && !home_has_files() {
log("新的電腦家目錄是空的:從最近的快照還原檔案和登入…");
let src = snapshot_path();
if let Err(e) = copy_in_snapshot(&src) {
log(&format!("自動還原沒有完成:{e}(電腦照常用,檔案以現在為準)。"));
}
}
wait_ready(BOOT_WAIT_MS)
})
}
Run::Asleep => {
set_run(Run::Starting("喚醒中"));
log("喚醒電腦…");
let st = dk(&["inspect", "-f", "{{.State.Status}}", container()]).unwrap_or_default();
let woke = if st == "paused" { dk(&["unpause", container()]) } else { dk(&["start", container()]) };
woke.map_err(|e| format!("叫不醒電腦:{e}")).and_then(|_| wait_ready(BOOT_WAIT_MS))
}
Run::Starting(_) => {
set_run(Run::Starting("連線中"));
wait_ready(BOOT_WAIT_MS)
}
Run::Rebuilding(..) => Err("Bot 的電腦正在更新,請等它完成再試。".into()),
Run::Failed(e) => {
// A broken container: start it again.
log(&format!("電腦狀態異常({e}),重新啟動…"));
set_run(Run::Starting("連線中"));
dk(&["restart", container()]).map_err(|e| e.to_string()).and_then(|_| wait_ready(BOOT_WAIT_MS))
}
};
match &result {
Ok(()) => {
set_run(Run::Running);
let _ = dk(&["exec", container(), "sh", "-c", &format!("mkdir -p {HOME} && chown bot:bot {HOME}")]);
}
Err(e) => {
log(e);
set_run(match probe() {
Run::NoDocker(x) => Run::NoDocker(x),
_ => Run::Failed(e.clone()),
});
}
}
touch();
result
}
/// Started once: keeps the box awake (wakes it when it was asleep), saves its files once a day,
/// and replays a rebuild cut short by a restart. The person asked for the bots' computer to be
/// always there: an idle box used to be paused after 15 minutes, and the person — and a bot
/// working from a brain far away — kept finding it "asleep".
pub fn start_background() {
static STARTED: AtomicBool = AtomicBool::new(false);
if STARTED.swap(true, Ordering::SeqCst) {
return;
}
std::thread::spawn(|| {
adopt_usual_box();
set_run(probe());
resume_interrupted();
loop {
// (First wait a little: a computer showing another brain says so right after start.)
std::thread::sleep(Duration::from_secs(20));
let (idle, rebuilding) = {
let s = shared().lock().unwrap();
(s.last_used.elapsed().as_secs(), s.rebuilding)
};
if !rebuilding {
if IS_BRAIN.load(Ordering::SeqCst) && matches!(probe(), Run::Asleep) {
log("Bot 的電腦不休眠:叫醒它。");
let _ = ensure_running();
}
let quiet = idle >= QUIET_SECS && !is_busy() && !HELD_BY_PERSON.load(Ordering::SeqCst) && !viewer_open();
let due = snapshot_time().map(|t| (chrono::Local::now() - t).num_seconds() >= SNAPSHOT_EVERY_SECS).unwrap_or(true);
if quiet && due && run_state().is_running() {
let _ = save_snapshot();
}
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
// (Only when a box exists on this Docker: the holder is its volume's seat-saver.)
if matches!(probe(), Run::Asleep | Run::Running) {
ensure_holder();
}
refresh();
maybe_scheduled_update();
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
update_when_idle(idle);
}
}
});
}
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
/// The brain's box is older than this AnyBot (it lacks parts the bots now use: the numbered
/// controls, the browser launcher): update it by itself once nobody has used it for a minute,
/// once per version. Files and logins stay (they live in the home volume).
fn update_when_idle(idle_secs: u64) {
static TRIED: Mutex<String> = Mutex::new(String::new());
if !IS_BRAIN.load(Ordering::SeqCst) || idle_secs < 60 || is_busy() || HELD_BY_PERSON.load(Ordering::SeqCst) || viewer_open() || !update_available() {
return;
}
let want = wanted_version();
{
let mut t = TRIED.lock().unwrap_or_else(|e| e.into_inner());
if *t == want {
return;
}
*t = want.clone();
}
log(&format!("Bot 的電腦是舊版,自動更新到第 {want} 版(檔案和登入都會保留)。"));
if let Err(e) = rebuild(Rebuild::Update) {
log(&format!("自動更新沒有成功:{e}"));
}
}
/// This AnyBot is the brain (its box is the bots' computer). A computer showing another brain
/// doesn't use its own box, so it isn't kept awake there.
static IS_BRAIN: AtomicBool = AtomicBool::new(true);
pub fn set_brain(on: bool) {
IS_BRAIN.store(on, Ordering::SeqCst);
}
static VIEWER_OPEN: AtomicBool = AtomicBool::new(false);
pub fn set_viewer_open(on: bool) {
VIEWER_OPEN.store(on, Ordering::SeqCst);
if on {
touch();
}
}
fn viewer_open() -> bool {
VIEWER_OPEN.load(Ordering::SeqCst)
}
// ---------- snapshots (the durable store) ----------
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
/// Save the bot's home folder to this Mac, as a new generation. Returns its size in bytes.
pub fn save_snapshot() -> Result<u64, String> {
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
let dir = snapshots_dir();
let _ = std::fs::create_dir_all(&dir);
let stamp = chrono::Local::now().format("%Y-%m-%d-%H%M%S");
let tmp = dir.join(format!("snapshot-{stamp}.tmp"));
let final_path = dir.join(format!("snapshot-{stamp}.tar.gz"));
let file = std::fs::File::create(&tmp).map_err(|e| e.to_string())?;
let mut child = docker()
// Caches are left out: the browser's grow to hundreds of MB and come back by themselves.
.args([
"exec", container(), "tar", "-C", HOME, "--exclude=./.cache", "--exclude=Cache", "--exclude=Code Cache", "--exclude=GPUCache", "--exclude=CacheStorage", "--exclude=ScriptCache", "--exclude=Crashpad",
"--warning=no-file-changed", "-czf", "-", ".",
])
.stdout(Stdio::from(file))
.stderr(Stdio::piped())
.spawn()
.map_err(|e| e.to_string())?;
// Drain stderr: tar's file-changed warnings pile up fast, and a full 64 KB pipe stalls
// tar (and the snapshot) until the kill timer fires.
let mut se = child.stderr.take().ok_or("沒有輸出")?;
let err_t = std::thread::spawn(move || {
use std::io::Read;
let mut v = vec![];
let _ = se.read_to_end(&mut v);
v
});
let start = Instant::now();
let ok = loop {
// tar says 1 when a file changed while it was read (the open browser writes all the
// time): the archive is still whole. Only 2 and up are failures.
if let Ok(Some(st)) = child.try_wait() {
break st.code().map(|c| c <= 1).unwrap_or(false);
}
if start.elapsed() > Duration::from_millis(SNAPSHOT_SAVE_TIMEOUT_MS) {
let _ = child.kill();
break false;
}
std::thread::sleep(Duration::from_millis(50));
};
let _ = err_t.join();
if !ok {
let _ = std::fs::remove_file(&tmp);
return Err("快照沒有存成功".into());
}
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
std::fs::rename(&tmp, &final_path).map_err(|e| e.to_string())?;
let n = std::fs::metadata(&final_path).map(|m| m.len()).unwrap_or(0);
prune_snapshots();
log(&format!("已儲存快照({},共 {} 代)。", human(n), list_snapshots().len()));
Ok(n)
}
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
/// Put a saved snapshot's files back into the box's home folder.
fn copy_in_snapshot(src: &std::path::Path) -> Result<(), String> {
use std::io::Write;
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
let data = std::fs::read(src).map_err(|_| "還沒有快照可以還原".to_string())?;
let mut child = docker()
.args(["exec", "-i", container(), "sh", "-c", &format!("tar -C {HOME} -xzf - && chown -R bot:bot {HOME} && echo '[box-copy-in] result outcome=hydrated' >> /tmp/anybot-copy-in.log")])
.stdin(Stdio::piped())
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.map_err(|e| e.to_string())?;
child.stdin.take().ok_or("寫不進電腦")?.write_all(&data).map_err(|e| e.to_string())?;
let out = child.wait_with_output().map_err(|e| e.to_string())?;
if !out.status.success() {
return Err(format!("還原失敗:{}", String::from_utf8_lossy(&out.stderr).trim()));
}
// Like readCopyInOutcome: wait for the copy-in to report "hydrated".
let start = Instant::now();
while start.elapsed() < Duration::from_millis(COPY_IN_WAIT_MS) {
if let Ok(s) = dk(&["exec", container(), "sh", "-c", "cat /tmp/anybot-copy-in.log 2>/dev/null || true"]) {
if s.contains("outcome=hydrated") {
return Ok(());
}
}
std::thread::sleep(Duration::from_millis(COPY_IN_POLL_MS));
}
Err("還原沒有完成:copy-in outcome=unknown".into())
}
fn home_has_files() -> bool {
dk(&["exec", container(), "sh", "-c", &format!("ls -A {HOME} | head -1")]).map(|s| !s.trim().is_empty()).unwrap_or(false)
}
pub fn human(b: u64) -> String {
match b {
b if b >= 1 << 30 => format!("{:.1} GB", b as f64 / (1u64 << 30) as f64),
b if b >= 1 << 20 => format!("{:.1} MB", b as f64 / (1u64 << 20) as f64),
b if b >= 1 << 10 => format!("{:.0} KB", b as f64 / 1024.0),
b => format!("{b} B"),
}
}
// ---------- rebuilds ----------
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Rebuild {
/// 重建電腦: keep files and logins, remove installed apps and packages.
Update,
/// 重設: a brand-new computer, nothing kept.
Reset,
/// 復原: start again from the last saved snapshot.
Restore,
}
impl Rebuild {
pub fn title(self) -> &'static str {
match self {
Rebuild::Update => "重建電腦",
Rebuild::Reset => "重設電腦",
Rebuild::Restore => "從快照復原",
}
}
/// The original's confirmation wording.
pub fn warning(self) -> &'static str {
match self {
Rebuild::Update => "你的檔案和登入資訊會保留,但已安裝的應用程式和套件會被移除。所有 Bot 會一起更新。正在工作的 Bot 會先停下來,聊天會保留。",
Rebuild::Reset => "重新建立電腦並重新連線。電腦裡的所有檔案、登入資訊和已安裝的應用程式都會被清除(你的 Bot 和聊天不受影響)。",
Rebuild::Restore => "從你上次儲存的快照重新啟動它。最近的變更可能會遺失,已安裝的 App 會被移除。你的 Bot、檔案和登入資訊都是安全的。",
}
}
}
#[derive(Serialize, Deserialize)]
struct ResumeMarker {
#[serde(rename = "resumeKind")]
kind: Rebuild,
#[serde(rename = "writtenAtMs")]
written_at_ms: u64,
}
fn now_ms() -> u64 {
SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0)
}
/// Start a rebuild in the background. Err = refused (already running, busy, no snapshot…).
pub fn rebuild(kind: Rebuild) -> Result<(), String> {
{
let mut s = shared().lock().unwrap();
if s.rebuilding {
return Err("已經有一個更新在進行中。".into());
}
if kind == Rebuild::Update && BUSY.load(Ordering::SeqCst) > 0 {
return Err("有 Bot 正在用這台電腦,等它做完再重建(或先停止它)。".into());
}
if kind == Rebuild::Restore && !snapshot_path().exists() {
return Err("還沒有儲存過快照,沒有東西可以還原。".into());
}
s.rebuilding = true;
}
std::thread::spawn(move || {
let _ = std::fs::create_dir_all(data_dir());
let _ = std::fs::write(resume_path(), serde_json::to_string(&ResumeMarker { kind, written_at_ms: now_ms() }).unwrap_or_default());
let phase = |p: &str, d: &str| {
log(&format!("{}:{d}", phase_label(p)));
shared().lock().unwrap().run = Run::Rebuilding(p.into(), d.into());
};
let r = run_rebuild(kind, &phase);
let _ = std::fs::remove_file(resume_path());
match r {
Ok(()) => {
log(&format!("{}完成。", kind.title()));
let mut s = shared().lock().unwrap();
s.rebuilding = false;
s.run = Run::Running;
s.last_used = Instant::now();
}
Err(e) => {
log(&format!("{}失敗:{e}", kind.title()));
let mut s = shared().lock().unwrap();
s.rebuilding = false;
s.run = Run::Failed(e);
}
}
});
Ok(())
}
fn run_rebuild(kind: Rebuild, phase: &dyn Fn(&str, &str)) -> Result<(), String> {
match kind {
Rebuild::Update => {
let had = matches!(probe(), Run::Running);
phase("backing-up", "儲存一份快照");
if had {
if let Err(e) = save_snapshot() {
log(&format!("快照沒有存成功({e}),改以掛載的資料為準。"));
}
}
let had_files = had && home_has_files();
phase("creating", "更新電腦的映像檔");
build_image(true)?;
phase("creating", "換上一台新的電腦");
let _ = dk(&["rm", "-f", container()]);
create_container()?;
wait_ready(BOOT_WAIT_MS)?;
phase("moving", "確認保留的檔案和新電腦的連線");
if had_files && !home_has_files() {
return Err("重建後找不到原本的檔案".into());
}
Ok(())
}
Rebuild::Reset => {
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
// The box's home is the only copy of every bot's logins: never wipe it without a
// snapshot of what's in it first — reset becomes undoable.
if matches!(probe(), Run::Running) {
phase("backing-up", "重設前:先存下現在的電腦");
save_snapshot().map_err(|e| format!("重設取消了:存不下現在的狀態({e})。先處理好這個,重設才救得回來。"))?;
}
phase("cleaning-up", "拆掉舊的電腦");
let _ = dk(&["rm", "-f", container()]);
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
let _ = dk(&["rm", "-f", &holder_name()]);
phase("wiping", "清除電腦裡的資料");
let _ = dk(&["volume", "rm", "-f", VOLUME]);
phase("creating", "開一台全新的電腦");
if !image_exists() {
build_image(false)?;
}
create_container()?;
wait_ready(BOOT_WAIT_MS)
}
Rebuild::Restore => {
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
// Say which snapshot to restore before saving the current state (saving adds a
// newer "latest" that restore must not pick up).
let src = snapshot_path();
if matches!(probe(), Run::Running) {
phase("backing-up", "復原前:先存下現在的電腦");
save_snapshot().map_err(|e| format!("復原取消了:存不下現在的狀態({e})。先處理好這個,最近的變更才不會無聲消失。"))?;
}
phase("cleaning-up", "拆掉舊的電腦和它的資料");
let _ = dk(&["rm", "-f", container()]);
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
let _ = dk(&["rm", "-f", &holder_name()]);
let _ = dk(&["volume", "rm", "-f", VOLUME]);
phase("creating", "從快照開一台新的電腦");
if !image_exists() {
build_image(false)?;
}
create_container()?;
wait_ready(BOOT_WAIT_MS)?;
phase("restoring", "把快照裡的檔案放回去");
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
copy_in_snapshot(&src)
}
}
}
/// resumeInterruptedRecreate: replay a fresh marker when the box isn't healthy.
fn resume_interrupted() {
let Some(m) = std::fs::read_to_string(resume_path()).ok().and_then(|s| serde_json::from_str::<ResumeMarker>(&s).ok()) else { return };
if now_ms().saturating_sub(m.written_at_ms) >= RESUME_FRESH_MS {
log("清掉一個太舊的未完成更新紀錄(超過十分鐘)。");
let _ = std::fs::remove_file(resume_path());
return;
}
if matches!(probe(), Run::Running) {
log("上次的更新被中斷,但電腦目前正常,所以沒有重做。需要的話請再執行一次。");
let _ = std::fs::remove_file(resume_path());
return;
}
log(&format!("接續上次被中斷的「{}」。", m.kind.title()));
let _ = rebuild(m.kind);
}
// ---------- disk ----------
/// Disk use in the box, for the settings page and the clean-up request.
pub fn disk_report() -> Result<String, String> {
ensure_running()?;
dk(&["exec", "-u", "bot", container(), "sh", "-c", &format!("df -h {HOME} | tail -1; echo; du -sh {HOME}/* {HOME}/.cache 2>/dev/null | sort -rh | head -8")])
}
// ---------- secrets (write-only) ----------
fn secrets_path() -> PathBuf {
data_dir().join("secrets.json")
}
fn env_file_path() -> PathBuf {
data_dir().join("secrets.env")
}
fn read_secrets() -> std::collections::BTreeMap<String, String> {
std::fs::read_to_string(secrets_path()).ok().and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default()
}
fn write_secrets(m: &std::collections::BTreeMap<String, String>) -> Result<(), String> {
let _ = std::fs::create_dir_all(data_dir());
let write = |p: PathBuf, body: String| -> Result<(), String> {
std::fs::write(&p, body).map_err(|e| e.to_string())?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o600));
}
Ok(())
};
write(secrets_path(), serde_json::to_string(m).unwrap_or_default())?;
// docker --env-file: one NAME=value per line (no newlines allowed in values).
write(env_file_path(), m.iter().map(|(k, v)| format!("{k}={}\n", v.replace('\n', "\\n"))).collect())
}
/// Names only: values can be written, never read back.
pub fn secret_names() -> Vec<String> {
read_secrets().into_keys().collect()
}
pub fn valid_secret_name(n: &str) -> bool {
let mut c = n.chars();
matches!(c.next(), Some('A'..='Z')) && c.all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit() || ch == '_')
}
pub fn set_secret(name: &str, value: &str) -> Result<(), String> {
let name = name.trim();
if !valid_secret_name(name) {
return Err("名稱要大寫字母開頭,只能用 A–Z、0–9 和 _,例如 CRM_API_TOKEN".into());
}
if value.is_empty() {
return Err("請填入值".into());
}
// docker's --env-file keeps values on one line; a value with newlines used to be written
// with literal "\n" escapes and arrived broken (multi-line PEM keys) — say so up front.
if value.contains(['\n', '\r']) {
return Err("值裡不能有換行(多行金鑰請壓成一行,例如 -----BEGIN KEY-----…-----END KEY-----)".into());
}
if value.len() > SECRET_MAX_VALUE_BYTES {
return Err(format!("值太長了(最多 {} KB)", SECRET_MAX_VALUE_BYTES / 1024));
}
let mut m = read_secrets();
if !m.contains_key(name) && m.len() >= SECRET_MAX_COUNT {
return Err(format!("最多 {SECRET_MAX_COUNT} 個密鑰"));
}
m.insert(name.to_string(), value.to_string());
write_secrets(&m)
}
pub fn remove_secret(name: &str) {
let mut m = read_secrets();
if m.remove(name).is_some() {
let _ = write_secrets(&m);
}
}
// ---------- commands ----------
/// The bot's own folder inside the box.
pub fn bot_dir(bot_id: u64) -> String {
format!("{HOME}/bot-{bot_id}")
}
/// A local shell string that runs `cmd` in the box as the bot user, in `dir`, with the secrets.
pub fn exec_wrapper(dir: &str, cmd: &str, interactive: bool, job: &str) -> String {
use crate::tools::sh_quote;
// The job tag rides in the command line so the app can kill the in-container process later
// (killing the local docker client alone leaves the program running in the box).
let tagged = if job.is_empty() { cmd.to_string() } else { format!("export ANYBOT_JOB={job}; {cmd}") };
let inner = format!("mkdir -p {d} && cd {d} && {tagged}", d = sh_quote(dir));
let env = env_file_path();
let env_arg = if env.exists() { format!(" --env-file {}", sh_quote(&env.display().to_string())) } else { String::new() };
format!("{} exec{} -u bot -e DISPLAY={DISPLAY} -e HOME={HOME}{}{env_arg} {box} bash -lc {}", docker_prefix(), if interactive { " -i" } else { "" }, egress_env(), sh_quote(&inner), r#box = container())
}
/// An argv that kills the box process carrying this job tag (and its children).
pub fn exec_kill_argv(job: &str) -> Vec<String> {
let mut argv = vec!["docker".to_string()];
let host = docker_host();
if !host.is_empty() {
argv.extend(["-H".to_string(), host]);
}
argv.extend([
"exec".to_string(),
container().to_string(),
"sh".to_string(),
"-c".to_string(),
format!("for p in $(pgrep -f 'export ANYBOT_JOB={job}'); do pkill -9 -P $p; pkill -9 -P $p; kill -9 $p 2>/dev/null; done; true"),
]);
argv
}
/// Put a file into a bot's folder on the box (only when the box is already running).
pub fn copy_in(local: &str, bot_id: u64, name: &str) -> Result<String, String> {
if !run_state().is_running() {
return Err("Bot 的電腦沒有在執行".into());
}
// The name arrives from uploads: keep only its file part, so no `../` can steer the
// root-run `docker cp` out of this bot's attachments folder.
let safe = std::path::Path::new(name)
.file_name()
.map(|n| n.to_string_lossy().chars().filter(|c| *c != '\0').collect::<String>())
.filter(|n| !n.is_empty() && n != "." && n != "..")
.ok_or_else(|| "檔案名稱無效".to_string())?;
let dir = format!("{}/附件", bot_dir(bot_id));
dk(&["exec", "-u", "bot", container(), "mkdir", "-p", &dir])?;
let dst = format!("{dir}/{safe}");
dk(&["cp", local, &format!("{c}:{dst}", c = container())])?;
let _ = dk(&["exec", container(), "chown", "bot:bot", &dst]);
Ok(dst)
}
/// Copy a file out of the box to this Mac (for showing the person). Returns the local path.
pub fn copy_out(path_in_box: &str) -> Result<PathBuf, String> {
ensure_running()?;
let name = std::path::Path::new(path_in_box).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "file".into());
let dir = std::env::temp_dir().join(format!("anybot-from-box-{}", chrono::Utc::now().timestamp_millis()));
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
let local = dir.join(&name);
dk(&["cp", &format!("{}:{path_in_box}", container()), &local.display().to_string()]).map_err(|e| format!("拿不到檔案:{e}"))?;
Ok(local)
}
// ---------- the box's screen (xdotool) ----------
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
// One `docker exec` per step: the xdotool commands, then (for the bot) a short wait and the
// picture of the result, in one shell script. Each exec costs 50–200 ms locally and more on a
// Docker host elsewhere; a click with a modifier used to be four of them, plus one more for the
// screenshot the bot then had to ask for.
/// The latest picture of the box's screen, small (for the 電腦 card). While the box runs it is
/// taken now (and kept on disk); while it sleeps the last one taken is shown.
pub fn thumbnail(width: u32) -> Option<image::RgbaImage> {
let path = data_dir().join("last-screen.png");
if let Some(img) = frame() {
let h = (img.height() as f32 * width as f32 / img.width().max(1) as f32).round() as u32;
let small = image::imageops::resize(&img, width, h.max(1), image::imageops::FilterType::Triangle);
let _ = std::fs::create_dir_all(data_dir());
let _ = small.save(&path);
return Some(small);
}
image::open(&path).ok().map(|i| i.to_rgba8())
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
/// The screen now, for a small picture (no waking: None when the box isn't running). People watch
/// it live through VNC (vnc.rs).
pub fn frame() -> Option<image::RgbaImage> {
if !run_state().is_running() {
return None;
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
let jpeg = dk_timeout(&["exec", container(), "import", "-window", "root", "-display", DISPLAY, "-quality", "80", "jpeg:-"], 5_000).ok()?;
image::load_from_memory(&jpeg).ok().map(|i| i.to_rgba8())
}
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
/// The desktop's size in pixels (asked once per box).
fn box_size() -> Result<(u32, u32), String> {
if let Some(s) = shared().lock().unwrap_or_else(|e| e.into_inner()).size {
return Ok(s);
}
let out = dk(&["exec", "-u", "bot", "-e", "DISPLAY=:1", container(), "xdotool", "getdisplaygeometry"])?;
let mut it = out.split_whitespace().filter_map(|n| n.parse::<u32>().ok());
let size = match (it.next(), it.next()) {
(Some(w), Some(h)) if w > 0 && h > 0 => (w, h),
_ => return Err("讀不到這台電腦的螢幕大小".to_string()),
};
shared().lock().unwrap_or_else(|e| e.into_inner()).size = Some(size);
Ok(size)
}
/// Desktop pixels per picture pixel: the bot's pictures are the desktop fitted to 1024 wide,
/// always the same size, so its coordinates map without remembering a screenshot.
fn box_scale() -> Result<(f64, f64), String> {
let (w, h) = box_size()?;
let (fw, fh, _) = crate::computer::fit(w, h);
Ok((w as f64 / fw.max(1) as f64, h as f64 / fh.max(1) as f64))
}
/// A step on the box's screen as a shell script (for `sh -c` inside the box, as the bot user),
/// the text it reads on stdin (typing), and what to tell the bot. Coordinates are picture pixels
/// times `scale` (desktop pixels per picture pixel; 1 for a person's viewer, which shows the
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
/// desktop at full size). `inline` puts typed text inside the script quoted (a batch runs its
/// steps in one exec with one stdin, so steps can't ride on it).
pub fn box_script(name: &str, v: &Value, scale: (f64, f64), inline: bool) -> Result<(String, Option<String>, String), String> {
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
// A control's number from the latest picture: its centre on the desktop.
let by_number = |p: &Value| -> Option<Result<((i64, i64), String), String>> {
let r = match &p["ref"] {
Value::String(r) if !r.trim().is_empty() => r.trim().to_string(),
Value::Number(n) => n.to_string(),
_ => return None,
};
let spots = BOX_SPOTS.lock().unwrap_or_else(|e| e.into_inner()).clone();
Some(spots.ok_or_else(|| "先用 computer_use_screenshot 看一下畫面,再照圖上的編號操作。".to_string()).and_then(|s| s.get(&r)).map(|((x, y), what)| ((x.round() as i64, y.round() as i64), format!("#{}({what})", crate::marks::number_of(&r).unwrap_or(0)))))
};
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
let pt = |p: &Value| -> Result<(i64, i64), String> {
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
if let Some(r) = by_number(p) {
return r.map(|(xy, _)| xy);
}
let (Some(x), Some(y)) = (p["x"].as_f64(), p["y"].as_f64()) else { return Err("請給編號 ref(截圖上的數字),或 x 和 y(截圖上的位置)".into()) };
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
Ok(((x * scale.0).round() as i64, (y * scale.1).round() as i64))
};
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
let at = |p: &Value| match by_number(p) {
Some(Ok((_, what))) => what,
_ => format!("({}, {})", p["x"].as_f64().unwrap_or(0.0), p["y"].as_f64().unwrap_or(0.0)),
};
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
let mods = mods_of(v);
let down: String = mods.iter().map(|m| format!(" keydown {m}")).collect();
let up: String = mods.iter().rev().map(|m| format!(" keyup {m}")).collect();
// Never `mousemove --sync`: it waits forever when the pointer is already there (a second
// click on the same button hung the box for two minutes).
let mv = |(x, y): (i64, i64)| format!("mousemove {x} {y} sleep 0.03");
let wrap = |body: String| -> String {
if mods.is_empty() {
format!("xdotool {body}")
} else {
// Keys held for the step are let go even when the step fails (a stuck ctrl spoils
// every later click and keystroke in the box).
format!("xdotool{down} {body}; rc=$?; xdotool{up}; [ $rc = 0 ]")
}
};
let button = button_no(v["button"].as_str().unwrap_or("left"));
Ok(match name {
"computer_use_click" => {
let n = v["count"].as_u64().unwrap_or(1).clamp(1, 3);
(wrap(format!("{} click --repeat {n} --delay {CLICK_REPEAT_MS} {button}", mv(pt(v)?))), None, format!("已{}{} {}。", if button == "3" { "右鍵" } else { "" }, if n >= 2 { format!("連點 {n} 下") } else { "點".into() }, at(v)))
}
"computer_use_mouse_move" => (format!("xdotool {}", mv(pt(v)?)), None, format!("已把滑鼠移到 {}。", at(v))),
"computer_use_scroll" => {
let amount = crate::computer::scroll_amount(v["amount"].as_i64());
let dir = v["direction"].as_str().unwrap_or("").trim().to_lowercase();
let wheel = match dir.as_str() {
"up" => "4",
"down" => "5",
"left" => "6",
"right" => "7",
_ => return Err(format!("方向「{dir}」要是 up、down、left 或 right")),
};
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
let go = if v["x"].is_null() && v["ref"].is_null() { String::new() } else { format!("{} ", mv(pt(v)?)) };
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
(wrap(format!("{go}click --repeat {amount} --delay 20 {wheel}")), None, format!("已捲動 {dir} ×{amount}。"))
}
"computer_use_drag" => {
// A point that fails to parse is an error, not a silently dropped waypoint.
let raw = v["path"].as_array().ok_or("拖曳要給 path:一個點陣,每個點有數字 x 和 y(用截圖上的座標)")?;
let path: Vec<(i64, i64)> = raw.iter().map(pt).collect::<Result<_, _>>()?;
if path.len() < 2 {
return Err("拖曳至少要兩個點(每個點要有數字 x 和 y,用截圖上的座標)".into());
}
let mut body = format!("{} mousedown {button} sleep 0.04", mv(path[0]));
let steps: Vec<(f64, f64)> = crate::computer::drag_points(&path.iter().map(|(x, y)| (*x as f64, *y as f64)).collect::<Vec<_>>());
for (x, y) in steps {
body.push_str(&format!(" mousemove {} {} sleep 0.008", x.round() as i64, y.round() as i64));
}
body.push_str(&format!(" mouseup {button}"));
(wrap(body), None, format!("已從 {} 拖曳到 {}。", at(&raw[0]), at(&raw[raw.len() - 1])))
}
"computer_use_typing" => {
let text = v["value"].as_str().unwrap_or("").to_string();
let submit = v["submit"].as_bool().unwrap_or(false);
if text.is_empty() && !submit {
return Err("請給要打的字".into());
}
let mut script = String::new();
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
// Into a field by its number: click it first, like a person would.
let field = match by_number(v) {
Some(r) => {
let ((x, y), what) = r?;
script.push_str(&format!("xdotool mousemove {x} {y} sleep 0.03 click 1 sleep 0.15"));
format!("在 {what}")
}
None => String::new(),
};
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
if !text.is_empty() {
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
script.push_str(&format!(" && {}", if inline { type_script_inline(&text) } else { type_script(&text) }));
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
}
if submit {
script.push_str(" && sleep 0.06 && xdotool key --clearmodifiers Return");
}
let script = script.trim_start_matches(" && ").to_string();
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
(script, if inline { None } else { Some(text.clone()).filter(|t| !t.is_empty()) }, format!("已打字({} 個字){field}{}。", text.chars().count(), if submit { ",再按了 return" } else { "" }))
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
}
"computer_use_press_key" => {
let keys = v["key"].as_str().unwrap_or("");
let hold = v["hold_duration_ms"].as_u64().unwrap_or(0).min(5_000);
let mut parts: Vec<String> = vec![];
for k in keys.split_whitespace() {
match combo_to_x(k) {
Ok(c) if hold > 0 => parts.push(format!("keydown {c} sleep {:.3} keyup {c}", hold as f64 / 1000.0)),
Ok(c) => parts.push(format!("key --clearmodifiers {c}")),
// 「@」「?」: a character, typed.
Err(_) if k.chars().count() == 1 => parts.push(format!("type --clearmodifiers -- {}", crate::tools::sh_quote(k))),
Err(e) => return Err(e),
}
}
if parts.is_empty() {
return Err("請給要按的鍵".into());
}
(format!("xdotool {}", parts.join(" sleep 0.02 ")), None, format!("已按 {keys}。"))
}
other => return Err(format!("沒有「{other}」這個工具")),
})
}
/// Typing, as a script reading the text on stdin. Plain ASCII goes key by key; anything else
/// (Chinese, emoji) is pasted, because X keyboards can't type characters they have no key for in
/// every program: both selections hold it (xclip keeps serving after the exec ends, setsid), and
/// terminals paste with shift+Insert, everything else with ctrl+v.
fn type_script(text: &str) -> String {
if text.is_ascii() {
return format!("xdotool type --clearmodifiers --delay {TYPE_DELAY_MS} --file -");
}
"cat > /tmp/anybot-paste.txt && for s in clipboard primary; do setsid xclip -selection $s -i /tmp/anybot-paste.txt </dev/null >/dev/null 2>&1 & done; sleep 0.15; \
case \"$(xprop -id $(xdotool getactivewindow) WM_CLASS 2>/dev/null | tr A-Z a-z)\" in *term*) k=shift+Insert;; *) k=ctrl+v;; esac; xdotool key --clearmodifiers $k"
.to_string()
}
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
/// The same, with the text quoted into the script (batch steps: one stdin for the whole chain).
fn type_script_inline(text: &str) -> String {
let q = crate::tools::sh_quote(text);
if text.is_ascii() {
return format!("xdotool type --clearmodifiers --delay {TYPE_DELAY_MS} -- {q}");
}
format!("printf '%s' {q} > /tmp/anybot-paste.txt && for s in clipboard primary; do setsid xclip -selection $s -i /tmp/anybot-paste.txt </dev/null >/dev/null 2>&1 & done; sleep 0.15; \
case \"$(xprop -id $(xdotool getactivewindow) WM_CLASS 2>/dev/null | tr A-Z a-z)\" in *term*) k=shift+Insert;; *) k=ctrl+v;; esac; xdotool key --clearmodifiers $k")
}
/// Wait for the screen to stop changing before the picture comes back: two matching probes mean
/// it settled. A probe is two small grayscale captures (top-left, center: ~60 ms on the box), so
/// a settled screen takes ~250 ms where every step used to pay a fixed 0.5 s — and a page that
/// is still loading is waited for (up to ~0.9 s) instead of captured mid-change. Plain sleep
/// when `import` is missing.
const SETTLE_SH: &str = "{ probe() { { import -window root -crop 160x140+80+30 -depth 8 pgm:-; import -window root -crop 160x160+560+320 -depth 8 pgm:-; } 2>/dev/null | md5sum; }; if command -v import >/dev/null 2>&1; then a=$(probe); i=0; while [ $i -lt 5 ]; do sleep 0.1; b=$(probe); [ \"$a\" = \"$b\" ] && break; a=$b; i=$((i+1)); done; else sleep 0.35; fi; }";
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
/// Run a step's script in the box; with `picture`, wait for the screen to settle and return it.
fn box_exec(script: &str, stdin: Option<&str>, picture: bool) -> Result<Option<image::RgbImage>, String> {
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
let full = if picture { format!("{{ {script}; }} || exit $?; {SETTLE_SH}; import -window root -quality 90 jpeg:- 2>/dev/null || true") } else { script.to_string() };
// Always an stdin (closed at once when there's no text), as `docker exec -i` expects.
let out = match dk_with_input(&["exec", "-i", "-u", "bot", "-e", "DISPLAY=:1", container(), "sh", "-c", &full], Some(stdin.unwrap_or("").as_bytes()), 30_000) {
Ok(o) => o,
Err(e) if e == "逾時" => return Err("這台電腦太久沒有回應".into()),
Err(e) if e.is_empty() => return Err("這一步沒有成功".into()),
Err(e) => return Err(format!("xdotool:{e}")),
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
};
if !picture || out.is_empty() {
return Ok(None);
}
let img = image::load_from_memory(&out).map(|i| i.to_rgb8()).ok();
if let Some(i) = &img {
shared().lock().unwrap_or_else(|e| e.into_inner()).size = Some(i.dimensions());
}
Ok(img)
}
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
/// The numbered controls of the box's latest look (centres in desktop pixels).
static BOX_SPOTS: Mutex<Option<crate::marks::Spots>> = Mutex::new(None);
/// The latest look's fingerprint (to say when a step changed nothing).
static BOX_LAST: Mutex<Option<u64>> = Mutex::new(None);
/// AT-SPI roles as words.
fn role_words(role: &str) -> String {
match role {
"push button" | "toggle button" => "按鈕",
"link" => "連結",
"entry" => "輸入框",
"password text" => "密碼欄",
"check box" => "勾選框",
"radio button" => "選項",
"menu item" | "check menu item" | "radio menu item" => "選單項目",
"menu" => "選單",
"page tab" | "tab" => "分頁",
"list item" | "tree item" => "項目",
"combo box" => "下拉選單",
"spin button" => "數字欄",
"slider" => "滑桿",
"heading" => "標題",
"image" | "icon" => "圖片",
"table cell" => "格子",
"column header" | "row header" => "欄名",
"dialog" | "alert" => "對話框",
"text" | "paragraph" | "section" | "document web" => "文字",
_ => "文字",
}
.to_string()
}
/// The box's controls from its accessibility tree (box/a11y's JSON).
pub fn elements_of(v: &Value) -> Vec<crate::marks::Element> {
v["elements"]
.as_array()
.into_iter()
.flatten()
.map(|e| {
let role = e["role"].as_str().unwrap_or("");
let active = e["active"].as_bool().unwrap_or(false);
let kind = if active && matches!(role, "text" | "paragraph" | "section" | "document web") { "輸入框".to_string() } else { role_words(role) };
let flags = e["flags"].as_array().into_iter().flatten().filter_map(|f| match f.as_str()? {
"focused" => Some("游標在這"),
"checked" => Some("已勾"),
"selected" => Some("已選"),
"disabled" => Some("停用"),
_ => None,
}).collect();
crate::marks::Element {
kind,
name: e["name"].as_str().unwrap_or("").to_string(),
value: e["value"].as_str().unwrap_or("").to_string(),
x: e["x"].as_f64().unwrap_or(0.0),
y: e["y"].as_f64().unwrap_or(0.0),
w: e["w"].as_f64().unwrap_or(0.0),
h: e["h"].as_f64().unwrap_or(0.0),
active,
flags,
window: e["window"].as_str().unwrap_or("").to_string(),
}
})
.collect()
}
/// Do a step's script (if any), wait for the screen to settle, and look: the picture with the
/// controls numbered and the list, in one `docker exec`. After a step it also says when nothing
/// changed, so the bot doesn't click the same empty place again and again.
fn box_look(script: Option<(&str, Option<&str>)>, after: Option<String>) -> Result<(String, Option<String>), String> {
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
let t_all = Instant::now();
// The picture is captured by a background `import` while the controls are read (the a11y
// read is the slow part on web pages); a JPEG costs less to make and to ship than a PNG.
let look = "( import -window root -quality 90 jpeg:- 2>/dev/null > /tmp/anybot-look.jpg & ip=$!; \
anybot-a11y --budget 3 > /tmp/anybot-look.json 2>/dev/null || echo '{\"elements\":[]}' > /tmp/anybot-look.json; \
wc -c < /tmp/anybot-look.json; cat /tmp/anybot-look.json; wait $ip; cat /tmp/anybot-look.jpg )";
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
let full = match script {
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
Some((s, _)) => format!("{{ {s}; }} || exit $?; {SETTLE_SH}; {look}"),
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
None => look.to_string(),
};
let input = script.and_then(|(_, i)| i).unwrap_or("");
let out = match dk_with_input(&["exec", "-i", "-u", "bot", "-e", "DISPLAY=:1", container(), "sh", "-c", &full], Some(input.as_bytes()), 30_000) {
Ok(o) => o,
Err(e) if e == "逾時" => return Err("這台電腦太久沒有回應".into()),
Err(e) if e.is_empty() => return Err("這一步沒有成功".into()),
Err(e) => return Err(format!("xdotool:{e}")),
};
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
let exec_ms = t_all.elapsed().as_millis() as u64;
// "<bytes of the list>\n<list JSON><JPEG>"
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
let nl = out.iter().position(|b| *b == b'\n').ok_or("這台電腦沒有回傳畫面")?;
let n: usize = String::from_utf8_lossy(&out[..nl]).trim().parse().map_err(|_| "這台電腦回傳的畫面讀不懂")?;
let json_end = (nl + 1 + n).min(out.len());
let tree: Value = serde_json::from_slice(&out[nl + 1..json_end]).unwrap_or(Value::Null);
let img = image::load_from_memory(&out[json_end..]).map_err(|e| format!("截圖讀不出來:{e}"))?.to_rgb8();
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
let (w, h) = img.dimensions();
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
shared().lock().unwrap_or_else(|e| e.into_inner()).size = Some((w, h));
let els = elements_of(&tree);
let (numbered, lines, spots) = crate::marks::number(&els, false);
*BOX_SPOTS.lock().unwrap_or_else(|e| e.into_inner()) = Some(spots);
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
let (fw, fh, resize) = crate::computer::fit(w, h);
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
let mut pic = if resize { image::imageops::resize(&img, fw, fh, image::imageops::FilterType::Triangle) } else { img };
let fp = crate::marks::fingerprint(&els) ^ crate::marks::picture_fingerprint(&pic).rotate_left(1);
crate::marks::draw(&mut pic, &numbered, (w as f64 / fw.max(1) as f64, h as f64 / fh.max(1) as f64));
let jpeg = crate::computer::encode_jpeg(&pic).ok_or("截圖太大了,沒辦法送出")?;
let mut text = after.clone().map(|n| format!("{n}\n")).unwrap_or_default();
text.push_str(&format!("螢幕截圖({fw}×{fh},Linux 桌面)。"));
let unchanged = {
let mut last = BOX_LAST.lock().unwrap_or_else(|e| e.into_inner());
let same = after.is_some() && *last == Some(fp);
*last = Some(fp);
same
};
if unchanged {
text.push_str("\n⚠️ 畫面和上一步完全一樣:這一步沒有點到東西。不要再做同一步;換一個編號、換一個方法(網頁用 browser_* 工具,或直接開網址),或用 hand_over_computer 請使用者幫忙。");
}
if lines.is_empty() {
text.push_str("\n畫面上沒有可以照編號點的東西:只好照圖上的位置點 x、y。");
} else {
text.push_str(&format!("\n可以操作的東西(圖上同樣的編號,用 ref 點):\n{}", lines.join("\n")));
}
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
// Step timing (for the 電腦 card): how long the box's side took vs. the whole step.
{
let total_ms = t_all.elapsed().as_millis() as u64;
let mut s = STEP_TIMES.lock().unwrap_or_else(|e| e.into_inner());
s.push((exec_ms, total_ms));
let n = s.len();
if n > 40 {
s.drain(..n - 40);
}
}
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
Ok((text, Some(base64::engine::general_purpose::STANDARD.encode(jpeg))))
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
}
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
/// The box's latest steps, timed: (count, average docker-exec ms, average whole-step ms).
/// Whole step = exec (xdotool + settle + 控件 + 截圖) + 這裡的編號繪製和壓縮.
pub fn step_stats() -> (usize, u64, u64) {
let s = STEP_TIMES.lock().unwrap_or_else(|e| e.into_inner());
if s.is_empty() {
return (0, 0, 0);
}
let n = s.len() as u64;
(s.len(), s.iter().map(|(a, _)| *a).sum::<u64>() / n, s.iter().map(|(_, b)| *b).sum::<u64>() / n)
}
static STEP_TIMES: Mutex<Vec<(u64, u64)>> = Mutex::new(Vec::new());
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
/// Programs people name, as the box knows them.
fn box_program(app: &str) -> Option<&'static str> {
let a = app.trim().to_lowercase();
if ["chrome", "chromium", "google chrome", "browser", "瀏覽器", "網頁瀏覽器"].iter().any(|n| a == *n) {
return Some("chromium");
}
if ["terminal", "終端機", "xterm", "shell", "命令列"].iter().any(|n| a == *n) {
return Some("xterm");
}
None
}
/// Open a program on the box, or bring its window to the front. The browser is AnyBot's
/// browser there (the one browser_* drive), never a second copy fighting over its profile.
fn open_on_box(app: &str) -> Result<String, String> {
let name = app.trim();
if name.is_empty() {
return Err("請給要打開的程式名稱".into());
}
let prog = match box_program(name) {
Some(p) => p.to_string(),
None if name.chars().all(|c| c.is_ascii_alphanumeric() || "._-".contains(c)) => name.to_lowercase(),
None => return Err(format!("這台是 Linux 桌面,沒有「{name}」。可以打開的例如 chromium(瀏覽器)、xterm(終端機),或用指令工具安裝。")),
};
if prog == "chromium" {
browser_port()?;
}
let q = crate::tools::sh_quote(&prog);
// Front if it has a window; else start it and wait (up to 8 s) for one.
let script = format!(
"w=$(xdotool search --onlyvisible --classname {q} 2>/dev/null | tail -1); \
if [ -z \"$w\" ]; then command -v {q} >/dev/null || {{ echo \"沒有 {prog} 這個程式\" >&2; exit 3; }}; (setsid {q} >/dev/null 2>&1 &); \
for i in $(seq 1 40); do sleep 0.2; w=$(xdotool search --onlyvisible --classname {q} 2>/dev/null | tail -1); [ -n \"$w\" ] && break; done; fi; \
[ -n \"$w\" ] && xdotool windowactivate $w 2>/dev/null; true"
);
box_exec(&script, None, false)?;
Ok(format!("已打開「{prog}」,放在最前面。"))
}
/// Run a computer_use tool on the box. Returns (text, screenshot as base64 JPEG).
pub fn computer_run(name: &str, v: &Value) -> Result<(String, Option<String>), String> {
ensure_running()?;
let _busy = Busy::new(true);
match name {
"computer_use_check_permissions" => return Ok(("這台電腦不需要另外的權限:可以截圖,也可以操作滑鼠和鍵盤。".into(), None)),
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
"computer_use_screenshot" => return box_look(None, None),
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
_ => {}
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
if HELD_BY_PERSON.load(Ordering::SeqCst) || crate::vnc::person_active() {
return Err("使用者正在自己操作這台電腦。請等他操作完(停下幾秒後)再繼續,或先截圖看看畫面。".into());
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
}
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
if name == "computer_use_open_app" {
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
let note = open_on_box(v["app"].as_str().unwrap_or(""))?;
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
return box_look(Some(("true", None)), Some(note));
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
}
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
if name == "computer_use_batch" {
return box_batch(v, box_scale()?);
}
let (script, stdin, note) = box_script(name, v, box_scale()?, false)?;
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
box_look(Some((&script, stdin.as_deref())), Some(note))
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
}
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
const BATCH_MARK: &str = "/tmp/anybot-batch.done";
/// Do a planned sequence of steps on the box in one round trip: the steps' scripts chained with
/// `&&` (so the chain stops at the first failure), one marker line per step that ran, and one
/// picture at the end — like computer_batch everywhere else, and one `docker exec` for all of
/// them instead of one per step.
fn box_batch(v: &Value, scale: (f64, f64)) -> Result<(String, Option<String>), String> {
let acts = v["actions"].as_array().ok_or("computer_use_batch 要給 actions:動作陣列,每個有 action(click、typing、press_key、scroll、mouse_move、drag)和它的參數")?;
if acts.is_empty() {
return Err("actions 是空的".into());
}
if acts.len() > crate::computer::MAX_BATCH {
return Err(format!("一次最多 {} 個動作({} 個太多了)", crate::computer::MAX_BATCH, acts.len()));
}
let mut parts = Vec::new();
let mut notes = Vec::new();
for (i, a) in acts.iter().enumerate() {
let name = crate::computer::action_tool(a["action"].as_str().unwrap_or(""))?;
let (script, _, note) = box_script(name, a, scale, true)?;
parts.push(format!("{{ {script} && echo x >> {BATCH_MARK}; }}"));
notes.push(format!("{}. {note}", i + 1));
}
box_exec(&format!("rm -f {BATCH_MARK}; {} ; true", parts.join(" && ")), None, false)?;
let done = dk(&["exec", container(), "sh", "-c", &format!("wc -l < {BATCH_MARK} 2>/dev/null || echo 0")])
.ok()
.and_then(|s| s.trim().parse::<usize>().ok())
.unwrap_or(0)
.min(acts.len());
let _ = dk(&["exec", container(), "sh", "-c", &format!("rm -f {BATCH_MARK}")]);
let note = if done == acts.len() {
format!("已依序做完 {done} 步:\n{}", notes.join("\n"))
} else {
let ran: Vec<&str> = notes.iter().take(done).map(String::as_str).collect();
format!("做完 {done}/{} 步,第 {} 步沒有成功(後面沒有做):\n{}", acts.len(), done + 1, ran.join("\n"))
};
box_look(Some(("true", None)), Some(note))
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
// ---------- text a person types in a viewer ----------
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
/// Text from a person's viewer (any language): typed or pasted into the window in front, like
/// the bot's typing. (Their mouse and keys go through VNC itself.)
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
pub fn viewer_text(text: &str) {
if !text.is_empty() {
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
let _ = box_exec(&type_script(text), Some(text), false);
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
}
touch();
}
fn key_to_x(name: &str) -> Option<String> {
let n = name.trim().to_lowercase();
let s = match n.as_str() {
"return" | "enter" => "Return",
"tab" => "Tab",
"space" => "space",
"delete" | "backspace" => "BackSpace",
"forwarddelete" | "del" => "Delete",
"escape" | "esc" => "Escape",
"home" => "Home",
"end" => "End",
"pageup" => "Prior",
"pagedown" => "Next",
"up" | "arrowup" => "Up",
"down" | "arrowdown" => "Down",
"left" | "arrowleft" => "Left",
"right" | "arrowright" => "Right",
"-" | "minus" => "minus",
"=" | "equal" => "equal",
"[" => "bracketleft",
"]" => "bracketright",
";" => "semicolon",
"'" => "apostrophe",
"," | "comma" => "comma",
"." | "period" => "period",
"/" | "slash" => "slash",
"\\" | "backslash" => "backslash",
"`" | "grave" => "grave",
f if f.starts_with('f') && f[1..].parse::<u8>().map(|x| (1..=12).contains(&x)).unwrap_or(false) => return Some(f.to_uppercase()),
c if c.chars().count() == 1 && c.chars().all(|ch| ch.is_ascii_alphanumeric()) => return Some(c.to_string()),
_ => return None,
};
Some(s.into())
}
fn modifier_to_x(m: &str) -> Option<&'static str> {
// On Linux the shortcuts people mean by cmd (copy, paste, select all) use ctrl.
match m.trim().to_lowercase().as_str() {
"cmd" | "command" | "meta" | "control" | "ctrl" => Some("ctrl"),
"shift" => Some("shift"),
"option" | "alt" | "opt" => Some("alt"),
"super" | "win" | "windows" => Some("super"),
_ => None,
}
}
/// "cmd+shift+t" → "ctrl+shift+t"
pub fn combo_to_x(combo: &str) -> Result<String, String> {
let parts: Vec<&str> = combo.split('+').map(str::trim).filter(|p| !p.is_empty()).collect();
let Some((key, mods)) = parts.split_last() else { return Err("請給要按的鍵".into()) };
let mut out: Vec<String> = vec![];
for m in mods {
out.push(modifier_to_x(m).ok_or_else(|| format!("不認得「{m}」這個修飾鍵"))?.to_string());
}
out.push(key_to_x(key).ok_or_else(|| format!("不認得「{key}」這個鍵"))?);
Ok(out.join("+"))
}
fn mods_of(v: &Value) -> Vec<String> {
v["modifier_keys"].as_array().map(|a| a.iter().filter_map(|m| m.as_str().and_then(modifier_to_x)).map(String::from).collect()).unwrap_or_default()
}
fn button_no(b: &str) -> &'static str {
match b {
"right" => "3",
"middle" => "2",
_ => "1",
}
}
// ---------- the browser viewer (noVNC) ----------
fn published_port() -> u16 {
dk(&["port", container(), &NOVNC_PORT.to_string()])
.ok()
.and_then(|s| s.lines().next().and_then(|l| l.rsplit(':').next()).and_then(|p| p.trim().parse().ok()))
.unwrap_or(HOST_PORT)
}
/// The noVNC address on this Mac (through an SSH tunnel when the box is on another host).
pub fn viewer_url(control: bool) -> Result<String, String> {
ensure_running()?;
let port = published_port();
let host = docker_host();
let local_port = if let Some(target) = host.strip_prefix("ssh://") {
let mut s = shared().lock().unwrap();
let alive = s.tunnel.as_mut().map(|c| matches!(c.try_wait(), Ok(None))).unwrap_or(false);
if !alive {
let lp = free_port(HOST_PORT + 100);
let child = Command::new("ssh")
.env("PATH", tool_path())
.args(["-N", "-o", "BatchMode=yes", "-o", "ExitOnForwardFailure=yes", "-L", &format!("{lp}:127.0.0.1:{port}"), target])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.map_err(|e| format!("開不了 SSH 通道:{e}"))?;
s.tunnel = Some(child);
s.host_port = lp;
drop(s);
std::thread::sleep(Duration::from_millis(800));
lp
} else {
s.host_port
}
} else {
port
};
// noVNC signs in with the box's VNC password (x11vnc refuses the keyless login now).
Ok(format!("http://127.0.0.1:{local_port}/vnc.html?autoconnect=1&resize=scale&reconnect=1&show_dot=1&password={}{}", vnc_password(), if control { "" } else { "&view_only=1" }))
}
pub fn shutdown() {
if let Some(mut t) = shared().lock().unwrap().tunnel.take() {
let _ = t.kill();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn secret_names_follow_the_original_rule() {
assert!(valid_secret_name("CRM_API_TOKEN"));
assert!(valid_secret_name("A1"));
assert!(!valid_secret_name("crm_token"));
assert!(!valid_secret_name("1ABC"));
assert!(!valid_secret_name("_ABC"));
assert!(!valid_secret_name("AB-C"));
}
#[test]
fn shortcuts_become_x_keys() {
assert_eq!(combo_to_x("cmd+c").unwrap(), "ctrl+c");
assert_eq!(combo_to_x("cmd+shift+t").unwrap(), "ctrl+shift+t");
assert_eq!(combo_to_x("return").unwrap(), "Return");
assert_eq!(combo_to_x("option+f4").unwrap(), "alt+F4");
assert!(combo_to_x("hyper+x").is_err());
}
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
#[test]
fn a_step_is_one_script() {
let s = (1.25, 1.25);
// Picture pixels → desktop pixels; never `mousemove --sync` (it hangs when the pointer is
// already there).
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
let (sc, input, note) = box_script("computer_use_click", &serde_json::json!({"x": 512, "y": 320}), s, false).unwrap();
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
assert_eq!(sc, "xdotool mousemove 640 400 sleep 0.03 click --repeat 1 --delay 80 1");
assert!(input.is_none() && note.contains("(512, 320)"));
// Keys held for a step are let go even when it fails.
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
let (sc, _, _) = box_script("computer_use_click", &serde_json::json!({"x": 8, "y": 8, "modifier_keys": ["cmd", "shift"], "count": 2}), s, false).unwrap();
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
assert!(sc.starts_with("xdotool keydown ctrl keydown shift mousemove 10 10") && sc.ends_with("rc=$?; xdotool keyup shift keyup ctrl; [ $rc = 0 ]"), "{sc}");
assert!(!sc.contains("--sync"));
// Text goes in on stdin (no quoting to get wrong); return after it when asked.
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "echo 'hi' && ls", "submit": true}), s, false).unwrap();
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
assert_eq!(input.as_deref(), Some("echo 'hi' && ls"));
assert!(sc.starts_with("xdotool type --clearmodifiers --delay 12 --file -") && sc.ends_with("xdotool key --clearmodifiers Return"), "{sc}");
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "中文"}), s, false).unwrap();
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
assert!(sc.contains("xclip") && input.as_deref() == Some("中文"));
// Keys by name, a character typed, several in order.
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
let (sc, _, _) = box_script("computer_use_press_key", &serde_json::json!({"key": "slash @ cmd+a"}), s, false).unwrap();
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
assert_eq!(sc, "xdotool key --clearmodifiers slash sleep 0.02 type --clearmodifiers -- '@' sleep 0.02 key --clearmodifiers ctrl+a");
// A drag walks its path in small steps and lets go at the end.
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
let (sc, _, _) = box_script("computer_use_drag", &serde_json::json!({"path": [{"x": 0, "y": 0}, {"x": 48, "y": 0}]}), s, false).unwrap();
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
assert!(sc.starts_with("xdotool mousemove 0 0 sleep 0.03 mousedown 1") && sc.contains("mousemove 60 0") && sc.ends_with("mouseup 1"), "{sc}");
// No app numbers on this desktop: said, not guessed.
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
assert!(box_script("computer_use_click", &serde_json::json!({"ref": "e3"}), s, false).is_err());
assert!(box_script("computer_use_scroll", &serde_json::json!({"direction": "sideways"}), s, false).is_err());
}
#[test]
fn batch_steps_carry_their_text_inline() {
let s = (1.0, 1.0);
// A batch runs its steps in one exec with one stdin: the text rides in the script, quoted.
let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "it's a 'test'"}), s, true).unwrap();
assert!(input.is_none(), "no stdin for a batch step");
assert!(sc.starts_with("xdotool type --clearmodifiers --delay 12 -- 'it'\\''s a '\\''test'\\'''"), "{sc}");
// Chinese in a batch: the paste path, text written inline.
let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "中文", "submit": true}), s, true).unwrap();
assert!(input.is_none() && sc.contains("printf '%s' '中文' > /tmp/anybot-paste.txt") && sc.ends_with("xdotool key --clearmodifiers Return"), "{sc}");
// Batch action names, and the ones a batch must not carry.
assert_eq!(crate::computer::action_tool("Click").unwrap(), "computer_use_click");
assert_eq!(crate::computer::action_tool("typing").unwrap(), "computer_use_typing");
assert!(crate::computer::action_tool("screenshot").is_err());
assert!(crate::computer::action_tool("open_app").is_err());
assert!(crate::computer::action_tool("").is_err());
}
#[test]
fn snapshot_names_carry_their_time() {
// Names sort by time, and the stamp is readable from the name (copies lose file dates).
let p = std::path::Path::new("/x/snapshot-2026-11-05-123456.tar.gz");
let t = snapshot_stamp(p).expect("stamp parses");
assert_eq!(t.format("%Y-%m-%d %H:%M:%S").to_string(), "2026-11-05 12:34:56");
let legacy = std::path::Path::new("/x/snapshot-2026-10-01-000000-legacy.tar.gz");
assert_eq!(snapshot_stamp(legacy).expect("legacy parses").format("%Y-%m-%d").to_string(), "2026-10-01");
assert!(snapshot_stamp(std::path::Path::new("/x/notes.txt")).is_none());
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
}
Bots click by number: every control on the screen is boxed and numbered, on every computer A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
/// A box made from this image (`ANYBOT_BOX_NAME=<test box> cargo test --release -- --ignored
/// box_numbers_are_exact --nocapture`, with a scratch ANYBOT_DATA_DIR). A page like Threads'
/// (small icon links down the side): every link, field and button gets a number on the
/// picture, a click by number lands on it, typing goes into the numbered field, and clicking
/// an empty spot twice is called out.
#[test]
#[ignore]
fn box_numbers_are_exact() {
ensure_running().expect("box starts");
let cancel = AtomicBool::new(false);
let page = r##"<!doctype html><meta charset=utf-8><title>start</title><body style="margin:0;font:16px sans-serif">
<nav style="position:fixed;left:0;top:0;width:64px;height:100%;background:#f3f3f3">
<a href="#h" aria-label="Home" onclick="document.title='Home';return false" style="display:block;margin:24px 16px;width:28px;height:28px;background:#888;border-radius:6px"></a>
<a href="#p" aria-label="Profile" onclick="document.title='Profile';return false" style="display:block;margin:24px 16px;width:28px;height:28px;background:#888;border-radius:6px"></a>
</nav><main style="margin-left:96px;padding:24px"><h1>測試頁</h1>
<input id=q aria-label="搜尋" style="width:280px;font-size:16px"> <button onclick="document.title='Go:'+document.getElementById('q').value">搜尋</button></main>"##;
dk_with_input(&["exec", "-i", "-u", "bot", container(), "sh", "-c", "mkdir -p /tmp/anybot-page && cat > /tmp/anybot-page/index.html && (cd /tmp/anybot-page && setsid python3 -m http.server 8765 --bind 127.0.0.1 >/dev/null 2>&1 &) ; sleep 0.5"], Some(page.as_bytes()), 10_000).expect("page served");
let opened = crate::botbrowser::in_box(|| crate::web::run_full("browser_open", r#"{"url":"http://127.0.0.1:8765/"}"#, &cancel));
assert!(opened.1, "{}", opened.0);
std::thread::sleep(Duration::from_millis(1500));
let title = || crate::botbrowser::in_box(|| crate::botbrowser::eval_current(None, "document.title")).map(|(_, v)| v.as_str().unwrap_or("").to_string()).unwrap_or_default();
let t0 = Instant::now();
let (txt, img) = computer_run("computer_use_screenshot", &serde_json::json!({})).unwrap();
println!("look: {} ms\n{txt}", t0.elapsed().as_millis());
if let Some(b) = &img {
let _ = std::fs::write(std::env::temp_dir().join("anybot-numbers.jpg"), base64::engine::general_purpose::STANDARD.decode(b).unwrap_or_default());
}
let number = |what: &str| -> String {
txt.lines().find(|l| l.contains(what)).and_then(|l| l.trim().strip_prefix('[')).and_then(|l| l.split(']').next()).unwrap_or_else(|| panic!("no number for {what} in:\n{txt}")).to_string()
};
// The small icon link down the side, by number.
let profile = number("連結「Profile」");
computer_run("computer_use_click", &serde_json::json!({"ref": profile})).unwrap();
assert_eq!(title(), "Profile");
// Typing into the numbered field (Chinese), then the numbered button.
let field = number("輸入框「搜尋」");
computer_run("computer_use_typing", &serde_json::json!({"ref": field, "value": "貓咪 cats"})).unwrap();
let (after, _) = computer_run("computer_use_click", &serde_json::json!({"ref": number("按鈕「搜尋」")})).unwrap();
assert_eq!(title(), "Go:貓咪 cats");
assert!(!after.contains("完全一樣"), "{after}");
// An empty spot, twice: the second time it says nothing changed.
computer_run("computer_use_click", &serde_json::json!({"x": 700, "y": 520})).unwrap();
let (again, _) = computer_run("computer_use_click", &serde_json::json!({"x": 700, "y": 520})).unwrap();
assert!(again.contains("畫面和上一步完全一樣"), "{again}");
}
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
/// Uses the real Docker: `cargo test --release -- --ignored box_screen_is_exact --nocapture`.
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
/// The bot's clicks land where its picture says, the same spot twice doesn't hang, typing (ASCII, Chinese, a key by name) reaches the app,
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
/// and every step comes back with the screen after it.
#[test]
#[ignore]
fn box_screen_is_exact() {
ensure_running().expect("box starts");
let pointer = || {
let out = dk(&["exec", "-u", "bot", "-e", "DISPLAY=:1", container(), "xdotool", "getmouselocation"]).unwrap();
let num = |k: &str| out.split_whitespace().find_map(|w| w.strip_prefix(&format!("{k}:")).and_then(|v| v.parse::<i64>().ok())).unwrap();
(num("x"), num("y"))
};
let (w, h) = box_size().unwrap();
let (fw, fh, _) = crate::computer::fit(w, h);
println!("desktop {w}×{h}, picture {fw}×{fh}");
// A move, in the bot's picture pixels, with no screenshot taken first.
shared().lock().unwrap().size = None;
let (txt, img) = computer_run("computer_use_mouse_move", &serde_json::json!({"x": fw / 2, "y": fh / 2})).unwrap();
assert!(img.is_some(), "a picture comes back: {txt}");
assert_eq!(pointer(), ((w / 2) as i64, (h / 2) as i64));
// The same spot twice: quick, not a two-minute hang.
let t = Instant::now();
computer_run("computer_use_click", &serde_json::json!({"x": 40, "y": 40})).unwrap();
computer_run("computer_use_click", &serde_json::json!({"x": 40, "y": 40})).unwrap();
println!("two clicks with pictures: {} ms", t.elapsed().as_millis());
assert!(t.elapsed() < Duration::from_secs(6));
assert_eq!(pointer(), ((40.0 * w as f64 / fw as f64).round() as i64, (40.0 * h as f64 / fh as f64).round() as i64));
// Typing reaches the app in front: a fresh terminal, ASCII then Chinese, a key by name, return.
let _ = dk(&["exec", "-u", "bot", container(), "rm", "-f", "/home/bot/exact.txt"]);
let (txt, img) = computer_run("computer_use_open_app", &serde_json::json!({"app": "終端機"})).unwrap();
println!("{txt}");
assert!(img.is_some());
computer_run("computer_use_typing", &serde_json::json!({"value": "echo 'a-b' "})).unwrap();
computer_run("computer_use_typing", &serde_json::json!({"value": "中文"})).unwrap();
computer_run("computer_use_press_key", &serde_json::json!({"key": "space slash"})).unwrap();
let (txt, img) = computer_run("computer_use_typing", &serde_json::json!({"value": " > /home/bot/exact.txt", "submit": true})).unwrap();
assert!(img.is_some() && txt.contains("return"));
std::thread::sleep(Duration::from_millis(500));
let out = dk(&["exec", "-u", "bot", container(), "cat", "/home/bot/exact.txt"]).unwrap_or_default();
println!("typed: {out}");
assert_eq!(out.trim(), "a-b 中文 /");
let _ = box_exec("xdotool key --clearmodifiers ctrl+d", None, false);
}
Bots do a planned sequence in one step; the box's screen settles by what it shows; no login is ever the only copy - computer_use_batch: up to 8 planned actions run in one docker exec (steps chained with && and a done-marker each), the first failure stops the rest, one picture comes back at the end. Typed text rides inside the script quoted (one stdin for the chain). The prompt tells the model when to use it (a form whose fields are all visible) and when not to (step 2 needs step 1's screen). - The box waits for the screen to stop changing, not the clock: two matching small probes (~60 ms each) mean settled — ~160 ms on a still screen where every step paid a fixed 500 ms, and a page mid-load is waited for (up to ~0.9 s) instead of captured mid-change. The picture is JPEG, and the a11y read runs in parallel with the capture (look ~307 ms; two steps with pictures 1102 ms). (SETTLE_SH is a group command: dash needs ';' after it — found by box_screen_is_exact.) - The home volume (every bot's logins) is never one command from loss: snapshots keep generations (last 7 days, then one per week; the old single snapshot migrates by file date), Reset and Restore save the running box first and refuse to wipe if saving fails, a box built on an empty volume re-fills itself from the newest snapshot, and a sleeping holder container keeps the volume attached so `docker volume prune` can't take it. make doctor shows volume and snapshot state; the 電腦 card shows the box's average step times and says resets save first. - box_batch_in_one_round_trip (ignored): three typed steps, ASCII and Chinese, land in one trip. On the real box after upgrading v3→v5: box_update_keeps_files (files survive, snapshot 2 代), box_numbers_are_exact, box_screen_is_exact, box_end_to_end, box_traffic_via_mac pass; 155 unit tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 01:47:37 +00:00
/// Uses the real Docker: `cargo test --release -- --ignored box_batch_in_one_round_trip --nocapture`.
/// A planned sequence runs in one round trip: ASCII and Chinese typing and a submit land in the
/// front app, the note lists every step, and one picture comes back.
#[test]
#[ignore]
fn box_batch_in_one_round_trip() {
ensure_running().expect("box starts");
let _ = dk(&["exec", "-u", "bot", container(), "rm", "-f", "/home/bot/batch.txt"]);
let (txt, img) = computer_run("computer_use_open_app", &serde_json::json!({"app": "終端機"})).unwrap();
assert!(img.is_some(), "{txt}");
let t = Instant::now();
let (txt, img) = computer_run(
"computer_use_batch",
&serde_json::json!({"actions": [
{"action": "typing", "value": "echo batch-"},
{"action": "typing", "value": "中文"},
{"action": "typing", "value": "ok > /home/bot/batch.txt", "submit": true}
]}),
)
.unwrap();
println!("batch of 3: {} ms\n{txt}", t.elapsed().as_millis());
assert!(img.is_some() && txt.contains("已依序做完 3 步"), "{txt}");
std::thread::sleep(Duration::from_millis(600));
let out = dk(&["exec", "-u", "bot", container(), "cat", "/home/bot/batch.txt"]).unwrap_or_default();
println!("typed: {out}");
assert_eq!(out.trim(), "batch-中文ok");
// Too many, empty: said before anything runs.
assert!(computer_run("computer_use_batch", &serde_json::json!({"actions": []})).is_err());
let many: Vec<_> = (0..9).map(|_| serde_json::json!({"action": "press_key", "key": "f15"})).collect();
assert!(computer_run("computer_use_batch", &serde_json::json!({"actions": many})).is_err());
let _ = box_exec("xdotool key --clearmodifiers ctrl+d", None, false);
}
/// Uses the real Docker: `cargo test --release -- --ignored box_end_to_end --nocapture`.
#[test]
#[ignore]
fn box_end_to_end() {
use crate::tools::{run_on, Place, RunIds, Target};
let cancel = AtomicBool::new(false);
let ids = RunIds::default();
ensure_running().expect("box starts");
assert!(run_state().is_running());
let t = Target { name: "Bot 的電腦".into(), place: Place::Box { dir: bot_dir(999) } };
let (out, ok) = run_on("shell", r#"{"command":"pwd && whoami && echo $DISPLAY"}"#, &t, &cancel, &ids);
println!("shell: {out}");
assert!(ok && out.contains("/home/bot/bot-999") && out.contains("bot") && out.contains(":1"));
let (out, ok) = run_on("edit_file", r#"{"path":"notes/hello.txt","old_text":"","new_text":"第一行\nhello world\n"}"#, &t, &cancel, &ids);
assert!(ok, "{out}");
let (out, ok) = run_on("edit_file", r#"{"path":"notes/hello.txt","old_text":"hello world","new_text":"你好 世界"}"#, &t, &cancel, &ids);
assert!(ok, "{out}");
let (out, ok) = run_on("read_file", r#"{"path":"notes/hello.txt"}"#, &t, &cancel, &ids);
println!("read: {out}");
assert!(ok && out.contains("你好 世界") && !out.contains("hello world"));
let (out, ok) = run_on("search", r#"{"pattern":"世界","path":"."}"#, &t, &cancel, &ids);
assert!(ok && out.contains("hello.txt"), "{out}");
// secrets reach commands, never shown by the app
set_secret("ANYBOT_TEST_TOKEN", "s3cret-value").unwrap();
let (out, ok) = run_on("shell", r#"{"command":"test \"$ANYBOT_TEST_TOKEN\" = s3cret-value && echo has-secret"}"#, &t, &cancel, &ids);
remove_secret("ANYBOT_TEST_TOKEN");
assert!(ok && out.contains("has-secret"), "{out}");
// background command takes input
let (out, ok) = run_on("shell", r#"{"command":"read -p 'name? ' n; echo got:$n","timeout_ms":1500}"#, &t, &cancel, &ids);
assert!(ok, "{out}");
let id: u64 = out.split('#').nth(1).and_then(|s| s.split(|c: char| !c.is_ascii_digit()).next()).and_then(|s| s.parse().ok()).expect("terminal id");
let (out, ok) = run_on("write_terminal", &format!(r#"{{"id":{id},"input":"daniel\n"}}"#), &t, &cancel, &ids);
println!("write_terminal: {out}");
assert!(ok && out.contains("got:daniel"), "{out}");
// screen
let (txt, img) = computer_run("computer_use_screenshot", &serde_json::json!({})).unwrap();
println!("{txt}");
assert!(img.is_some());
run_on("shell", r#"{"command":"(xterm >/dev/null 2>&1 &) ; sleep 1.5"}"#, &t, &cancel, &ids);
// Click the middle of that window, wherever it opened (other windows — a browser left
// open — may cover the corner), in the screenshot's pixels. (Its shell renames it, so it
// is found by its class: the newest xterm.)
let (geo, _) = run_on("shell", r#"{"command":"W=$(DISPLAY=:1 xdotool search --onlyvisible --class xterm | tail -1); DISPLAY=:1 xdotool getwindowgeometry --shell $W | tail -n +2 | head -4 | tr '\n' ' '"}"#, &t, &cancel, &ids);
let num = |k: &str| geo.split_whitespace().find_map(|w| w.strip_prefix(&format!("{k}=")).and_then(|v| v.parse::<f64>().ok())).unwrap_or(0.0);
let (cx, cy) = (num("X") + num("WIDTH") / 2.0, num("Y") + num("HEIGHT") / 2.0);
let k = 1024.0 / 1280.0;
computer_run("computer_use_click", &serde_json::json!({"x": cx * k, "y": cy * k})).unwrap();
computer_run("computer_use_typing", &serde_json::json!({"value": "echo 中文輸入OK > /home/bot/typed.txt"})).unwrap();
computer_run("computer_use_press_key", &serde_json::json!({"key": "return"})).unwrap();
std::thread::sleep(Duration::from_millis(800));
let (out, _) = run_on("shell", r#"{"command":"cat /home/bot/typed.txt"}"#, &t, &cancel, &ids);
println!("typed: {out}");
assert!(out.contains("中文輸入OK"), "{out}");
// snapshot (with the browser and its caches in the home folder), then restore — which
// rebuilds the whole box, so only on a box meant for it (ANYBOT_TEST_RESTORE=1).
let size = save_snapshot().expect("snapshot saves even while the browser writes");
println!("snapshot: {}", human(size));
assert!(size > 0);
if std::env::var("ANYBOT_TEST_RESTORE").as_deref() != Ok("1") {
let _ = run_on("shell", r#"{"command":"rm -rf /home/bot/bot-999 /home/bot/typed.txt"}"#, &t, &cancel, &ids);
return;
}
run_on("shell", r#"{"command":"rm -rf notes"}"#, &t, &cancel, &ids);
rebuild(Rebuild::Restore).unwrap();
let start = Instant::now();
while shared().lock().unwrap().rebuilding && start.elapsed() < Duration::from_secs(180) {
std::thread::sleep(Duration::from_millis(500));
}
println!("after restore: {:?}", run_state());
let (out, ok) = run_on("read_file", r#"{"path":"notes/hello.txt"}"#, &t, &cancel, &ids);
assert!(ok && out.contains("你好 世界"), "{out}");
let _ = run_on("shell", r#"{"command":"rm -rf /home/bot/bot-999 /home/bot/typed.txt"}"#, &t, &cancel, &ids);
}
/// `cargo test --release -- --ignored box_traffic_via_mac --nocapture`
#[test]
#[ignore]
fn box_traffic_via_mac() {
use crate::tools::{run_on, Place, RunIds, Target};
let cancel = AtomicBool::new(false);
ensure_running().unwrap();
set_egress_via_mac(true).unwrap();
let before = crate::egress::CONNECTIONS.load(Ordering::SeqCst);
let t = Target { name: "Bot 的電腦".into(), place: Place::Box { dir: bot_dir(997) } };
let (out, ok) = run_on("shell", r#"{"command":"curl -sS -o /dev/null -w '%{http_code}' https://example.com"}"#, &t, &cancel, &RunIds::default());
set_egress_via_mac(false).unwrap();
println!("{out} via mac: {}", crate::egress::CONNECTIONS.load(Ordering::SeqCst) - before);
assert!(ok && out.contains("200"), "{out}");
assert!(crate::egress::CONNECTIONS.load(Ordering::SeqCst) > before);
}
/// `cargo test --release -- --ignored box_update_keeps_files --nocapture`
#[test]
#[ignore]
fn box_update_keeps_files() {
use crate::tools::{run_on, Place, RunIds, Target};
let cancel = AtomicBool::new(false);
let ids = RunIds::default();
ensure_running().unwrap();
let t = Target { name: "Bot 的電腦".into(), place: Place::Box { dir: bot_dir(998) } };
let (o, ok) = run_on("shell", r#"{"command":"echo keep-me > kept.txt && sudo apt-get update -qq >/dev/null 2>&1 && sudo apt-get install -y -qq cowsay >/dev/null 2>&1; ls /usr/games/cowsay","timeout_ms":180000}"#, &t, &cancel, &ids);
assert!(ok, "{o}");
rebuild(Rebuild::Update).unwrap();
assert!(rebuild(Rebuild::Reset).is_err(), "only one rebuild at a time");
let start = Instant::now();
while shared().lock().unwrap().rebuilding && start.elapsed() < Duration::from_secs(600) {
std::thread::sleep(Duration::from_millis(500));
}
for l in log_lines() {
println!("{l}");
}
assert!(run_state().is_running(), "{:?}", run_state());
let (out, ok) = run_on("shell", r#"{"command":"cat kept.txt; ls /usr/games/cowsay 2>&1 || echo no-cowsay; cat /etc/anybot/menu | head -1"}"#, &t, &cancel, &ids);
println!("{out}");
assert!(ok && out.contains("keep-me") && out.contains("no-cowsay") && out.contains("Bot 的電腦"));
let _ = run_on("shell", r#"{"command":"cd .. && rm -rf bot-998"}"#, &t, &cancel, &ids);
}
#[test]
fn schedule_window_is_one_hour() {
let t = |s: &str| chrono::NaiveTime::parse_from_str(s, "%H:%M").unwrap();
assert!(in_schedule_window("03:00", t("03:00")));
assert!(in_schedule_window("03:00", t("03:59")));
assert!(!in_schedule_window("03:00", t("04:00")));
assert!(!in_schedule_window("03:00", t("02:59")));
assert!(in_schedule_window("23:30", t("00:10")));
assert!(!in_schedule_window("bad", t("00:10")));
assert_eq!(wanted_version(), "5");
}
#[test]
fn wrapper_runs_as_the_bot_in_its_folder() {
let w = exec_wrapper("/home/bot/bot-3", "ls -la", false, "");
assert!(w.contains("exec -u bot"));
assert!(w.contains(container()));
assert!(w.contains("/home/bot/bot-3"));
}
}