AnyBot/src/cloudbox.rs

1858 lines
82 KiB
Rust
Raw Normal View History

// 「Bot 的電腦」: AnyBot's own Linux computer that every bot shares, like Grok Bot's cloud "box".
//
// What the original does (Grok Bot 0.78.1, main-app.cjs dev-box control plane + local-exec-daemon):
// - The box is a container with an X11 desktop driven by xdotool (ComputerUseDriver XDOTOOL /
// NATIVE_X11), its size read with xrandr, watched through noVNC websockify on port 6080
// (`Ap.primaryNovncWebsockify:6080`).
// - Run states: ABSENT → 「建立中」, HIBERNATED → 「喚醒中」, STARTING → 「連線中」, RUNNING.
// - Health: GET /health with a 3 s timeout (QQn=3e3) that also says whether it's busy.
// - Store flush before a rebuild: POST /api/snapshotBoxStoreNow, 30 s (VQn=3e4).
// - Three rebuilds, one at a time (`isRecreateInFlight`):
// update (keep data, refused while busy): backing-up → creating → moving
// reset (wipe): cleaning-up → wiping → creating
// restore (force, from the durable store): cleaning-up → creating, then wait up to 90 s
// (zQn=9e4) polling every 2 s (HQn=2e3) for the copy-in to say "hydrated"
// - recreate-resume.json replays a rebuild cut short by a restart if it is under 10 minutes old
// ($Qn=10*6e4) and the box isn't healthy.
// Here the box is a local Docker container (or one on another Docker host, `ssh://me@server`),
// the durable store is a snapshot of the bot's home folder kept on this Mac, and xdotool runs
// through `docker exec`.
use base64::Engine;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::path::PathBuf;
use std::process::{Command, Stdio};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant, SystemTime};
/// The box's usual name. Two AnyBots on one Docker host (a server's brain and a desktop AnyBot
/// left on the same machine) each took "anybot-box" for their own, and the desktop one kept
/// pausing the brain's box as "idle". ANYBOT_BOX_NAME gives an AnyBot a box of its own name.
pub const CONTAINER: &str = "anybot-box";
/// This AnyBot's box (ANYBOT_BOX_NAME, else "anybot-box").
pub fn container() -> &'static str {
static C: OnceLock<String> = OnceLock::new();
C.get_or_init(|| {
std::env::var("ANYBOT_BOX_NAME")
.ok()
.map(|n| n.trim().to_string())
.filter(|n| !n.is_empty() && n.len() <= 60 && n.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'))
.unwrap_or_else(|| CONTAINER.to_string())
})
}
/// A box of its own name takes over the usual one this AnyBot used before (same files: the
/// home folder is a volume, kept by name; a rename doesn't stop it).
fn adopt_usual_box() {
let me = container();
if me == CONTAINER || dk(&["inspect", "-f", "{{.Id}}", me]).is_ok() {
return;
}
if dk(&["inspect", "-f", "{{.Id}}", CONTAINER]).is_ok() && dk(&["rename", CONTAINER, me]).is_ok() {
log(&format!("Bot 的電腦改用自己的名字 {me}(原本叫 {CONTAINER})。"));
}
}
pub const IMAGE: &str = "anybot-box:1";
/// The bot's files live here and survive an update (the original's mounted sand-data tree).
pub const VOLUME: &str = "anybot-box-home";
pub const HOME: &str = "/home/bot";
pub const DISPLAY: &str = ":1";
/// noVNC inside the box (Ap.primaryNovncWebsockify).
pub const NOVNC_PORT: u16 = 6080;
/// Where noVNC is published on this Mac (estimated; any free port works).
pub const HOST_PORT: u16 = 16080;
/// Desktop size (estimated; the original reads it with xrandr).
pub const SCREEN: &str = "1280x800x24";
/// Health probe timeout (QQn).
pub const HEALTH_TIMEOUT_MS: u64 = 3_000;
/// Saving a snapshot ("flush the durable store"). The original waits 30 s (VQn); a home folder of
/// a GB with an open browser takes minutes to pack, and a cut-off snapshot is no snapshot.
pub const SNAPSHOT_SAVE_TIMEOUT_MS: u64 = 600_000;
/// Restore copy-in: longest wait and poll interval (zQn, HQn).
pub const COPY_IN_WAIT_MS: u64 = 90_000;
pub const COPY_IN_POLL_MS: u64 = 2_000;
/// A rebuild marker older than this is cleared instead of replayed ($Qn).
pub const RESUME_FRESH_MS: u64 = 600_000;
/// Boot: longest wait for the desktop to come up (estimated).
pub const BOOT_WAIT_MS: u64 = 60_000;
/// Quiet time before the daily snapshot is taken (nobody using the box for this long).
pub const QUIET_SECS: u64 = 900;
/// The box's files are saved to this computer about once a day (it no longer sleeps, which is
/// when they used to be saved).
pub const SNAPSHOT_EVERY_SECS: i64 = 24 * 3600;
/// Secret names: an upper-case letter, then A–Z, 0–9, _ (original rule); size limits estimated.
pub const SECRET_MAX_VALUE_BYTES: usize = 32_768;
pub const SECRET_MAX_COUNT: usize = 100;
/// xdotool timing, matching the Mac helper where it applies (CUActuator).
const CLICK_REPEAT_MS: u64 = 80;
const TYPE_DELAY_MS: u64 = 12;
const DOCKERFILE: &str = include_str!("../box/Dockerfile");
const START_SH: &str = include_str!("../box/start.sh");
const FLUXBOX_INIT: &str = include_str!("../box/fluxbox-init");
const OVERLAY: &str = include_str!("../box/overlay");
const XRESOURCES: &str = include_str!("../box/Xresources");
const MENU: &str = include_str!("../box/menu");
// ---------- state ----------
#[derive(Clone, Debug, PartialEq)]
pub enum Run {
/// Docker isn't installed or isn't running.
NoDocker(String),
/// Not made yet (ABSENT).
Absent,
/// Asleep (HIBERNATED): paused or stopped.
Asleep,
/// Being made, woken or connected (the transient states).
Starting(&'static str),
Running,
/// A rebuild is under way: (phase, detail).
Rebuilding(String, String),
Failed(String),
}
impl Run {
pub fn label(&self) -> String {
match self {
Run::NoDocker(_) => "沒有 Docker".into(),
Run::Absent => "尚未建立".into(),
Run::Asleep => "休眠中".into(),
Run::Starting(s) => (*s).into(),
Run::Running => "執行中".into(),
Run::Rebuilding(p, _) => phase_label(p).into(),
Run::Failed(_) => "發生問題".into(),
}
}
pub fn is_running(&self) -> bool {
*self == Run::Running
}
}
pub fn phase_label(p: &str) -> &'static str {
match p {
"backing-up" => "正在備份",
"creating" => "正在建立",
"moving" => "正在搬移資料",
"cleaning-up" => "正在拆除舊電腦",
"wiping" => "正在清除資料",
"restoring" => "正在從快照還原",
"done" => "完成",
"failed" => "失敗",
_ => "處理中",
}
}
#[derive(Clone, Serialize, Deserialize, Default)]
pub struct Config {
/// Empty = this Mac's Docker; otherwise a Docker host such as `ssh://me@server`.
#[serde(default)]
pub docker_host: String,
/// 排程更新: local "HH:MM" to update when a newer computer is available (None = off).
#[serde(default)]
pub auto_update_at: Option<String>,
/// The day a scheduled update last ran (so it runs once).
#[serde(default)]
pub auto_update_done: String,
/// 網路流量改走你的電腦 (new connections only).
#[serde(default)]
pub egress_via_mac: bool,
/// VNC password for the desktop viewer (generated on first use; the published port is
/// loopback-only, but the box's own programs can reach it, and they should need a key).
#[serde(default)]
pub vnc_password: String,
}
/// A scheduled update that was missed may still run for this long (the original: one hour).
pub const SCHEDULE_GRACE_MINUTES: i64 = 60;
struct Shared {
run: Run,
last_used: Instant,
rebuilding: bool,
log: Vec<String>,
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
/// The desktop's size in pixels (the bot's pictures are this, fitted; see box_scale).
size: Option<(u32, u32)>,
tunnel: Option<std::process::Child>,
host_port: u16,
}
fn shared() -> &'static Mutex<Shared> {
static S: OnceLock<Mutex<Shared>> = OnceLock::new();
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
S.get_or_init(|| Mutex::new(Shared { run: Run::Absent, last_used: Instant::now(), rebuilding: false, log: vec![], size: None, tunnel: None, host_port: HOST_PORT }))
}
fn config() -> &'static Mutex<Config> {
static C: OnceLock<Mutex<Config>> = OnceLock::new();
C.get_or_init(|| Mutex::new(load_config()))
}
/// The box's VNC password: made once, kept across restarts (the container is started with it,
/// the viewer opens with it). x11vnc ran with no password at all before — any program inside
/// the box (a page the bot opened, for instance) could watch and drive the desktop.
pub fn vnc_password() -> String {
{
let c = config().lock().unwrap();
if !c.vnc_password.is_empty() {
return c.vnc_password.clone();
}
}
let pw = crate::httpd::random_hex(10);
config().lock().unwrap().vnc_password.clone_from(&pw);
save_config(&config().lock().unwrap());
pw
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
/// The password the box's VNC server really has: the one it was made with (its VNC_PASSWORD), so a
/// viewer signs in even when this AnyBot's settings were made anew; else this AnyBot's own.
pub fn box_vnc_password() -> String {
dk_timeout(&["exec", container(), "printenv", "VNC_PASSWORD"], 5_000)
.ok()
.map(|b| String::from_utf8_lossy(&b).trim().to_string())
.filter(|p| !p.is_empty())
.unwrap_or_else(vnc_password)
}
static BUSY: AtomicUsize = AtomicUsize::new(0);
/// The person took control in the viewer: the bot's screen actions wait.
pub static HELD_BY_PERSON: AtomicBool = AtomicBool::new(false);
/// Marks the box as in use while alive (keeps it awake; an update waits for it).
pub struct Busy(bool);
impl Busy {
pub fn new(on: bool) -> Self {
if on {
BUSY.fetch_add(1, Ordering::SeqCst);
touch();
}
Busy(on)
}
}
impl Drop for Busy {
fn drop(&mut self) {
if self.0 {
BUSY.fetch_sub(1, Ordering::SeqCst);
touch();
}
}
}
pub fn is_busy() -> bool {
BUSY.load(Ordering::SeqCst) > 0
}
pub fn touch() {
shared().lock().unwrap().last_used = Instant::now();
}
pub fn run_state() -> Run {
shared().lock().unwrap().run.clone()
}
fn set_run(r: Run) {
shared().lock().unwrap().run = r;
}
pub fn log_lines() -> Vec<String> {
shared().lock().unwrap().log.clone()
}
fn log(line: &str) {
let mut s = shared().lock().unwrap();
s.log.push(format!("{} {line}", chrono::Local::now().format("%H:%M:%S")));
let n = s.log.len();
if n > 200 {
s.log.drain(..n - 200);
}
}
// ---------- files on this Mac ----------
fn data_dir() -> PathBuf {
crate::model::State::path().parent().map(|p| p.to_path_buf()).unwrap_or_else(std::env::temp_dir).join("box")
}
fn config_path() -> PathBuf {
data_dir().join("config.json")
}
fn load_config() -> Config {
std::fs::read_to_string(config_path()).ok().and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default()
}
pub fn docker_host() -> String {
config().lock().unwrap().docker_host.clone()
}
pub fn set_docker_host(h: &str) {
let h = h.trim().to_string();
let mut c = config().lock().unwrap();
if c.docker_host == h {
return;
}
c.docker_host = h;
let _ = std::fs::create_dir_all(data_dir());
let _ = std::fs::write(config_path(), serde_json::to_string_pretty(&*c).unwrap_or_default());
drop(c);
if let Some(mut t) = shared().lock().unwrap().tunnel.take() {
let _ = t.kill();
}
set_run(Run::Absent);
refresh();
}
fn save_config(c: &Config) {
let _ = std::fs::create_dir_all(data_dir());
let _ = std::fs::write(config_path(), serde_json::to_string_pretty(c).unwrap_or_default());
}
pub fn egress_via_mac() -> bool {
config().lock().unwrap().egress_via_mac
}
/// Turn routing through this Mac on or off. Only this Mac's Docker can reach it.
pub fn set_egress_via_mac(on: bool) -> Result<(), String> {
if on {
if !docker_host().is_empty() {
return Err("Bot 的電腦在別的主機上時,不能改走這台 Mac。".into());
}
crate::egress::start()?;
}
let mut c = config().lock().unwrap();
c.egress_via_mac = on;
save_config(&c);
Ok(())
}
/// Proxy settings for new commands in the box when routing through this Mac.
fn egress_env() -> String {
if !egress_via_mac() || !docker_host().is_empty() || !crate::policy::read_egress().allows() {
return String::new();
}
match crate::egress::start() {
Ok(port) => {
let url = format!("http://host.docker.internal:{port}");
["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"].iter().map(|k| format!(" -e {k}={url}")).collect::<String>() + " -e NO_PROXY=localhost,127.0.0.1 -e no_proxy=localhost,127.0.0.1"
}
Err(_) => String::new(),
}
}
pub fn auto_update_at() -> Option<String> {
config().lock().unwrap().auto_update_at.clone()
}
pub fn set_auto_update_at(at: Option<String>) {
let mut c = config().lock().unwrap();
c.auto_update_at = at;
save_config(&c);
}
/// The computer version baked into this AnyBot (the Dockerfile's label).
pub fn wanted_version() -> String {
DOCKERFILE.lines().find_map(|l| l.trim().strip_prefix("LABEL anybot.box.version=")).map(|v| v.trim_matches('"').to_string()).unwrap_or_default()
}
/// The running computer is older than the one this AnyBot carries.
pub fn update_available() -> bool {
match dk(&["inspect", "-f", "{{index .Config.Labels \"anybot.box.version\"}}", container()]) {
Ok(v) => v.trim() != wanted_version(),
Err(_) => false,
}
}
/// Is `now` inside the window [at, at + grace)? (`at` is "HH:MM").
pub fn in_schedule_window(at: &str, now: chrono::NaiveTime) -> bool {
let Ok(t) = chrono::NaiveTime::parse_from_str(at.trim(), "%H:%M") else { return false };
let mins = (now - t).num_minutes().rem_euclid(24 * 60);
mins < SCHEDULE_GRACE_MINUTES
}
fn maybe_scheduled_update() {
let (at, done) = {
let c = config().lock().unwrap();
(c.auto_update_at.clone(), c.auto_update_done.clone())
};
let Some(at) = at else { return };
let now = chrono::Local::now();
let today = now.format("%Y-%m-%d").to_string();
if done == today || !in_schedule_window(&at, now.time()) || is_busy() || !update_available() {
return;
}
log(&format!("排程更新({at})開始。"));
if rebuild(Rebuild::Update).is_ok() {
let mut c = config().lock().unwrap();
c.auto_update_done = today;
save_config(&c);
}
}
pub fn snapshot_path() -> PathBuf {
data_dir().join("snapshot.tar.gz")
}
/// When the last snapshot was saved.
pub fn snapshot_time() -> Option<chrono::DateTime<chrono::Local>> {
std::fs::metadata(snapshot_path()).ok()?.modified().ok().map(chrono::DateTime::<chrono::Local>::from)
}
fn resume_path() -> PathBuf {
data_dir().join("recreate-resume.json")
}
/// Where docker and ssh live when the app is started from Finder (no shell PATH).
pub fn tool_path() -> String {
let base = std::env::var("PATH").unwrap_or_default();
let mut parts: Vec<String> = base.split(':').filter(|s| !s.is_empty()).map(String::from).collect();
let home = dirs::home_dir().map(|h| h.display().to_string()).unwrap_or_default();
for extra in ["/usr/local/bin", "/opt/homebrew/bin", "/usr/bin", "/bin", "/usr/sbin", "/sbin", &format!("{home}/.orbstack/bin"), "/Applications/Docker.app/Contents/Resources/bin"] {
if !parts.iter().any(|p| p == extra) {
parts.push(extra.to_string());
}
}
parts.join(":")
}
fn docker() -> Command {
let mut c = Command::new("docker");
c.env("PATH", tool_path());
let host = docker_host();
if !host.is_empty() {
c.env("DOCKER_HOST", host);
}
c.stdin(Stdio::null());
c
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
/// A program in the box with its stdin and stdout piped to this side (as the bot user).
pub fn exec_piped(args: &[&str]) -> Result<std::process::Child, String> {
let mut a = vec!["exec", "-i", "-u", "bot", container()];
a.extend_from_slice(args);
docker().args(&a).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().map_err(|e| format!("找不到 docker:{e}"))
}
/// The `docker …` prefix for shell strings (used by the command tools).
fn docker_prefix() -> String {
let host = docker_host();
if host.is_empty() {
"docker".into()
} else {
format!("DOCKER_HOST={} docker", crate::tools::sh_quote(&host))
}
}
/// Run docker; Ok(stdout) or Err(stderr).
fn dk(args: &[&str]) -> Result<String, String> {
// Never block forever on the daemon: a hung docker daemon used to pile up a new stuck
// thread every 20 s from the refresh loop. (docker's own CLI has no default timeout.)
let out = dk_timeout(args, 120_000)?;
Ok(String::from_utf8_lossy(&out).trim().to_string())
}
fn dk_timeout(args: &[&str], ms: u64) -> Result<Vec<u8>, String> {
use std::io::Read;
let mut child = docker().args(args).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().map_err(|e| format!("找不到 docker:{e}"))?;
// Drain both pipes while waiting, or a big screenshot fills the pipe and stalls docker.
let mut so = child.stdout.take().ok_or("沒有輸出")?;
let mut se = child.stderr.take().ok_or("沒有輸出")?;
let out_t = std::thread::spawn(move || {
let mut v = vec![];
let _ = so.read_to_end(&mut v);
v
});
let err_t = std::thread::spawn(move || {
let mut v = vec![];
let _ = se.read_to_end(&mut v);
v
});
let start = Instant::now();
let status = loop {
if let Ok(Some(st)) = child.try_wait() {
break st;
}
if start.elapsed() > Duration::from_millis(ms) {
let _ = child.kill();
let _ = child.wait();
return Err("逾時".into());
}
std::thread::sleep(Duration::from_millis(10));
};
let out = out_t.join().unwrap_or_default();
let err = err_t.join().unwrap_or_default();
if status.success() {
Ok(out)
} else {
Err(String::from_utf8_lossy(&err).trim().to_string())
}
}
// ---------- status ----------
/// Ask Docker what state the box is in (doesn't change anything).
pub fn probe() -> Run {
if let Err(e) = dk(&["version", "--format", "{{.Server.Version}}"]) {
return Run::NoDocker(if e.contains("找不到") && !cfg!(target_os = "macos") {
"這台電腦沒有安裝 Docker。安裝 Docker Engine(Ubuntu / Debian:sudo apt install docker.io),再把自己加進 docker 群組(sudo usermod -aG docker $USER,然後重新登入),Bot 就會有自己的完整電腦。".into()
} else if e.contains("找不到") {
"這台 Mac 沒有安裝 Docker。安裝 OrbStack 或 Docker Desktop 後,Bot 就會有自己的完整電腦。".into()
} else if e.contains("permission denied") && !cfg!(target_os = "macos") {
"Docker 不讓 AnyBot 使用:請執行 sudo usermod -aG docker $USER,然後登出再登入。".into()
} else {
format!("Docker 沒有在執行:{}", e.lines().last().unwrap_or(""))
});
}
match dk(&["inspect", "-f", "{{.State.Status}}", container()]) {
Err(_) => Run::Absent,
Ok(s) => match s.as_str() {
"running" => {
// The health probe (3 s, like the original's GET /health).
if dk_timeout(&["exec", container(), "test", "-f", "/tmp/anybot-box.ready"], HEALTH_TIMEOUT_MS).is_ok() {
Run::Running
} else {
Run::Starting("連線中")
}
}
"paused" | "exited" | "created" => Run::Asleep,
"restarting" => Run::Starting("連線中"),
other => Run::Failed(format!("容器狀態:{other}")),
},
}
}
/// Refresh the shown state in the background (unless a rebuild or start owns it).
pub fn refresh() {
std::thread::spawn(|| {
let r = probe();
let mut s = shared().lock().unwrap();
if s.rebuilding || matches!(s.run, Run::Starting(_)) && !matches!(r, Run::Running) {
return;
}
s.run = r;
});
}
pub fn available() -> bool {
!matches!(run_state(), Run::NoDocker(_))
}
// ---------- image ----------
fn image_exists() -> bool {
dk(&["image", "inspect", IMAGE]).is_ok()
}
/// Build the box image from the files baked into AnyBot.
fn build_image(pull: bool) -> Result<(), String> {
let dir = std::env::temp_dir().join(format!("anybot-box-build-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
for (name, body) in [("Dockerfile", DOCKERFILE), ("start.sh", START_SH), ("fluxbox-init", FLUXBOX_INIT), ("overlay", OVERLAY), ("Xresources", XRESOURCES), ("menu", MENU)] {
std::fs::write(dir.join(name), body).map_err(|e| e.to_string())?;
}
log("正在建立電腦的映像檔(第一次要幾分鐘)…");
let mut args = vec!["build", "-q", "-t", IMAGE];
if pull {
args.push("--pull");
}
let d = dir.display().to_string();
args.push(&d);
// Builds take minutes (a cold one more than dk's short timeout allows).
let r = dk_timeout(&args, 20 * 60_000).map(|_| String::new());
let _ = std::fs::remove_dir_all(&dir);
r.map(|_| log("映像檔完成。")).map_err(|e| format!("建立映像檔失敗:{}", tail(&e, 600)))
}
fn tail(s: &str, n: usize) -> String {
let c: Vec<char> = s.chars().collect();
if c.len() <= n {
s.to_string()
} else {
c[c.len() - n..].iter().collect()
}
}
// ---------- lifecycle ----------
fn free_port(start: u16) -> u16 {
if !docker_host().is_empty() {
return start;
}
(start..start + 50).find(|p| std::net::TcpListener::bind(("127.0.0.1", *p)).is_ok()).unwrap_or(start)
}
fn create_container() -> Result<(), String> {
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
shared().lock().unwrap_or_else(|e| e.into_inner()).size = None;
if !image_exists() {
build_image(false)?;
}
let port = free_port(HOST_PORT);
let publish = format!("127.0.0.1:{port}:{NOVNC_PORT}");
let vol = format!("{VOLUME}:{HOME}");
let env = format!("SCREEN_SIZE={SCREEN}");
let vnc = format!("VNC_PASSWORD={}", vnc_password());
log("正在開一台新的電腦…");
// host.docker.internal → this Mac (for 網路流量改走你的電腦; Docker Desktop and OrbStack already have it).
dk(&["run", "-d", "--name", container(), "--hostname", "bot-computer", "--shm-size", "1g", "--pids-limit", "4096", "--memory", "2g", "--cpus", "2", "--add-host", "host.docker.internal:host-gateway", "-p", &publish, "-v", &vol, "-e", &env, "-e", &vnc, "--label", "anybot.box=1", IMAGE]).map_err(|e| format!("開不了電腦:{}", tail(&e, 400)))?;
Ok(())
}
fn wait_ready(ms: u64) -> Result<(), String> {
let start = Instant::now();
while start.elapsed() < Duration::from_millis(ms) {
if dk(&["exec", container(), "test", "-f", "/tmp/anybot-box.ready"]).is_ok() {
return Ok(());
}
std::thread::sleep(Duration::from_millis(250));
}
Err("電腦開機逾時".into())
}
// ---------- the box's own browser ----------
// Browser steps on Bot 的電腦 drive the Chromium on its desktop (the person can watch it in the
// box's window). Its DevTools port stays inside the box; a local port reaches it through
// `docker exec` (no published port, so it also works with a Docker host elsewhere).
const BOX_CDP_PORT: u16 = 9222;
fn box_cdp_alive(local: u16) -> bool {
crate::botbrowser::local().get(&format!("http://127.0.0.1:{local}/json/version")).call().map(|r| r.status().as_u16() == 200).unwrap_or(false)
}
/// A local port that reaches the box's DevTools port.
fn box_cdp_forward() -> Result<u16, String> {
static PORT: Mutex<u16> = Mutex::new(0);
let mut p = PORT.lock().unwrap();
if *p != 0 {
return Ok(*p);
}
let l = std::net::TcpListener::bind("127.0.0.1:0").map_err(|e| e.to_string())?;
let port = l.local_addr().map_err(|e| e.to_string())?.port();
std::thread::spawn(move || {
for conn in l.incoming().flatten() {
std::thread::spawn(move || {
let script = format!("exec 3<>/dev/tcp/127.0.0.1/{BOX_CDP_PORT} || exit 1; cat <&3 & cat >&3; kill $! 2>/dev/null");
let Ok(mut child) = docker().args(["exec", "-i", container(), "bash", "-c", &script]).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()).spawn() else { return };
let (Some(mut cin), Some(mut cout)) = (child.stdin.take(), child.stdout.take()) else { return };
let (Ok(mut r), Ok(mut w)) = (conn.try_clone(), conn.try_clone()) else { return };
let up = std::thread::spawn(move || {
let _ = std::io::copy(&mut r, &mut cin);
});
let _ = std::io::copy(&mut cout, &mut w);
let _ = conn.shutdown(std::net::Shutdown::Both);
let _ = child.kill();
let _ = up.join();
let _ = child.wait();
});
}
});
*p = port;
Ok(port)
}
/// Is the box's Chromium answering already?
pub fn browser_running() -> bool {
box_cdp_forward().map(box_cdp_alive).unwrap_or(false)
}
/// The box's Chromium, started on its desktop when it isn't running. Gives the local port.
pub fn browser_port() -> Result<u16, String> {
ensure_running()?;
touch();
let port = box_cdp_forward()?;
if box_cdp_alive(port) {
return Ok(port);
}
let dbg = format!("--remote-debugging-port={BOX_CDP_PORT}");
dk(&["exec", "-d", "-u", "bot", "-e", "DISPLAY=:1", "-e", "HOME=/home/bot", container(), "chromium", &dbg, "--remote-allow-origins=*", "--user-data-dir=/home/bot/.config/anybot-browser", "about:blank"]).map_err(|e| format!("Bot 的電腦打不開瀏覽器:{e}"))?;
let start = Instant::now();
while start.elapsed() < Duration::from_secs(20) {
if box_cdp_alive(port) {
return Ok(port);
}
std::thread::sleep(Duration::from_millis(300));
}
Err("這台電腦的瀏覽器沒有及時打開,請再試一次。".into())
}
static START_LOCK: Mutex<()> = Mutex::new(());
/// Make sure the box is up, waking or creating it as needed. Blocks until it's ready.
pub fn ensure_running() -> Result<(), String> {
let _g = START_LOCK.lock().unwrap();
if shared().lock().unwrap().rebuilding {
return Err("Bot 的電腦正在更新,請等它完成再試。".into());
}
touch();
let r = probe();
let result = match r {
Run::Running => Ok(()),
Run::NoDocker(e) => Err(e),
Run::Absent => {
set_run(Run::Starting("建立中"));
create_container().and_then(|_| wait_ready(BOOT_WAIT_MS))
}
Run::Asleep => {
set_run(Run::Starting("喚醒中"));
log("喚醒電腦…");
let st = dk(&["inspect", "-f", "{{.State.Status}}", container()]).unwrap_or_default();
let woke = if st == "paused" { dk(&["unpause", container()]) } else { dk(&["start", container()]) };
woke.map_err(|e| format!("叫不醒電腦:{e}")).and_then(|_| wait_ready(BOOT_WAIT_MS))
}
Run::Starting(_) => {
set_run(Run::Starting("連線中"));
wait_ready(BOOT_WAIT_MS)
}
Run::Rebuilding(..) => Err("Bot 的電腦正在更新,請等它完成再試。".into()),
Run::Failed(e) => {
// A broken container: start it again.
log(&format!("電腦狀態異常({e}),重新啟動…"));
set_run(Run::Starting("連線中"));
dk(&["restart", container()]).map_err(|e| e.to_string()).and_then(|_| wait_ready(BOOT_WAIT_MS))
}
};
match &result {
Ok(()) => {
set_run(Run::Running);
let _ = dk(&["exec", container(), "sh", "-c", &format!("mkdir -p {HOME} && chown bot:bot {HOME}")]);
}
Err(e) => {
log(e);
set_run(match probe() {
Run::NoDocker(x) => Run::NoDocker(x),
_ => Run::Failed(e.clone()),
});
}
}
touch();
result
}
/// Started once: keeps the box awake (wakes it when it was asleep), saves its files once a day,
/// and replays a rebuild cut short by a restart. The person asked for the bots' computer to be
/// always there: an idle box used to be paused after 15 minutes, and the person — and a bot
/// working from a brain far away — kept finding it "asleep".
pub fn start_background() {
static STARTED: AtomicBool = AtomicBool::new(false);
if STARTED.swap(true, Ordering::SeqCst) {
return;
}
std::thread::spawn(|| {
adopt_usual_box();
set_run(probe());
resume_interrupted();
loop {
// (First wait a little: a computer showing another brain says so right after start.)
std::thread::sleep(Duration::from_secs(20));
let (idle, rebuilding) = {
let s = shared().lock().unwrap();
(s.last_used.elapsed().as_secs(), s.rebuilding)
};
if !rebuilding {
if IS_BRAIN.load(Ordering::SeqCst) && matches!(probe(), Run::Asleep) {
log("Bot 的電腦不休眠:叫醒它。");
let _ = ensure_running();
}
let quiet = idle >= QUIET_SECS && !is_busy() && !HELD_BY_PERSON.load(Ordering::SeqCst) && !viewer_open();
let due = snapshot_time().map(|t| (chrono::Local::now() - t).num_seconds() >= SNAPSHOT_EVERY_SECS).unwrap_or(true);
if quiet && due && run_state().is_running() {
let _ = save_snapshot();
}
refresh();
maybe_scheduled_update();
}
}
});
}
/// This AnyBot is the brain (its box is the bots' computer). A computer showing another brain
/// doesn't use its own box, so it isn't kept awake there.
static IS_BRAIN: AtomicBool = AtomicBool::new(true);
pub fn set_brain(on: bool) {
IS_BRAIN.store(on, Ordering::SeqCst);
}
static VIEWER_OPEN: AtomicBool = AtomicBool::new(false);
pub fn set_viewer_open(on: bool) {
VIEWER_OPEN.store(on, Ordering::SeqCst);
if on {
touch();
}
}
fn viewer_open() -> bool {
VIEWER_OPEN.load(Ordering::SeqCst)
}
// ---------- snapshots (the durable store) ----------
/// Save the bot's home folder to this Mac. Returns its size in bytes.
pub fn save_snapshot() -> Result<u64, String> {
let _ = std::fs::create_dir_all(data_dir());
let tmp = data_dir().join("snapshot.tmp");
let file = std::fs::File::create(&tmp).map_err(|e| e.to_string())?;
let mut child = docker()
// Caches are left out: the browser's grow to hundreds of MB and come back by themselves.
.args([
"exec", container(), "tar", "-C", HOME, "--exclude=./.cache", "--exclude=Cache", "--exclude=Code Cache", "--exclude=GPUCache", "--exclude=CacheStorage", "--exclude=ScriptCache", "--exclude=Crashpad",
"--warning=no-file-changed", "-czf", "-", ".",
])
.stdout(Stdio::from(file))
.stderr(Stdio::piped())
.spawn()
.map_err(|e| e.to_string())?;
// Drain stderr: tar's file-changed warnings pile up fast, and a full 64 KB pipe stalls
// tar (and the snapshot) until the kill timer fires.
let mut se = child.stderr.take().ok_or("沒有輸出")?;
let err_t = std::thread::spawn(move || {
use std::io::Read;
let mut v = vec![];
let _ = se.read_to_end(&mut v);
v
});
let start = Instant::now();
let ok = loop {
// tar says 1 when a file changed while it was read (the open browser writes all the
// time): the archive is still whole. Only 2 and up are failures.
if let Ok(Some(st)) = child.try_wait() {
break st.code().map(|c| c <= 1).unwrap_or(false);
}
if start.elapsed() > Duration::from_millis(SNAPSHOT_SAVE_TIMEOUT_MS) {
let _ = child.kill();
break false;
}
std::thread::sleep(Duration::from_millis(50));
};
let _ = err_t.join();
if !ok {
let _ = std::fs::remove_file(&tmp);
return Err("快照沒有存成功".into());
}
std::fs::rename(&tmp, snapshot_path()).map_err(|e| e.to_string())?;
let n = std::fs::metadata(snapshot_path()).map(|m| m.len()).unwrap_or(0);
log(&format!("已儲存快照({})。", human(n)));
Ok(n)
}
fn copy_in_snapshot() -> Result<(), String> {
use std::io::Write;
let data = std::fs::read(snapshot_path()).map_err(|_| "還沒有快照可以還原".to_string())?;
let mut child = docker()
.args(["exec", "-i", container(), "sh", "-c", &format!("tar -C {HOME} -xzf - && chown -R bot:bot {HOME} && echo '[box-copy-in] result outcome=hydrated' >> /tmp/anybot-copy-in.log")])
.stdin(Stdio::piped())
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.map_err(|e| e.to_string())?;
child.stdin.take().ok_or("寫不進電腦")?.write_all(&data).map_err(|e| e.to_string())?;
let out = child.wait_with_output().map_err(|e| e.to_string())?;
if !out.status.success() {
return Err(format!("還原失敗:{}", String::from_utf8_lossy(&out.stderr).trim()));
}
// Like readCopyInOutcome: wait for the copy-in to report "hydrated".
let start = Instant::now();
while start.elapsed() < Duration::from_millis(COPY_IN_WAIT_MS) {
if let Ok(s) = dk(&["exec", container(), "sh", "-c", "cat /tmp/anybot-copy-in.log 2>/dev/null || true"]) {
if s.contains("outcome=hydrated") {
return Ok(());
}
}
std::thread::sleep(Duration::from_millis(COPY_IN_POLL_MS));
}
Err("還原沒有完成:copy-in outcome=unknown".into())
}
fn home_has_files() -> bool {
dk(&["exec", container(), "sh", "-c", &format!("ls -A {HOME} | head -1")]).map(|s| !s.trim().is_empty()).unwrap_or(false)
}
pub fn human(b: u64) -> String {
match b {
b if b >= 1 << 30 => format!("{:.1} GB", b as f64 / (1u64 << 30) as f64),
b if b >= 1 << 20 => format!("{:.1} MB", b as f64 / (1u64 << 20) as f64),
b if b >= 1 << 10 => format!("{:.0} KB", b as f64 / 1024.0),
b => format!("{b} B"),
}
}
// ---------- rebuilds ----------
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Rebuild {
/// 重建電腦: keep files and logins, remove installed apps and packages.
Update,
/// 重設: a brand-new computer, nothing kept.
Reset,
/// 復原: start again from the last saved snapshot.
Restore,
}
impl Rebuild {
pub fn title(self) -> &'static str {
match self {
Rebuild::Update => "重建電腦",
Rebuild::Reset => "重設電腦",
Rebuild::Restore => "從快照復原",
}
}
/// The original's confirmation wording.
pub fn warning(self) -> &'static str {
match self {
Rebuild::Update => "你的檔案和登入資訊會保留,但已安裝的應用程式和套件會被移除。所有 Bot 會一起更新。正在工作的 Bot 會先停下來,聊天會保留。",
Rebuild::Reset => "重新建立電腦並重新連線。電腦裡的所有檔案、登入資訊和已安裝的應用程式都會被清除(你的 Bot 和聊天不受影響)。",
Rebuild::Restore => "從你上次儲存的快照重新啟動它。最近的變更可能會遺失,已安裝的 App 會被移除。你的 Bot、檔案和登入資訊都是安全的。",
}
}
}
#[derive(Serialize, Deserialize)]
struct ResumeMarker {
#[serde(rename = "resumeKind")]
kind: Rebuild,
#[serde(rename = "writtenAtMs")]
written_at_ms: u64,
}
fn now_ms() -> u64 {
SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0)
}
/// Start a rebuild in the background. Err = refused (already running, busy, no snapshot…).
pub fn rebuild(kind: Rebuild) -> Result<(), String> {
{
let mut s = shared().lock().unwrap();
if s.rebuilding {
return Err("已經有一個更新在進行中。".into());
}
if kind == Rebuild::Update && BUSY.load(Ordering::SeqCst) > 0 {
return Err("有 Bot 正在用這台電腦,等它做完再重建(或先停止它)。".into());
}
if kind == Rebuild::Restore && !snapshot_path().exists() {
return Err("還沒有儲存過快照,沒有東西可以還原。".into());
}
s.rebuilding = true;
}
std::thread::spawn(move || {
let _ = std::fs::create_dir_all(data_dir());
let _ = std::fs::write(resume_path(), serde_json::to_string(&ResumeMarker { kind, written_at_ms: now_ms() }).unwrap_or_default());
let phase = |p: &str, d: &str| {
log(&format!("{}:{d}", phase_label(p)));
shared().lock().unwrap().run = Run::Rebuilding(p.into(), d.into());
};
let r = run_rebuild(kind, &phase);
let _ = std::fs::remove_file(resume_path());
match r {
Ok(()) => {
log(&format!("{}完成。", kind.title()));
let mut s = shared().lock().unwrap();
s.rebuilding = false;
s.run = Run::Running;
s.last_used = Instant::now();
}
Err(e) => {
log(&format!("{}失敗:{e}", kind.title()));
let mut s = shared().lock().unwrap();
s.rebuilding = false;
s.run = Run::Failed(e);
}
}
});
Ok(())
}
fn run_rebuild(kind: Rebuild, phase: &dyn Fn(&str, &str)) -> Result<(), String> {
match kind {
Rebuild::Update => {
let had = matches!(probe(), Run::Running);
phase("backing-up", "儲存一份快照");
if had {
if let Err(e) = save_snapshot() {
log(&format!("快照沒有存成功({e}),改以掛載的資料為準。"));
}
}
let had_files = had && home_has_files();
phase("creating", "更新電腦的映像檔");
build_image(true)?;
phase("creating", "換上一台新的電腦");
let _ = dk(&["rm", "-f", container()]);
create_container()?;
wait_ready(BOOT_WAIT_MS)?;
phase("moving", "確認保留的檔案和新電腦的連線");
if had_files && !home_has_files() {
return Err("重建後找不到原本的檔案".into());
}
Ok(())
}
Rebuild::Reset => {
phase("cleaning-up", "拆掉舊的電腦");
let _ = dk(&["rm", "-f", container()]);
phase("wiping", "清除電腦裡的資料");
let _ = dk(&["volume", "rm", "-f", VOLUME]);
phase("creating", "開一台全新的電腦");
if !image_exists() {
build_image(false)?;
}
create_container()?;
wait_ready(BOOT_WAIT_MS)
}
Rebuild::Restore => {
phase("cleaning-up", "拆掉舊的電腦和它的資料");
let _ = dk(&["rm", "-f", container()]);
let _ = dk(&["volume", "rm", "-f", VOLUME]);
phase("creating", "從快照開一台新的電腦");
if !image_exists() {
build_image(false)?;
}
create_container()?;
wait_ready(BOOT_WAIT_MS)?;
phase("restoring", "把快照裡的檔案放回去");
copy_in_snapshot()
}
}
}
/// resumeInterruptedRecreate: replay a fresh marker when the box isn't healthy.
fn resume_interrupted() {
let Some(m) = std::fs::read_to_string(resume_path()).ok().and_then(|s| serde_json::from_str::<ResumeMarker>(&s).ok()) else { return };
if now_ms().saturating_sub(m.written_at_ms) >= RESUME_FRESH_MS {
log("清掉一個太舊的未完成更新紀錄(超過十分鐘)。");
let _ = std::fs::remove_file(resume_path());
return;
}
if matches!(probe(), Run::Running) {
log("上次的更新被中斷,但電腦目前正常,所以沒有重做。需要的話請再執行一次。");
let _ = std::fs::remove_file(resume_path());
return;
}
log(&format!("接續上次被中斷的「{}」。", m.kind.title()));
let _ = rebuild(m.kind);
}
// ---------- disk ----------
/// Disk use in the box, for the settings page and the clean-up request.
pub fn disk_report() -> Result<String, String> {
ensure_running()?;
dk(&["exec", "-u", "bot", container(), "sh", "-c", &format!("df -h {HOME} | tail -1; echo; du -sh {HOME}/* {HOME}/.cache 2>/dev/null | sort -rh | head -8")])
}
// ---------- secrets (write-only) ----------
fn secrets_path() -> PathBuf {
data_dir().join("secrets.json")
}
fn env_file_path() -> PathBuf {
data_dir().join("secrets.env")
}
fn read_secrets() -> std::collections::BTreeMap<String, String> {
std::fs::read_to_string(secrets_path()).ok().and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default()
}
fn write_secrets(m: &std::collections::BTreeMap<String, String>) -> Result<(), String> {
let _ = std::fs::create_dir_all(data_dir());
let write = |p: PathBuf, body: String| -> Result<(), String> {
std::fs::write(&p, body).map_err(|e| e.to_string())?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o600));
}
Ok(())
};
write(secrets_path(), serde_json::to_string(m).unwrap_or_default())?;
// docker --env-file: one NAME=value per line (no newlines allowed in values).
write(env_file_path(), m.iter().map(|(k, v)| format!("{k}={}\n", v.replace('\n', "\\n"))).collect())
}
/// Names only: values can be written, never read back.
pub fn secret_names() -> Vec<String> {
read_secrets().into_keys().collect()
}
pub fn valid_secret_name(n: &str) -> bool {
let mut c = n.chars();
matches!(c.next(), Some('A'..='Z')) && c.all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit() || ch == '_')
}
pub fn set_secret(name: &str, value: &str) -> Result<(), String> {
let name = name.trim();
if !valid_secret_name(name) {
return Err("名稱要大寫字母開頭,只能用 A–Z、0–9 和 _,例如 CRM_API_TOKEN".into());
}
if value.is_empty() {
return Err("請填入值".into());
}
// docker's --env-file keeps values on one line; a value with newlines used to be written
// with literal "\n" escapes and arrived broken (multi-line PEM keys) — say so up front.
if value.contains(['\n', '\r']) {
return Err("值裡不能有換行(多行金鑰請壓成一行,例如 -----BEGIN KEY-----…-----END KEY-----)".into());
}
if value.len() > SECRET_MAX_VALUE_BYTES {
return Err(format!("值太長了(最多 {} KB)", SECRET_MAX_VALUE_BYTES / 1024));
}
let mut m = read_secrets();
if !m.contains_key(name) && m.len() >= SECRET_MAX_COUNT {
return Err(format!("最多 {SECRET_MAX_COUNT} 個密鑰"));
}
m.insert(name.to_string(), value.to_string());
write_secrets(&m)
}
pub fn remove_secret(name: &str) {
let mut m = read_secrets();
if m.remove(name).is_some() {
let _ = write_secrets(&m);
}
}
// ---------- commands ----------
/// The bot's own folder inside the box.
pub fn bot_dir(bot_id: u64) -> String {
format!("{HOME}/bot-{bot_id}")
}
/// A local shell string that runs `cmd` in the box as the bot user, in `dir`, with the secrets.
pub fn exec_wrapper(dir: &str, cmd: &str, interactive: bool, job: &str) -> String {
use crate::tools::sh_quote;
// The job tag rides in the command line so the app can kill the in-container process later
// (killing the local docker client alone leaves the program running in the box).
let tagged = if job.is_empty() { cmd.to_string() } else { format!("export ANYBOT_JOB={job}; {cmd}") };
let inner = format!("mkdir -p {d} && cd {d} && {tagged}", d = sh_quote(dir));
let env = env_file_path();
let env_arg = if env.exists() { format!(" --env-file {}", sh_quote(&env.display().to_string())) } else { String::new() };
format!("{} exec{} -u bot -e DISPLAY={DISPLAY} -e HOME={HOME}{}{env_arg} {box} bash -lc {}", docker_prefix(), if interactive { " -i" } else { "" }, egress_env(), sh_quote(&inner), r#box = container())
}
/// An argv that kills the box process carrying this job tag (and its children).
pub fn exec_kill_argv(job: &str) -> Vec<String> {
let mut argv = vec!["docker".to_string()];
let host = docker_host();
if !host.is_empty() {
argv.extend(["-H".to_string(), host]);
}
argv.extend([
"exec".to_string(),
container().to_string(),
"sh".to_string(),
"-c".to_string(),
format!("for p in $(pgrep -f 'export ANYBOT_JOB={job}'); do pkill -9 -P $p; pkill -9 -P $p; kill -9 $p 2>/dev/null; done; true"),
]);
argv
}
/// Put a file into a bot's folder on the box (only when the box is already running).
pub fn copy_in(local: &str, bot_id: u64, name: &str) -> Result<String, String> {
if !run_state().is_running() {
return Err("Bot 的電腦沒有在執行".into());
}
// The name arrives from uploads: keep only its file part, so no `../` can steer the
// root-run `docker cp` out of this bot's attachments folder.
let safe = std::path::Path::new(name)
.file_name()
.map(|n| n.to_string_lossy().chars().filter(|c| *c != '\0').collect::<String>())
.filter(|n| !n.is_empty() && n != "." && n != "..")
.ok_or_else(|| "檔案名稱無效".to_string())?;
let dir = format!("{}/附件", bot_dir(bot_id));
dk(&["exec", "-u", "bot", container(), "mkdir", "-p", &dir])?;
let dst = format!("{dir}/{safe}");
dk(&["cp", local, &format!("{c}:{dst}", c = container())])?;
let _ = dk(&["exec", container(), "chown", "bot:bot", &dst]);
Ok(dst)
}
/// Copy a file out of the box to this Mac (for showing the person). Returns the local path.
pub fn copy_out(path_in_box: &str) -> Result<PathBuf, String> {
ensure_running()?;
let name = std::path::Path::new(path_in_box).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "file".into());
let dir = std::env::temp_dir().join(format!("anybot-from-box-{}", chrono::Utc::now().timestamp_millis()));
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
let local = dir.join(&name);
dk(&["cp", &format!("{}:{path_in_box}", container()), &local.display().to_string()]).map_err(|e| format!("拿不到檔案:{e}"))?;
Ok(local)
}
// ---------- the box's screen (xdotool) ----------
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
// One `docker exec` per step: the xdotool commands, then (for the bot) a short wait and the
// picture of the result, in one shell script. Each exec costs 50–200 ms locally and more on a
// Docker host elsewhere; a click with a modifier used to be four of them, plus one more for the
// screenshot the bot then had to ask for.
/// The desktop as an image (full size).
pub fn grab() -> Result<image::RgbImage, String> {
let png = dk_timeout(&["exec", container(), "import", "-window", "root", "-display", DISPLAY, "png:-"], 10_000)?;
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
let img = image::load_from_memory(&png).map(|i| i.to_rgb8()).map_err(|e| format!("截圖讀不出來:{e}"))?;
shared().lock().unwrap_or_else(|e| e.into_inner()).size = Some(img.dimensions());
Ok(img)
}
/// The latest picture of the box's screen, small (for the 電腦 card). While the box runs it is
/// taken now (and kept on disk); while it sleeps the last one taken is shown.
pub fn thumbnail(width: u32) -> Option<image::RgbaImage> {
let path = data_dir().join("last-screen.png");
if let Some(img) = frame() {
let h = (img.height() as f32 * width as f32 / img.width().max(1) as f32).round() as u32;
let small = image::imageops::resize(&img, width, h.max(1), image::imageops::FilterType::Triangle);
let _ = std::fs::create_dir_all(data_dir());
let _ = small.save(&path);
return Some(small);
}
image::open(&path).ok().map(|i| i.to_rgba8())
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
/// The screen now, for a small picture (no waking: None when the box isn't running). People watch
/// it live through VNC (vnc.rs).
pub fn frame() -> Option<image::RgbaImage> {
if !run_state().is_running() {
return None;
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
let jpeg = dk_timeout(&["exec", container(), "import", "-window", "root", "-display", DISPLAY, "-quality", "80", "jpeg:-"], 5_000).ok()?;
image::load_from_memory(&jpeg).ok().map(|i| i.to_rgba8())
}
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
/// The desktop's size in pixels (asked once per box).
fn box_size() -> Result<(u32, u32), String> {
if let Some(s) = shared().lock().unwrap_or_else(|e| e.into_inner()).size {
return Ok(s);
}
let out = dk(&["exec", "-u", "bot", "-e", "DISPLAY=:1", container(), "xdotool", "getdisplaygeometry"])?;
let mut it = out.split_whitespace().filter_map(|n| n.parse::<u32>().ok());
let size = match (it.next(), it.next()) {
(Some(w), Some(h)) if w > 0 && h > 0 => (w, h),
_ => return Err("讀不到這台電腦的螢幕大小".to_string()),
};
shared().lock().unwrap_or_else(|e| e.into_inner()).size = Some(size);
Ok(size)
}
/// Desktop pixels per picture pixel: the bot's pictures are the desktop fitted to 1024 wide,
/// always the same size, so its coordinates map without remembering a screenshot.
fn box_scale() -> Result<(f64, f64), String> {
let (w, h) = box_size()?;
let (fw, fh, _) = crate::computer::fit(w, h);
Ok((w as f64 / fw.max(1) as f64, h as f64 / fh.max(1) as f64))
}
/// A step on the box's screen as a shell script (for `sh -c` inside the box, as the bot user),
/// the text it reads on stdin (typing), and what to tell the bot. Coordinates are picture pixels
/// times `scale` (desktop pixels per picture pixel; 1 for a person's viewer, which shows the
/// desktop at full size).
pub fn box_script(name: &str, v: &Value, scale: (f64, f64)) -> Result<(String, Option<String>, String), String> {
let pt = |p: &Value| -> Result<(i64, i64), String> {
let (Some(x), Some(y)) = (p["x"].as_f64(), p["y"].as_f64()) else { return Err("請給 x 和 y(截圖上的位置)".into()) };
Ok(((x * scale.0).round() as i64, (y * scale.1).round() as i64))
};
let at = |p: &Value| format!("({}, {})", p["x"].as_f64().unwrap_or(0.0), p["y"].as_f64().unwrap_or(0.0));
if v["ref"].as_str().map(|r| !r.trim().is_empty()).unwrap_or(false) {
return Err("這台是 Linux 桌面,沒有 App 的編號(ref):請用截圖上的 x、y。".into());
}
let mods = mods_of(v);
let down: String = mods.iter().map(|m| format!(" keydown {m}")).collect();
let up: String = mods.iter().rev().map(|m| format!(" keyup {m}")).collect();
// Never `mousemove --sync`: it waits forever when the pointer is already there (a second
// click on the same button hung the box for two minutes).
let mv = |(x, y): (i64, i64)| format!("mousemove {x} {y} sleep 0.03");
let wrap = |body: String| -> String {
if mods.is_empty() {
format!("xdotool {body}")
} else {
// Keys held for the step are let go even when the step fails (a stuck ctrl spoils
// every later click and keystroke in the box).
format!("xdotool{down} {body}; rc=$?; xdotool{up}; [ $rc = 0 ]")
}
};
let button = button_no(v["button"].as_str().unwrap_or("left"));
Ok(match name {
"computer_use_click" => {
let n = v["count"].as_u64().unwrap_or(1).clamp(1, 3);
(wrap(format!("{} click --repeat {n} --delay {CLICK_REPEAT_MS} {button}", mv(pt(v)?))), None, format!("已{}{} {}。", if button == "3" { "右鍵" } else { "" }, if n >= 2 { format!("連點 {n} 下") } else { "點".into() }, at(v)))
}
"computer_use_mouse_move" => (format!("xdotool {}", mv(pt(v)?)), None, format!("已把滑鼠移到 {}。", at(v))),
"computer_use_mouse_button" => {
let go = if v["x"].is_null() { String::new() } else { format!("{} ", mv(pt(v)?)) };
let release = v["action"].as_str() == Some("release");
(format!("xdotool {go}{} {button}", if release { "mouseup" } else { "mousedown" }), None, (if release { "已放開滑鼠鍵。" } else { "已按住滑鼠鍵(記得之後放開)。" }).to_string())
}
"computer_use_scroll" => {
let amount = crate::computer::scroll_amount(v["amount"].as_i64());
let dir = v["direction"].as_str().unwrap_or("").trim().to_lowercase();
let wheel = match dir.as_str() {
"up" => "4",
"down" => "5",
"left" => "6",
"right" => "7",
_ => return Err(format!("方向「{dir}」要是 up、down、left 或 right")),
};
let go = if v["x"].is_null() { String::new() } else { format!("{} ", mv(pt(v)?)) };
(wrap(format!("{go}click --repeat {amount} --delay 20 {wheel}")), None, format!("已捲動 {dir} ×{amount}。"))
}
"computer_use_drag" => {
// A point that fails to parse is an error, not a silently dropped waypoint.
let raw = v["path"].as_array().ok_or("拖曳要給 path:一個點陣,每個點有數字 x 和 y(用截圖上的座標)")?;
let path: Vec<(i64, i64)> = raw.iter().map(pt).collect::<Result<_, _>>()?;
if path.len() < 2 {
return Err("拖曳至少要兩個點(每個點要有數字 x 和 y,用截圖上的座標)".into());
}
let mut body = format!("{} mousedown {button} sleep 0.04", mv(path[0]));
let steps: Vec<(f64, f64)> = crate::computer::drag_points(&path.iter().map(|(x, y)| (*x as f64, *y as f64)).collect::<Vec<_>>());
for (x, y) in steps {
body.push_str(&format!(" mousemove {} {} sleep 0.008", x.round() as i64, y.round() as i64));
}
body.push_str(&format!(" mouseup {button}"));
(wrap(body), None, format!("已從 {} 拖曳到 {}。", at(&raw[0]), at(&raw[raw.len() - 1])))
}
"computer_use_typing" => {
let text = v["value"].as_str().unwrap_or("").to_string();
let submit = v["submit"].as_bool().unwrap_or(false);
if text.is_empty() && !submit {
return Err("請給要打的字".into());
}
let mut script = String::new();
if !text.is_empty() {
script.push_str(&type_script(&text));
}
if submit {
script.push_str(" && sleep 0.06 && xdotool key --clearmodifiers Return");
}
let script = script.trim_start_matches(" && ").to_string();
(script, Some(text.clone()).filter(|t| !t.is_empty()), format!("已打字({} 個字){}。", text.chars().count(), if submit { ",再按了 return" } else { "" }))
}
"computer_use_press_key" => {
let keys = v["key"].as_str().unwrap_or("");
let hold = v["hold_duration_ms"].as_u64().unwrap_or(0).min(5_000);
let mut parts: Vec<String> = vec![];
for k in keys.split_whitespace() {
match combo_to_x(k) {
Ok(c) if hold > 0 => parts.push(format!("keydown {c} sleep {:.3} keyup {c}", hold as f64 / 1000.0)),
Ok(c) => parts.push(format!("key --clearmodifiers {c}")),
// 「@」「?」: a character, typed.
Err(_) if k.chars().count() == 1 => parts.push(format!("type --clearmodifiers -- {}", crate::tools::sh_quote(k))),
Err(e) => return Err(e),
}
}
if parts.is_empty() {
return Err("請給要按的鍵".into());
}
(format!("xdotool {}", parts.join(" sleep 0.02 ")), None, format!("已按 {keys}。"))
}
other => return Err(format!("沒有「{other}」這個工具")),
})
}
/// Typing, as a script reading the text on stdin. Plain ASCII goes key by key; anything else
/// (Chinese, emoji) is pasted, because X keyboards can't type characters they have no key for in
/// every program: both selections hold it (xclip keeps serving after the exec ends, setsid), and
/// terminals paste with shift+Insert, everything else with ctrl+v.
fn type_script(text: &str) -> String {
if text.is_ascii() {
return format!("xdotool type --clearmodifiers --delay {TYPE_DELAY_MS} --file -");
}
"cat > /tmp/anybot-paste.txt && for s in clipboard primary; do setsid xclip -selection $s -i /tmp/anybot-paste.txt </dev/null >/dev/null 2>&1 & done; sleep 0.15; \
case \"$(xprop -id $(xdotool getactivewindow) WM_CLASS 2>/dev/null | tr A-Z a-z)\" in *term*) k=shift+Insert;; *) k=ctrl+v;; esac; xdotool key --clearmodifiers $k"
.to_string()
}
/// Run a step's script in the box; with `picture`, wait for the screen to settle and return it.
fn box_exec(script: &str, stdin: Option<&str>, picture: bool) -> Result<Option<image::RgbImage>, String> {
use std::io::{Read, Write};
let full = if picture { format!("{{ {script}; }} || exit $?; sleep {:.2}; import -window root png:- 2>/dev/null || true", crate::computer::SETTLE_MS as f64 / 1000.0) } else { script.to_string() };
let mut child = docker()
.args(["exec", "-i", "-u", "bot", "-e", "DISPLAY=:1", container(), "sh", "-c", &full])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|e| format!("找不到 docker:{e}"))?;
{
let mut si = child.stdin.take().ok_or("寫不進 Bot 的電腦")?;
if let Some(t) = stdin {
si.write_all(t.as_bytes()).map_err(|e| e.to_string())?;
}
}
let mut so = child.stdout.take().ok_or("沒有輸出")?;
let mut se = child.stderr.take().ok_or("沒有輸出")?;
let out_t = std::thread::spawn(move || {
let mut v = vec![];
let _ = so.read_to_end(&mut v);
v
});
let err_t = std::thread::spawn(move || {
let mut v = vec![];
let _ = se.read_to_end(&mut v);
v
});
let start = Instant::now();
let status = loop {
if let Ok(Some(st)) = child.try_wait() {
break st;
}
if start.elapsed() > Duration::from_secs(30) {
let _ = child.kill();
let _ = child.wait();
return Err("這台電腦太久沒有回應".into());
}
std::thread::sleep(Duration::from_millis(5));
};
let out = out_t.join().unwrap_or_default();
let err = err_t.join().unwrap_or_default();
if !status.success() {
let e = String::from_utf8_lossy(&err).trim().to_string();
return Err(if e.is_empty() { "這一步沒有成功".into() } else { format!("xdotool:{e}") });
}
if !picture || out.is_empty() {
return Ok(None);
}
let img = image::load_from_memory(&out).map(|i| i.to_rgb8()).ok();
if let Some(i) = &img {
shared().lock().unwrap_or_else(|e| e.into_inner()).size = Some(i.dimensions());
}
Ok(img)
}
/// The bot's picture of the box: fitted to 1024 wide, JPEG, base64.
fn picture_of(img: image::RgbImage) -> Result<(String, String), String> {
let (w, h) = img.dimensions();
let (fw, fh, resize) = crate::computer::fit(w, h);
let img = if resize { image::imageops::resize(&img, fw, fh, image::imageops::FilterType::Triangle) } else { img };
let jpeg = crate::computer::encode_jpeg(&img).ok_or("截圖太大了,沒辦法送出")?;
Ok((format!("螢幕截圖({fw}×{fh},Linux 桌面)。座標用這張圖上的像素。"), base64::engine::general_purpose::STANDARD.encode(jpeg)))
}
/// Programs people name, as the box knows them.
fn box_program(app: &str) -> Option<&'static str> {
let a = app.trim().to_lowercase();
if ["chrome", "chromium", "google chrome", "browser", "瀏覽器", "網頁瀏覽器"].iter().any(|n| a == *n) {
return Some("chromium");
}
if ["terminal", "終端機", "xterm", "shell", "命令列"].iter().any(|n| a == *n) {
return Some("xterm");
}
None
}
/// Open a program on the box, or bring its window to the front. The browser is AnyBot's
/// browser there (the one browser_* drive), never a second copy fighting over its profile.
fn open_on_box(app: &str) -> Result<String, String> {
let name = app.trim();
if name.is_empty() {
return Err("請給要打開的程式名稱".into());
}
let prog = match box_program(name) {
Some(p) => p.to_string(),
None if name.chars().all(|c| c.is_ascii_alphanumeric() || "._-".contains(c)) => name.to_lowercase(),
None => return Err(format!("這台是 Linux 桌面,沒有「{name}」。可以打開的例如 chromium(瀏覽器)、xterm(終端機),或用指令工具安裝。")),
};
if prog == "chromium" {
browser_port()?;
}
let q = crate::tools::sh_quote(&prog);
// Front if it has a window; else start it and wait (up to 8 s) for one.
let script = format!(
"w=$(xdotool search --onlyvisible --classname {q} 2>/dev/null | tail -1); \
if [ -z \"$w\" ]; then command -v {q} >/dev/null || {{ echo \"沒有 {prog} 這個程式\" >&2; exit 3; }}; (setsid {q} >/dev/null 2>&1 &); \
for i in $(seq 1 40); do sleep 0.2; w=$(xdotool search --onlyvisible --classname {q} 2>/dev/null | tail -1); [ -n \"$w\" ] && break; done; fi; \
[ -n \"$w\" ] && xdotool windowactivate $w 2>/dev/null; true"
);
box_exec(&script, None, false)?;
Ok(format!("已打開「{prog}」,放在最前面。"))
}
/// Run a computer_use tool on the box. Returns (text, screenshot as base64 JPEG).
pub fn computer_run(name: &str, v: &Value) -> Result<(String, Option<String>), String> {
ensure_running()?;
let _busy = Busy::new(true);
match name {
"computer_use_check_permissions" => return Ok(("這台電腦不需要另外的權限:可以截圖,也可以操作滑鼠和鍵盤。".into(), None)),
"computer_use_screenshot" => return picture_of(grab()?).map(|(t, img)| (t, Some(img))),
"computer_use_read_app" => {
return Err("這台是 Linux 桌面,沒有讀 App 文字的方式:用 computer_use_screenshot 看畫面、用座標操作。網頁請用 browser_* 工具(有編號,比座標準)。".into());
}
_ => {}
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
if HELD_BY_PERSON.load(Ordering::SeqCst) || crate::vnc::person_active() {
return Err("使用者正在自己操作這台電腦。請等他操作完(停下幾秒後)再繼續,或先截圖看看畫面。".into());
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
}
let (note, picture) = if name == "computer_use_open_app" {
let note = open_on_box(v["app"].as_str().unwrap_or(""))?;
std::thread::sleep(Duration::from_millis(crate::computer::SETTLE_MS));
(note, grab().ok())
} else {
let (script, stdin, note) = box_script(name, v, box_scale()?)?;
let picture = box_exec(&script, stdin.as_deref(), name != "computer_use_mouse_button")?;
(note, picture)
};
match picture.map(picture_of).transpose()? {
Some((t, img)) => Ok((format!("{note}\n{t}"), Some(img))),
None => Ok((note, None)),
}
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
// ---------- text a person types in a viewer ----------
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
/// Text from a person's viewer (any language): typed or pasted into the window in front, like
/// the bot's typing. (Their mouse and keys go through VNC itself.)
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
pub fn viewer_text(text: &str) {
if !text.is_empty() {
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
let _ = box_exec(&type_script(text), Some(text), false);
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
}
touch();
}
fn key_to_x(name: &str) -> Option<String> {
let n = name.trim().to_lowercase();
let s = match n.as_str() {
"return" | "enter" => "Return",
"tab" => "Tab",
"space" => "space",
"delete" | "backspace" => "BackSpace",
"forwarddelete" | "del" => "Delete",
"escape" | "esc" => "Escape",
"home" => "Home",
"end" => "End",
"pageup" => "Prior",
"pagedown" => "Next",
"up" | "arrowup" => "Up",
"down" | "arrowdown" => "Down",
"left" | "arrowleft" => "Left",
"right" | "arrowright" => "Right",
"-" | "minus" => "minus",
"=" | "equal" => "equal",
"[" => "bracketleft",
"]" => "bracketright",
";" => "semicolon",
"'" => "apostrophe",
"," | "comma" => "comma",
"." | "period" => "period",
"/" | "slash" => "slash",
"\\" | "backslash" => "backslash",
"`" | "grave" => "grave",
f if f.starts_with('f') && f[1..].parse::<u8>().map(|x| (1..=12).contains(&x)).unwrap_or(false) => return Some(f.to_uppercase()),
c if c.chars().count() == 1 && c.chars().all(|ch| ch.is_ascii_alphanumeric()) => return Some(c.to_string()),
_ => return None,
};
Some(s.into())
}
fn modifier_to_x(m: &str) -> Option<&'static str> {
// On Linux the shortcuts people mean by cmd (copy, paste, select all) use ctrl.
match m.trim().to_lowercase().as_str() {
"cmd" | "command" | "meta" | "control" | "ctrl" => Some("ctrl"),
"shift" => Some("shift"),
"option" | "alt" | "opt" => Some("alt"),
"super" | "win" | "windows" => Some("super"),
_ => None,
}
}
/// "cmd+shift+t" → "ctrl+shift+t"
pub fn combo_to_x(combo: &str) -> Result<String, String> {
let parts: Vec<&str> = combo.split('+').map(str::trim).filter(|p| !p.is_empty()).collect();
let Some((key, mods)) = parts.split_last() else { return Err("請給要按的鍵".into()) };
let mut out: Vec<String> = vec![];
for m in mods {
out.push(modifier_to_x(m).ok_or_else(|| format!("不認得「{m}」這個修飾鍵"))?.to_string());
}
out.push(key_to_x(key).ok_or_else(|| format!("不認得「{key}」這個鍵"))?);
Ok(out.join("+"))
}
fn mods_of(v: &Value) -> Vec<String> {
v["modifier_keys"].as_array().map(|a| a.iter().filter_map(|m| m.as_str().and_then(modifier_to_x)).map(String::from).collect()).unwrap_or_default()
}
fn button_no(b: &str) -> &'static str {
match b {
"right" => "3",
"middle" => "2",
_ => "1",
}
}
// ---------- the browser viewer (noVNC) ----------
fn published_port() -> u16 {
dk(&["port", container(), &NOVNC_PORT.to_string()])
.ok()
.and_then(|s| s.lines().next().and_then(|l| l.rsplit(':').next()).and_then(|p| p.trim().parse().ok()))
.unwrap_or(HOST_PORT)
}
/// The noVNC address on this Mac (through an SSH tunnel when the box is on another host).
pub fn viewer_url(control: bool) -> Result<String, String> {
ensure_running()?;
let port = published_port();
let host = docker_host();
let local_port = if let Some(target) = host.strip_prefix("ssh://") {
let mut s = shared().lock().unwrap();
let alive = s.tunnel.as_mut().map(|c| matches!(c.try_wait(), Ok(None))).unwrap_or(false);
if !alive {
let lp = free_port(HOST_PORT + 100);
let child = Command::new("ssh")
.env("PATH", tool_path())
.args(["-N", "-o", "BatchMode=yes", "-o", "ExitOnForwardFailure=yes", "-L", &format!("{lp}:127.0.0.1:{port}"), target])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.map_err(|e| format!("開不了 SSH 通道:{e}"))?;
s.tunnel = Some(child);
s.host_port = lp;
drop(s);
std::thread::sleep(Duration::from_millis(800));
lp
} else {
s.host_port
}
} else {
port
};
// noVNC signs in with the box's VNC password (x11vnc refuses the keyless login now).
Ok(format!("http://127.0.0.1:{local_port}/vnc.html?autoconnect=1&resize=scale&reconnect=1&show_dot=1&password={}{}", vnc_password(), if control { "" } else { "&view_only=1" }))
}
pub fn shutdown() {
if let Some(mut t) = shared().lock().unwrap().tunnel.take() {
let _ = t.kill();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn secret_names_follow_the_original_rule() {
assert!(valid_secret_name("CRM_API_TOKEN"));
assert!(valid_secret_name("A1"));
assert!(!valid_secret_name("crm_token"));
assert!(!valid_secret_name("1ABC"));
assert!(!valid_secret_name("_ABC"));
assert!(!valid_secret_name("AB-C"));
}
#[test]
fn shortcuts_become_x_keys() {
assert_eq!(combo_to_x("cmd+c").unwrap(), "ctrl+c");
assert_eq!(combo_to_x("cmd+shift+t").unwrap(), "ctrl+shift+t");
assert_eq!(combo_to_x("return").unwrap(), "Return");
assert_eq!(combo_to_x("option+f4").unwrap(), "alt+F4");
assert!(combo_to_x("hyper+x").is_err());
}
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
#[test]
fn a_step_is_one_script() {
let s = (1.25, 1.25);
// Picture pixels → desktop pixels; never `mousemove --sync` (it hangs when the pointer is
// already there).
let (sc, input, note) = box_script("computer_use_click", &serde_json::json!({"x": 512, "y": 320}), s).unwrap();
assert_eq!(sc, "xdotool mousemove 640 400 sleep 0.03 click --repeat 1 --delay 80 1");
assert!(input.is_none() && note.contains("(512, 320)"));
// Keys held for a step are let go even when it fails.
let (sc, _, _) = box_script("computer_use_click", &serde_json::json!({"x": 8, "y": 8, "modifier_keys": ["cmd", "shift"], "count": 2}), s).unwrap();
assert!(sc.starts_with("xdotool keydown ctrl keydown shift mousemove 10 10") && sc.ends_with("rc=$?; xdotool keyup shift keyup ctrl; [ $rc = 0 ]"), "{sc}");
assert!(!sc.contains("--sync"));
// Text goes in on stdin (no quoting to get wrong); return after it when asked.
let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "echo 'hi' && ls", "submit": true}), s).unwrap();
assert_eq!(input.as_deref(), Some("echo 'hi' && ls"));
assert!(sc.starts_with("xdotool type --clearmodifiers --delay 12 --file -") && sc.ends_with("xdotool key --clearmodifiers Return"), "{sc}");
let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "中文"}), s).unwrap();
assert!(sc.contains("xclip") && input.as_deref() == Some("中文"));
// Keys by name, a character typed, several in order.
let (sc, _, _) = box_script("computer_use_press_key", &serde_json::json!({"key": "slash @ cmd+a"}), s).unwrap();
assert_eq!(sc, "xdotool key --clearmodifiers slash sleep 0.02 type --clearmodifiers -- '@' sleep 0.02 key --clearmodifiers ctrl+a");
// A drag walks its path in small steps and lets go at the end.
let (sc, _, _) = box_script("computer_use_drag", &serde_json::json!({"path": [{"x": 0, "y": 0}, {"x": 48, "y": 0}]}), s).unwrap();
assert!(sc.starts_with("xdotool mousemove 0 0 sleep 0.03 mousedown 1") && sc.contains("mousemove 60 0") && sc.ends_with("mouseup 1"), "{sc}");
// No app numbers on this desktop: said, not guessed.
assert!(box_script("computer_use_click", &serde_json::json!({"ref": "e3"}), s).is_err());
assert!(box_script("computer_use_scroll", &serde_json::json!({"direction": "sideways"}), s).is_err());
}
/// Uses the real Docker: `cargo test --release -- --ignored box_screen_is_exact --nocapture`.
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
/// The bot's clicks land where its picture says, the same spot twice doesn't hang, typing (ASCII, Chinese, a key by name) reaches the app,
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
/// and every step comes back with the screen after it.
#[test]
#[ignore]
fn box_screen_is_exact() {
ensure_running().expect("box starts");
let pointer = || {
let out = dk(&["exec", "-u", "bot", "-e", "DISPLAY=:1", container(), "xdotool", "getmouselocation"]).unwrap();
let num = |k: &str| out.split_whitespace().find_map(|w| w.strip_prefix(&format!("{k}:")).and_then(|v| v.parse::<i64>().ok())).unwrap();
(num("x"), num("y"))
};
let (w, h) = box_size().unwrap();
let (fw, fh, _) = crate::computer::fit(w, h);
println!("desktop {w}×{h}, picture {fw}×{fh}");
// A move, in the bot's picture pixels, with no screenshot taken first.
shared().lock().unwrap().size = None;
let (txt, img) = computer_run("computer_use_mouse_move", &serde_json::json!({"x": fw / 2, "y": fh / 2})).unwrap();
assert!(img.is_some(), "a picture comes back: {txt}");
assert_eq!(pointer(), ((w / 2) as i64, (h / 2) as i64));
// The same spot twice: quick, not a two-minute hang.
let t = Instant::now();
computer_run("computer_use_click", &serde_json::json!({"x": 40, "y": 40})).unwrap();
computer_run("computer_use_click", &serde_json::json!({"x": 40, "y": 40})).unwrap();
println!("two clicks with pictures: {} ms", t.elapsed().as_millis());
assert!(t.elapsed() < Duration::from_secs(6));
assert_eq!(pointer(), ((40.0 * w as f64 / fw as f64).round() as i64, (40.0 * h as f64 / fh as f64).round() as i64));
// Typing reaches the app in front: a fresh terminal, ASCII then Chinese, a key by name, return.
let _ = dk(&["exec", "-u", "bot", container(), "rm", "-f", "/home/bot/exact.txt"]);
let (txt, img) = computer_run("computer_use_open_app", &serde_json::json!({"app": "終端機"})).unwrap();
println!("{txt}");
assert!(img.is_some());
computer_run("computer_use_typing", &serde_json::json!({"value": "echo 'a-b' "})).unwrap();
computer_run("computer_use_typing", &serde_json::json!({"value": "中文"})).unwrap();
computer_run("computer_use_press_key", &serde_json::json!({"key": "space slash"})).unwrap();
let (txt, img) = computer_run("computer_use_typing", &serde_json::json!({"value": " > /home/bot/exact.txt", "submit": true})).unwrap();
assert!(img.is_some() && txt.contains("return"));
std::thread::sleep(Duration::from_millis(500));
let out = dk(&["exec", "-u", "bot", container(), "cat", "/home/bot/exact.txt"]).unwrap_or_default();
println!("typed: {out}");
assert_eq!(out.trim(), "a-b 中文 /");
let _ = box_exec("xdotool key --clearmodifiers ctrl+d", None, false);
}
/// Uses the real Docker: `cargo test --release -- --ignored box_end_to_end --nocapture`.
#[test]
#[ignore]
fn box_end_to_end() {
use crate::tools::{run_on, Place, RunIds, Target};
let cancel = AtomicBool::new(false);
let ids = RunIds::default();
ensure_running().expect("box starts");
assert!(run_state().is_running());
let t = Target { name: "Bot 的電腦".into(), place: Place::Box { dir: bot_dir(999) } };
let (out, ok) = run_on("shell", r#"{"command":"pwd && whoami && echo $DISPLAY"}"#, &t, &cancel, &ids);
println!("shell: {out}");
assert!(ok && out.contains("/home/bot/bot-999") && out.contains("bot") && out.contains(":1"));
let (out, ok) = run_on("edit_file", r#"{"path":"notes/hello.txt","old_text":"","new_text":"第一行\nhello world\n"}"#, &t, &cancel, &ids);
assert!(ok, "{out}");
let (out, ok) = run_on("edit_file", r#"{"path":"notes/hello.txt","old_text":"hello world","new_text":"你好 世界"}"#, &t, &cancel, &ids);
assert!(ok, "{out}");
let (out, ok) = run_on("read_file", r#"{"path":"notes/hello.txt"}"#, &t, &cancel, &ids);
println!("read: {out}");
assert!(ok && out.contains("你好 世界") && !out.contains("hello world"));
let (out, ok) = run_on("search", r#"{"pattern":"世界","path":"."}"#, &t, &cancel, &ids);
assert!(ok && out.contains("hello.txt"), "{out}");
// secrets reach commands, never shown by the app
set_secret("ANYBOT_TEST_TOKEN", "s3cret-value").unwrap();
let (out, ok) = run_on("shell", r#"{"command":"test \"$ANYBOT_TEST_TOKEN\" = s3cret-value && echo has-secret"}"#, &t, &cancel, &ids);
remove_secret("ANYBOT_TEST_TOKEN");
assert!(ok && out.contains("has-secret"), "{out}");
// background command takes input
let (out, ok) = run_on("shell", r#"{"command":"read -p 'name? ' n; echo got:$n","timeout_ms":1500}"#, &t, &cancel, &ids);
assert!(ok, "{out}");
let id: u64 = out.split('#').nth(1).and_then(|s| s.split(|c: char| !c.is_ascii_digit()).next()).and_then(|s| s.parse().ok()).expect("terminal id");
let (out, ok) = run_on("write_terminal", &format!(r#"{{"id":{id},"input":"daniel\n"}}"#), &t, &cancel, &ids);
println!("write_terminal: {out}");
assert!(ok && out.contains("got:daniel"), "{out}");
// screen
let (txt, img) = computer_run("computer_use_screenshot", &serde_json::json!({})).unwrap();
println!("{txt}");
assert!(img.is_some());
run_on("shell", r#"{"command":"(xterm >/dev/null 2>&1 &) ; sleep 1.5"}"#, &t, &cancel, &ids);
// Click the middle of that window, wherever it opened (other windows — a browser left
// open — may cover the corner), in the screenshot's pixels. (Its shell renames it, so it
// is found by its class: the newest xterm.)
let (geo, _) = run_on("shell", r#"{"command":"W=$(DISPLAY=:1 xdotool search --onlyvisible --class xterm | tail -1); DISPLAY=:1 xdotool getwindowgeometry --shell $W | tail -n +2 | head -4 | tr '\n' ' '"}"#, &t, &cancel, &ids);
let num = |k: &str| geo.split_whitespace().find_map(|w| w.strip_prefix(&format!("{k}=")).and_then(|v| v.parse::<f64>().ok())).unwrap_or(0.0);
let (cx, cy) = (num("X") + num("WIDTH") / 2.0, num("Y") + num("HEIGHT") / 2.0);
let k = 1024.0 / 1280.0;
computer_run("computer_use_click", &serde_json::json!({"x": cx * k, "y": cy * k})).unwrap();
computer_run("computer_use_typing", &serde_json::json!({"value": "echo 中文輸入OK > /home/bot/typed.txt"})).unwrap();
computer_run("computer_use_press_key", &serde_json::json!({"key": "return"})).unwrap();
std::thread::sleep(Duration::from_millis(800));
let (out, _) = run_on("shell", r#"{"command":"cat /home/bot/typed.txt"}"#, &t, &cancel, &ids);
println!("typed: {out}");
assert!(out.contains("中文輸入OK"), "{out}");
// snapshot (with the browser and its caches in the home folder), then restore — which
// rebuilds the whole box, so only on a box meant for it (ANYBOT_TEST_RESTORE=1).
let size = save_snapshot().expect("snapshot saves even while the browser writes");
println!("snapshot: {}", human(size));
assert!(size > 0);
if std::env::var("ANYBOT_TEST_RESTORE").as_deref() != Ok("1") {
let _ = run_on("shell", r#"{"command":"rm -rf /home/bot/bot-999 /home/bot/typed.txt"}"#, &t, &cancel, &ids);
return;
}
run_on("shell", r#"{"command":"rm -rf notes"}"#, &t, &cancel, &ids);
rebuild(Rebuild::Restore).unwrap();
let start = Instant::now();
while shared().lock().unwrap().rebuilding && start.elapsed() < Duration::from_secs(180) {
std::thread::sleep(Duration::from_millis(500));
}
println!("after restore: {:?}", run_state());
let (out, ok) = run_on("read_file", r#"{"path":"notes/hello.txt"}"#, &t, &cancel, &ids);
assert!(ok && out.contains("你好 世界"), "{out}");
let _ = run_on("shell", r#"{"command":"rm -rf /home/bot/bot-999 /home/bot/typed.txt"}"#, &t, &cancel, &ids);
}
/// `cargo test --release -- --ignored box_traffic_via_mac --nocapture`
#[test]
#[ignore]
fn box_traffic_via_mac() {
use crate::tools::{run_on, Place, RunIds, Target};
let cancel = AtomicBool::new(false);
ensure_running().unwrap();
set_egress_via_mac(true).unwrap();
let before = crate::egress::CONNECTIONS.load(Ordering::SeqCst);
let t = Target { name: "Bot 的電腦".into(), place: Place::Box { dir: bot_dir(997) } };
let (out, ok) = run_on("shell", r#"{"command":"curl -sS -o /dev/null -w '%{http_code}' https://example.com"}"#, &t, &cancel, &RunIds::default());
set_egress_via_mac(false).unwrap();
println!("{out} via mac: {}", crate::egress::CONNECTIONS.load(Ordering::SeqCst) - before);
assert!(ok && out.contains("200"), "{out}");
assert!(crate::egress::CONNECTIONS.load(Ordering::SeqCst) > before);
}
/// `cargo test --release -- --ignored box_update_keeps_files --nocapture`
#[test]
#[ignore]
fn box_update_keeps_files() {
use crate::tools::{run_on, Place, RunIds, Target};
let cancel = AtomicBool::new(false);
let ids = RunIds::default();
ensure_running().unwrap();
let t = Target { name: "Bot 的電腦".into(), place: Place::Box { dir: bot_dir(998) } };
let (o, ok) = run_on("shell", r#"{"command":"echo keep-me > kept.txt && sudo apt-get update -qq >/dev/null 2>&1 && sudo apt-get install -y -qq cowsay >/dev/null 2>&1; ls /usr/games/cowsay","timeout_ms":180000}"#, &t, &cancel, &ids);
assert!(ok, "{o}");
rebuild(Rebuild::Update).unwrap();
assert!(rebuild(Rebuild::Reset).is_err(), "only one rebuild at a time");
let start = Instant::now();
while shared().lock().unwrap().rebuilding && start.elapsed() < Duration::from_secs(600) {
std::thread::sleep(Duration::from_millis(500));
}
for l in log_lines() {
println!("{l}");
}
assert!(run_state().is_running(), "{:?}", run_state());
let (out, ok) = run_on("shell", r#"{"command":"cat kept.txt; ls /usr/games/cowsay 2>&1 || echo no-cowsay; cat /etc/anybot/menu | head -1"}"#, &t, &cancel, &ids);
println!("{out}");
assert!(ok && out.contains("keep-me") && out.contains("no-cowsay") && out.contains("Bot 的電腦"));
let _ = run_on("shell", r#"{"command":"cd .. && rm -rf bot-998"}"#, &t, &cancel, &ids);
}
#[test]
fn schedule_window_is_one_hour() {
let t = |s: &str| chrono::NaiveTime::parse_from_str(s, "%H:%M").unwrap();
assert!(in_schedule_window("03:00", t("03:00")));
assert!(in_schedule_window("03:00", t("03:59")));
assert!(!in_schedule_window("03:00", t("04:00")));
assert!(!in_schedule_window("03:00", t("02:59")));
assert!(in_schedule_window("23:30", t("00:10")));
assert!(!in_schedule_window("bad", t("00:10")));
assert_eq!(wanted_version(), "3");
}
#[test]
fn wrapper_runs_as_the_bot_in_its_folder() {
let w = exec_wrapper("/home/bot/bot-3", "ls -la", false, "");
assert!(w.contains("exec -u bot"));
assert!(w.contains(container()));
assert!(w.contains("/home/bot/bot-3"));
}
}