diff --git a/Makefile b/Makefile index 97bfa9c..42720e0 100644 --- a/Makefile +++ b/Makefile @@ -88,6 +88,8 @@ ifeq ($(UNAME),Darwin) @xcode-select -p >/dev/null 2>&1 && echo " ✓ Xcode Command Line Tools" || echo " ✗ Xcode Command Line Tools：make deps" endif @test -x $(BIN) && echo " ✓ 已建置 $(BIN)" || echo " · 還沒建置：make" + @command -v docker >/dev/null 2>&1 && { docker volume inspect anybot-box-home >/dev/null 2>&1 && echo " ✓ Bot 的電腦的資料碟 anybot-box-home（檔案和登入都在裡面）" || echo " · 沒有 anybot-box-home 資料碟（Bot 的電腦還沒建立過，或被清除過——有快照會在下次建立時自動還原）"; } || true + @d="$${HOME}/Library/Application Support/AnyBot/box/snapshots"; [ -d "$$d" ] || d="$${HOME}/.local/share/AnyBot/box/snapshots"; [ -d "$$d" ] && echo " ✓ 快照 $$(ls "$$d" 2>/dev/null | grep -c '^snapshot-') 份（留最近 7 天＋之前每週一份；重設前也會先存）" || echo " · 還沒有快照（Bot 的電腦每天自動存，重設前也會先存）" install: app ifeq ($(UNAME),Darwin) diff --git a/src/app.rs b/src/app.rs index 009b211..516d3fc 100644 --- a/src/app.rs +++ b/src/app.rs @@ -4032,6 +4032,7 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu - 如果讀到的是登入頁：把登入頁開在那台電腦上，用 hand_over_computer 請使用者登入一次（之後會記住），他按「我完成了」再讀。絕對不要叫使用者去改瀏覽器的設定或選單（例如「允許來自 Apple 事件的 JavaScript」），那不需要。也不要說要從 Chrome 複製登入或 Cookie，沒有這個功能。\n\ - AnyBot 瀏覽器有設定檔（profile），每個有自己的登入。使用者說要換 profile／設定檔／另一個帳號時，用 browser_profiles：list 看有哪些，use 切換，add 新增（新的要登入一次）。也可以在 browser_open / browser_read 帶 profile 參數。使用者想用自己平常的 Chrome（自己的 Chrome 設定檔、已經登入的帳號）時，用 profile「你的瀏覽器」：請使用者在 Chrome 切到想要的設定檔，再 browser_read。\n\ - 你也能像人一樣操作螢幕（{screens}），用在檔案、指令和網頁工具做不到的事（例如操作其他 App）。先 computer_use_screenshot：你會拿到一張每個按鈕、連結、欄位、選單項目都框起來、標了編號的截圖，和同樣編號的清單。一律照編號操作（click／typing／scroll 的 ref，例如 ref 填 12），點下去就是那個東西的正中央；只有圖上沒有編號的東西（圖片裡的一個點、遊戲）才用 x、y。每個動作做完都會附上新的編號截圖，不要另外截圖。結果說「畫面沒有變化」就是沒點到：換一個編號或換方法，絕對不要重複同一步。要在某個 App 裡打字：先 computer_use_open_app，再 computer_use_typing（ref 是輸入框的編號；submit=true 順便按 return）\n\ + - 已經從目前畫面知道接下來的幾個動作時（填一張表：點欄位、打字、下一欄、打字、按 return；一組捷徑），用 computer_use_batch 一次做完：actions 是動作陣列，每個 {{\"action\":\"click\",\"ref\":\"12\"}} 這樣（最多 8 個）。會依序執行，最後只附一張截圖；第 2 步要看第 1 步的結果才知道怎麼做的，就拆開用單個動作。每次看截圖用一句話說畫面顯示什麼、這步成功與否。\n\ {}\ - 不要說「我沒有瀏覽器或工具」：先試這些工具。需要權限時 App 會自己跳出視窗問使用者；工具回報要開什麼設定時，照實轉告。", now.format("%Y-%m-%d %H:%M"), diff --git a/src/app/computers_ui.rs b/src/app/computers_ui.rs index 06c7b22..5c1fed6 100644 --- a/src/app/computers_ui.rs +++ b/src/app/computers_ui.rs @@ -425,6 +425,11 @@ impl App { act = Act::Browser; } }); + // What a step on this box really costs (from the app's own timings). + let (n, exec_ms, total_ms) = cloudbox::step_stats(); + if n > 0 { + hint(ui, format!("最近 {n} 步畫面操作：在電腦上做平均 {exec_ms} ms，整步（含編號畫面）平均 {total_ms} ms"), p); + } } } // 更新 Bot 的電腦: two clicks (the first arms it for 3 s). @@ -467,7 +472,7 @@ impl App { // 重設 Bot 的電腦: from the last snapshot (red, asks first). let snap = cloudbox::snapshot_time().map(|t| t.format("%m/%d %H:%M").to_string()); let desc = match &snap { Some(t) => format!("上次快照：{t}"), None => t("還沒有快照") }; - sec_row_s(ui, p, "重設 Bot 的電腦", &desc, "電腦卡住時可重新開始。它會從你上次儲存的快照重建，最近的變更可能會遺失；沒有快照時會換一台全新的電腦。「儲存快照」把 Bot 的檔案備份到這台電腦，每天也會自動存一份。", |ui| { + sec_row_s(ui, p, "重設 Bot 的電腦", &desc, "電腦卡住時可重新開始。它會從你上次儲存的快照重建，最近的變更可能會遺失；沒有快照時會換一台全新的電腦。「儲存快照」把 Bot 的檔案備份到這台電腦，每天也會自動存一份。重設或還原之前，會先把現在的電腦存成一份快照（存不下就不會動手），之後還能從它找回；快照保留最近 7 天，之前每週各留一份。", |ui| { if ui.add_enabled(!busy_rebuild, danger_button("重設", p)).clicked() { act = Act::Reset; } @@ -682,7 +687,7 @@ impl App { let close = dialog(ctx, "reset-computer", "要重設 AnyBot 的電腦嗎？", 460.0, true, p, |ui| { dialog_body(ui, "reset-computer-body", 320.0, |ui| { let w = if working == 0 { t("所有執行中的工作都將被中斷。") } else { format!("{working} 個 Bot 正在工作，將會被中斷。") }; - ui.label(if has_snap { format!("從上次儲存的快照還原。這會關閉目前的電腦，並以最新版本啟動一台全新的電腦。{w}") } else { format!("還沒有快照：這會換一台全新的電腦，裡面的檔案和登入資訊都會清除。{w}") }); + ui.label(if has_snap { format!("從上次儲存的快照還原。這會關閉目前的電腦，並以最新版本啟動一台全新的電腦。目前電腦的內容會先存成快照，事後可以找回。{w}") } else { format!("還沒有快照：這會換一台全新的電腦，裡面的檔案和登入資訊都會清除。{w}") }); ui.add_space(8.0); Frame::new().fill(p.danger.gamma_multiply(0.08)).corner_radius(8).inner_margin(Margin::same(10)).show(ui, |ui| { ui.add(egui::Label::new(RichText::new(t("警告：尚未同步到快照的工作（包括最近的檔案）將會遺失。已安裝的應用程式和套件會被移除。")).size(FONT_SM).color(p.danger)).wrap()); diff --git a/src/cloudbox.rs b/src/cloudbox.rs index 0337474..ed919eb 100644 --- a/src/cloudbox.rs +++ b/src/cloudbox.rs @@ -19,6 +19,7 @@ // through `docker exec`. use base64::Engine; +use chrono::Datelike; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::path::PathBuf; @@ -393,12 +394,86 @@ fn maybe_scheduled_update() { } pub fn snapshot_path() -> PathBuf { - data_dir().join("snapshot.tar.gz") + list_snapshots().pop().unwrap_or_else(|| snapshots_dir().join("snapshot.tar.gz")) +} + +fn snapshots_dir() -> PathBuf { + data_dir().join("snapshots") +} + +/// Every saved snapshot, oldest first. Names carry the time (`snapshot-YYYY-MM-DD-HHMMSS.tar.gz`) +/// so they sort by time and a copy keeps its stamp. The single old `snapshot.tar.gz` (from +/// before generations were kept) is moved in, stamped by its file date. +pub fn list_snapshots() -> Vec { + let dir = snapshots_dir(); + let _ = std::fs::create_dir_all(&dir); + let legacy = data_dir().join("snapshot.tar.gz"); + if legacy.exists() { + let name = std::fs::metadata(&legacy) + .and_then(|m| m.modified()) + .ok() + .map(|t| format!("snapshot-{}-legacy.tar.gz", chrono::DateTime::::from(t).format("%Y-%m-%d-%H%M%S"))) + .unwrap_or_else(|| "snapshot-0000-00-00-000000-legacy.tar.gz".into()); + let _ = std::fs::rename(&legacy, dir.join(name)); + } + let mut v: Vec = std::fs::read_dir(&dir) + .map(|rd| { + rd.flatten() + .map(|e| e.path()) + .filter(|p| { + let n = p.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + n.starts_with("snapshot-") && n.ends_with(".tar.gz") + }) + .collect() + }) + .unwrap_or_default(); + v.sort(); + v +} + +/// When a snapshot was saved (from its name; file dates don't survive copies). +fn snapshot_stamp(p: &std::path::Path) -> Option> { + let stem = p.file_name()?.to_string_lossy().strip_suffix(".tar.gz")?.strip_prefix("snapshot-")?.trim_end_matches("-legacy").to_string(); + chrono::NaiveDateTime::parse_from_str(&stem, "%Y-%m-%d-%H%M%S") + .ok() + .and_then(|t| chrono::TimeZone::from_local_datetime(&chrono::Local, &t).single()) + .or_else(|| std::fs::metadata(p).ok().and_then(|m| m.modified().ok()).map(chrono::DateTime::::from)) } /// When the last snapshot was saved. pub fn snapshot_time() -> Option> { - std::fs::metadata(snapshot_path()).ok()?.modified().ok().map(chrono::DateTime::::from) + snapshot_stamp(&snapshot_path()) +} + +/// Snapshot generations: the last 7 days in full, then the newest one of each of the three +/// weeks before. The home folder holds every bot's logins — one bad snapshot overwriting the +/// only copy used to be one command away. +fn prune_snapshots() { + let list = list_snapshots(); + if list.len() <= 7 { + return; + } + let now = chrono::Local::now(); + let mut weeks_kept: Vec<(i32, u32)> = vec![]; + let mut del = vec![]; + for p in list.iter().rev() { + let Some(t) = snapshot_stamp(p) else { continue }; + let age = (now - t).num_days(); + if age <= 7 { + continue; + } + if age <= 28 { + let w = (t.iso_week().year(), t.iso_week().week()); + if !weeks_kept.contains(&w) { + weeks_kept.push(w); + continue; + } + } + del.push(p.clone()); + } + for p in del { + let _ = std::fs::remove_file(&p); + } } fn resume_path() -> PathBuf { @@ -615,9 +690,30 @@ fn create_container() -> Result<(), String> { log("正在開一台新的電腦…"); // host.docker.internal → this Mac (for 網路流量改走你的電腦; Docker Desktop and OrbStack already have it). dk(&["run", "-d", "--name", container(), "--hostname", "bot-computer", "--shm-size", "1g", "--pids-limit", "4096", "--memory", "2g", "--cpus", "2", "--add-host", "host.docker.internal:host-gateway", "-p", &publish, "-v", &vol, "-e", &env, "-e", &vnc, "--label", "anybot.box=1", IMAGE]).map_err(|e| format!("開不了電腦：{}", tail(&e, 400)))?; + ensure_holder(); Ok(()) } +fn holder_name() -> String { + format!("{}-home-holder", container()) +} + +/// Keep the home volume attached to a container. A box removed by an update is a moment with no +/// container using the volume — and `docker volume prune` (a command people run to reclaim +/// disk) deletes volumes that are attached to nothing, logins and all. One sleeping holder +/// keeps it always attached. +fn ensure_holder() { + if dk(&["volume", "inspect", "-f", "{{.Name}}", VOLUME]).is_err() { + return; + } + if dk(&["inspect", "-f", "{{.State.Running}}", &holder_name()]).map(|s| s == "true").unwrap_or(false) { + return; + } + let _ = dk(&["rm", "-f", &holder_name()]); + let vol = format!("{VOLUME}:{HOME}"); + let _ = dk(&["run", "-d", "--name", &holder_name(), "--label", "anybot.box=1", "-v", &vol, IMAGE, "sleep", "infinity"]); +} + fn wait_ready(ms: u64) -> Result<(), String> { let start = Instant::now(); while start.elapsed() < Duration::from_millis(ms) { @@ -712,7 +808,18 @@ pub fn ensure_running() -> Result<(), String> { Run::NoDocker(e) => Err(e), Run::Absent => { set_run(Run::Starting("建立中")); - create_container().and_then(|_| wait_ready(BOOT_WAIT_MS)) + create_container().and_then(|_| { + // A brand-new box on an empty volume (a wiped Docker, a new host): the saved + // home brings the files and every bot's logins back by themselves. + if snapshot_path().exists() && !home_has_files() { + log("新的電腦家目錄是空的：從最近的快照還原檔案和登入…"); + let src = snapshot_path(); + if let Err(e) = copy_in_snapshot(&src) { + log(&format!("自動還原沒有完成：{e}（電腦照常用，檔案以現在為準）。")); + } + } + wait_ready(BOOT_WAIT_MS) + }) } Run::Asleep => { set_run(Run::Starting("喚醒中")); @@ -780,6 +887,10 @@ pub fn start_background() { if quiet && due && run_state().is_running() { let _ = save_snapshot(); } + // (Only when a box exists on this Docker: the holder is its volume's seat-saver.) + if matches!(probe(), Run::Asleep | Run::Running) { + ensure_holder(); + } refresh(); maybe_scheduled_update(); update_when_idle(idle); @@ -831,10 +942,13 @@ fn viewer_open() -> bool { // ---------- snapshots (the durable store) ---------- -/// Save the bot's home folder to this Mac. Returns its size in bytes. +/// Save the bot's home folder to this Mac, as a new generation. Returns its size in bytes. pub fn save_snapshot() -> Result { - let _ = std::fs::create_dir_all(data_dir()); - let tmp = data_dir().join("snapshot.tmp"); + let dir = snapshots_dir(); + let _ = std::fs::create_dir_all(&dir); + let stamp = chrono::Local::now().format("%Y-%m-%d-%H%M%S"); + let tmp = dir.join(format!("snapshot-{stamp}.tmp")); + let final_path = dir.join(format!("snapshot-{stamp}.tar.gz")); let file = std::fs::File::create(&tmp).map_err(|e| e.to_string())?; let mut child = docker() // Caches are left out: the browser's grow to hundreds of MB and come back by themselves. @@ -873,15 +987,17 @@ pub fn save_snapshot() -> Result { let _ = std::fs::remove_file(&tmp); return Err("快照沒有存成功".into()); } - std::fs::rename(&tmp, snapshot_path()).map_err(|e| e.to_string())?; - let n = std::fs::metadata(snapshot_path()).map(|m| m.len()).unwrap_or(0); - log(&format!("已儲存快照（{}）。", human(n))); + std::fs::rename(&tmp, &final_path).map_err(|e| e.to_string())?; + let n = std::fs::metadata(&final_path).map(|m| m.len()).unwrap_or(0); + prune_snapshots(); + log(&format!("已儲存快照（{}，共 {} 代）。", human(n), list_snapshots().len())); Ok(n) } -fn copy_in_snapshot() -> Result<(), String> { +/// Put a saved snapshot's files back into the box's home folder. +fn copy_in_snapshot(src: &std::path::Path) -> Result<(), String> { use std::io::Write; - let data = std::fs::read(snapshot_path()).map_err(|_| "還沒有快照可以還原".to_string())?; + let data = std::fs::read(src).map_err(|_| "還沒有快照可以還原".to_string())?; let mut child = docker() .args(["exec", "-i", container(), "sh", "-c", &format!("tar -C {HOME} -xzf - && chown -R bot:bot {HOME} && echo '[box-copy-in] result outcome=hydrated' >> /tmp/anybot-copy-in.log")]) .stdin(Stdio::piped()) @@ -1030,8 +1146,15 @@ fn run_rebuild(kind: Rebuild, phase: &dyn Fn(&str, &str)) -> Result<(), String> Ok(()) } Rebuild::Reset => { + // The box's home is the only copy of every bot's logins: never wipe it without a + // snapshot of what's in it first — reset becomes undoable. + if matches!(probe(), Run::Running) { + phase("backing-up", "重設前：先存下現在的電腦"); + save_snapshot().map_err(|e| format!("重設取消了：存不下現在的狀態（{e}）。先處理好這個，重設才救得回來。"))?; + } phase("cleaning-up", "拆掉舊的電腦"); let _ = dk(&["rm", "-f", container()]); + let _ = dk(&["rm", "-f", &holder_name()]); phase("wiping", "清除電腦裡的資料"); let _ = dk(&["volume", "rm", "-f", VOLUME]); phase("creating", "開一台全新的電腦"); @@ -1042,8 +1165,16 @@ fn run_rebuild(kind: Rebuild, phase: &dyn Fn(&str, &str)) -> Result<(), String> wait_ready(BOOT_WAIT_MS) } Rebuild::Restore => { + // Say which snapshot to restore before saving the current state (saving adds a + // newer "latest" that restore must not pick up). + let src = snapshot_path(); + if matches!(probe(), Run::Running) { + phase("backing-up", "復原前：先存下現在的電腦"); + save_snapshot().map_err(|e| format!("復原取消了：存不下現在的狀態（{e}）。先處理好這個，最近的變更才不會無聲消失。"))?; + } phase("cleaning-up", "拆掉舊的電腦和它的資料"); let _ = dk(&["rm", "-f", container()]); + let _ = dk(&["rm", "-f", &holder_name()]); let _ = dk(&["volume", "rm", "-f", VOLUME]); phase("creating", "從快照開一台新的電腦"); if !image_exists() { @@ -1052,7 +1183,7 @@ fn run_rebuild(kind: Rebuild, phase: &dyn Fn(&str, &str)) -> Result<(), String> create_container()?; wait_ready(BOOT_WAIT_MS)?; phase("restoring", "把快照裡的檔案放回去"); - copy_in_snapshot() + copy_in_snapshot(&src) } } } @@ -1276,8 +1407,9 @@ fn box_scale() -> Result<(f64, f64), String> { /// A step on the box's screen as a shell script (for `sh -c` inside the box, as the bot user), /// the text it reads on stdin (typing), and what to tell the bot. Coordinates are picture pixels /// times `scale` (desktop pixels per picture pixel; 1 for a person's viewer, which shows the -/// desktop at full size). -pub fn box_script(name: &str, v: &Value, scale: (f64, f64)) -> Result<(String, Option, String), String> { +/// desktop at full size). `inline` puts typed text inside the script quoted (a batch runs its +/// steps in one exec with one stdin, so steps can't ride on it). +pub fn box_script(name: &str, v: &Value, scale: (f64, f64), inline: bool) -> Result<(String, Option, String), String> { // A control's number from the latest picture: its centre on the desktop. let by_number = |p: &Value| -> Option> { let r = match &p["ref"] { @@ -1366,13 +1498,13 @@ pub fn box_script(name: &str, v: &Value, scale: (f64, f64)) -> Result<(String, O None => String::new(), }; if !text.is_empty() { - script.push_str(&format!(" && {}", type_script(&text))); + script.push_str(&format!(" && {}", if inline { type_script_inline(&text) } else { type_script(&text) })); } if submit { script.push_str(" && sleep 0.06 && xdotool key --clearmodifiers Return"); } let script = script.trim_start_matches(" && ").to_string(); - (script, Some(text.clone()).filter(|t| !t.is_empty()), format!("已打字（{} 個字）{field}{}。", text.chars().count(), if submit { "，再按了 return" } else { "" })) + (script, if inline { None } else { Some(text.clone()).filter(|t| !t.is_empty()) }, format!("已打字（{} 個字）{field}{}。", text.chars().count(), if submit { "，再按了 return" } else { "" })) } "computer_use_press_key" => { let keys = v["key"].as_str().unwrap_or(""); @@ -1409,9 +1541,26 @@ fn type_script(text: &str) -> String { .to_string() } +/// The same, with the text quoted into the script (batch steps: one stdin for the whole chain). +fn type_script_inline(text: &str) -> String { + let q = crate::tools::sh_quote(text); + if text.is_ascii() { + return format!("xdotool type --clearmodifiers --delay {TYPE_DELAY_MS} -- {q}"); + } + format!("printf '%s' {q} > /tmp/anybot-paste.txt && for s in clipboard primary; do setsid xclip -selection $s -i /tmp/anybot-paste.txt /dev/null 2>&1 & done; sleep 0.15; \ + case \"$(xprop -id $(xdotool getactivewindow) WM_CLASS 2>/dev/null | tr A-Z a-z)\" in *term*) k=shift+Insert;; *) k=ctrl+v;; esac; xdotool key --clearmodifiers $k") +} + +/// Wait for the screen to stop changing before the picture comes back: two matching probes mean +/// it settled. A probe is two small grayscale captures (top-left, center: ~60 ms on the box), so +/// a settled screen takes ~250 ms where every step used to pay a fixed 0.5 s — and a page that +/// is still loading is waited for (up to ~0.9 s) instead of captured mid-change. Plain sleep +/// when `import` is missing. +const SETTLE_SH: &str = "{ probe() { { import -window root -crop 160x140+80+30 -depth 8 pgm:-; import -window root -crop 160x160+560+320 -depth 8 pgm:-; } 2>/dev/null | md5sum; }; if command -v import >/dev/null 2>&1; then a=$(probe); i=0; while [ $i -lt 5 ]; do sleep 0.1; b=$(probe); [ \"$a\" = \"$b\" ] && break; a=$b; i=$((i+1)); done; else sleep 0.35; fi; }"; + /// Run a step's script in the box; with `picture`, wait for the screen to settle and return it. fn box_exec(script: &str, stdin: Option<&str>, picture: bool) -> Result, String> { - let full = if picture { format!("{{ {script}; }} || exit $?; sleep {:.2}; import -window root png:- 2>/dev/null || true", crate::computer::SETTLE_MS as f64 / 1000.0) } else { script.to_string() }; + let full = if picture { format!("{{ {script}; }} || exit $?; {SETTLE_SH}; import -window root -quality 90 jpeg:- 2>/dev/null || true") } else { script.to_string() }; // Always an stdin (closed at once when there's no text), as `docker exec -i` expects. let out = match dk_with_input(&["exec", "-i", "-u", "bot", "-e", "DISPLAY=:1", container(), "sh", "-c", &full], Some(stdin.unwrap_or("").as_bytes()), 30_000) { Ok(o) => o, @@ -1498,10 +1647,14 @@ pub fn elements_of(v: &Value) -> Vec { /// controls numbered and the list, in one `docker exec`. After a step it also says when nothing /// changed, so the bot doesn't click the same empty place again and again. fn box_look(script: Option<(&str, Option<&str>)>, after: Option) -> Result<(String, Option), String> { - let look = "anybot-a11y --budget 3 > /tmp/anybot-look.json 2>/dev/null || echo '{\"elements\":[]}' > /tmp/anybot-look.json; \ - wc -c < /tmp/anybot-look.json; cat /tmp/anybot-look.json; import -window root png:- 2>/dev/null"; + let t_all = Instant::now(); + // The picture is captured by a background `import` while the controls are read (the a11y + // read is the slow part on web pages); a JPEG costs less to make and to ship than a PNG. + let look = "( import -window root -quality 90 jpeg:- 2>/dev/null > /tmp/anybot-look.jpg & ip=$!; \ + anybot-a11y --budget 3 > /tmp/anybot-look.json 2>/dev/null || echo '{\"elements\":[]}' > /tmp/anybot-look.json; \ + wc -c < /tmp/anybot-look.json; cat /tmp/anybot-look.json; wait $ip; cat /tmp/anybot-look.jpg )"; let full = match script { - Some((s, _)) => format!("{{ {s}; }} || exit $?; sleep {:.2}; {look}", crate::computer::SETTLE_MS as f64 / 1000.0), + Some((s, _)) => format!("{{ {s}; }} || exit $?; {SETTLE_SH}; {look}"), None => look.to_string(), }; let input = script.and_then(|(_, i)| i).unwrap_or(""); @@ -1511,7 +1664,8 @@ fn box_look(script: Option<(&str, Option<&str>)>, after: Option) -> Resu Err(e) if e.is_empty() => return Err("這一步沒有成功".into()), Err(e) => return Err(format!("xdotool：{e}")), }; - // "\n" + let exec_ms = t_all.elapsed().as_millis() as u64; + // "\n" let nl = out.iter().position(|b| *b == b'\n').ok_or("這台電腦沒有回傳畫面")?; let n: usize = String::from_utf8_lossy(&out[..nl]).trim().parse().map_err(|_| "這台電腦回傳的畫面讀不懂")?; let json_end = (nl + 1 + n).min(out.len()); @@ -1543,9 +1697,32 @@ fn box_look(script: Option<(&str, Option<&str>)>, after: Option) -> Resu } else { text.push_str(&format!("\n可以操作的東西（圖上同樣的編號，用 ref 點）：\n{}", lines.join("\n"))); } + // Step timing (for the 電腦 card): how long the box's side took vs. the whole step. + { + let total_ms = t_all.elapsed().as_millis() as u64; + let mut s = STEP_TIMES.lock().unwrap_or_else(|e| e.into_inner()); + s.push((exec_ms, total_ms)); + let n = s.len(); + if n > 40 { + s.drain(..n - 40); + } + } Ok((text, Some(base64::engine::general_purpose::STANDARD.encode(jpeg)))) } +/// The box's latest steps, timed: (count, average docker-exec ms, average whole-step ms). +/// Whole step = exec (xdotool + settle + 控件 + 截圖) + 這裡的編號繪製和壓縮. +pub fn step_stats() -> (usize, u64, u64) { + let s = STEP_TIMES.lock().unwrap_or_else(|e| e.into_inner()); + if s.is_empty() { + return (0, 0, 0); + } + let n = s.len() as u64; + (s.len(), s.iter().map(|(a, _)| *a).sum::() / n, s.iter().map(|(_, b)| *b).sum::() / n) +} + +static STEP_TIMES: Mutex> = Mutex::new(Vec::new()); + /// Programs people name, as the box knows them. fn box_program(app: &str) -> Option<&'static str> { let a = app.trim().to_lowercase(); @@ -1601,10 +1778,51 @@ pub fn computer_run(name: &str, v: &Value) -> Result<(String, Option), S let note = open_on_box(v["app"].as_str().unwrap_or(""))?; return box_look(Some(("true", None)), Some(note)); } - let (script, stdin, note) = box_script(name, v, box_scale()?)?; + if name == "computer_use_batch" { + return box_batch(v, box_scale()?); + } + let (script, stdin, note) = box_script(name, v, box_scale()?, false)?; box_look(Some((&script, stdin.as_deref())), Some(note)) } +const BATCH_MARK: &str = "/tmp/anybot-batch.done"; + +/// Do a planned sequence of steps on the box in one round trip: the steps' scripts chained with +/// `&&` (so the chain stops at the first failure), one marker line per step that ran, and one +/// picture at the end — like computer_batch everywhere else, and one `docker exec` for all of +/// them instead of one per step. +fn box_batch(v: &Value, scale: (f64, f64)) -> Result<(String, Option), String> { + let acts = v["actions"].as_array().ok_or("computer_use_batch 要給 actions：動作陣列，每個有 action（click、typing、press_key、scroll、mouse_move、drag）和它的參數")?; + if acts.is_empty() { + return Err("actions 是空的".into()); + } + if acts.len() > crate::computer::MAX_BATCH { + return Err(format!("一次最多 {} 個動作（{} 個太多了）", crate::computer::MAX_BATCH, acts.len())); + } + let mut parts = Vec::new(); + let mut notes = Vec::new(); + for (i, a) in acts.iter().enumerate() { + let name = crate::computer::action_tool(a["action"].as_str().unwrap_or(""))?; + let (script, _, note) = box_script(name, a, scale, true)?; + parts.push(format!("{{ {script} && echo x >> {BATCH_MARK}; }}")); + notes.push(format!("{}. {note}", i + 1)); + } + box_exec(&format!("rm -f {BATCH_MARK}; {} ; true", parts.join(" && ")), None, false)?; + let done = dk(&["exec", container(), "sh", "-c", &format!("wc -l < {BATCH_MARK} 2>/dev/null || echo 0")]) + .ok() + .and_then(|s| s.trim().parse::().ok()) + .unwrap_or(0) + .min(acts.len()); + let _ = dk(&["exec", container(), "sh", "-c", &format!("rm -f {BATCH_MARK}")]); + let note = if done == acts.len() { + format!("已依序做完 {done} 步：\n{}", notes.join("\n")) + } else { + let ran: Vec<&str> = notes.iter().take(done).map(String::as_str).collect(); + format!("做完 {done}/{} 步，第 {} 步沒有成功（後面沒有做）：\n{}", acts.len(), done + 1, ran.join("\n")) + }; + box_look(Some(("true", None)), Some(note)) +} + // ---------- text a person types in a viewer ---------- /// Text from a person's viewer (any language): typed or pasted into the window in front, like @@ -1762,28 +1980,57 @@ mod tests { let s = (1.25, 1.25); // Picture pixels → desktop pixels; never `mousemove --sync` (it hangs when the pointer is // already there). - let (sc, input, note) = box_script("computer_use_click", &serde_json::json!({"x": 512, "y": 320}), s).unwrap(); + let (sc, input, note) = box_script("computer_use_click", &serde_json::json!({"x": 512, "y": 320}), s, false).unwrap(); assert_eq!(sc, "xdotool mousemove 640 400 sleep 0.03 click --repeat 1 --delay 80 1"); assert!(input.is_none() && note.contains("(512, 320)")); // Keys held for a step are let go even when it fails. - let (sc, _, _) = box_script("computer_use_click", &serde_json::json!({"x": 8, "y": 8, "modifier_keys": ["cmd", "shift"], "count": 2}), s).unwrap(); + let (sc, _, _) = box_script("computer_use_click", &serde_json::json!({"x": 8, "y": 8, "modifier_keys": ["cmd", "shift"], "count": 2}), s, false).unwrap(); assert!(sc.starts_with("xdotool keydown ctrl keydown shift mousemove 10 10") && sc.ends_with("rc=$?; xdotool keyup shift keyup ctrl; [ $rc = 0 ]"), "{sc}"); assert!(!sc.contains("--sync")); // Text goes in on stdin (no quoting to get wrong); return after it when asked. - let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "echo 'hi' && ls", "submit": true}), s).unwrap(); + let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "echo 'hi' && ls", "submit": true}), s, false).unwrap(); assert_eq!(input.as_deref(), Some("echo 'hi' && ls")); assert!(sc.starts_with("xdotool type --clearmodifiers --delay 12 --file -") && sc.ends_with("xdotool key --clearmodifiers Return"), "{sc}"); - let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "中文"}), s).unwrap(); + let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "中文"}), s, false).unwrap(); assert!(sc.contains("xclip") && input.as_deref() == Some("中文")); // Keys by name, a character typed, several in order. - let (sc, _, _) = box_script("computer_use_press_key", &serde_json::json!({"key": "slash @ cmd+a"}), s).unwrap(); + let (sc, _, _) = box_script("computer_use_press_key", &serde_json::json!({"key": "slash @ cmd+a"}), s, false).unwrap(); assert_eq!(sc, "xdotool key --clearmodifiers slash sleep 0.02 type --clearmodifiers -- '@' sleep 0.02 key --clearmodifiers ctrl+a"); // A drag walks its path in small steps and lets go at the end. - let (sc, _, _) = box_script("computer_use_drag", &serde_json::json!({"path": [{"x": 0, "y": 0}, {"x": 48, "y": 0}]}), s).unwrap(); + let (sc, _, _) = box_script("computer_use_drag", &serde_json::json!({"path": [{"x": 0, "y": 0}, {"x": 48, "y": 0}]}), s, false).unwrap(); assert!(sc.starts_with("xdotool mousemove 0 0 sleep 0.03 mousedown 1") && sc.contains("mousemove 60 0") && sc.ends_with("mouseup 1"), "{sc}"); // No app numbers on this desktop: said, not guessed. - assert!(box_script("computer_use_click", &serde_json::json!({"ref": "e3"}), s).is_err()); - assert!(box_script("computer_use_scroll", &serde_json::json!({"direction": "sideways"}), s).is_err()); + assert!(box_script("computer_use_click", &serde_json::json!({"ref": "e3"}), s, false).is_err()); + assert!(box_script("computer_use_scroll", &serde_json::json!({"direction": "sideways"}), s, false).is_err()); + } + + #[test] + fn batch_steps_carry_their_text_inline() { + let s = (1.0, 1.0); + // A batch runs its steps in one exec with one stdin: the text rides in the script, quoted. + let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "it's a 'test'"}), s, true).unwrap(); + assert!(input.is_none(), "no stdin for a batch step"); + assert!(sc.starts_with("xdotool type --clearmodifiers --delay 12 -- 'it'\\''s a '\\''test'\\'''"), "{sc}"); + // Chinese in a batch: the paste path, text written inline. + let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "中文", "submit": true}), s, true).unwrap(); + assert!(input.is_none() && sc.contains("printf '%s' '中文' > /tmp/anybot-paste.txt") && sc.ends_with("xdotool key --clearmodifiers Return"), "{sc}"); + // Batch action names, and the ones a batch must not carry. + assert_eq!(crate::computer::action_tool("Click").unwrap(), "computer_use_click"); + assert_eq!(crate::computer::action_tool("typing").unwrap(), "computer_use_typing"); + assert!(crate::computer::action_tool("screenshot").is_err()); + assert!(crate::computer::action_tool("open_app").is_err()); + assert!(crate::computer::action_tool("").is_err()); + } + + #[test] + fn snapshot_names_carry_their_time() { + // Names sort by time, and the stamp is readable from the name (copies lose file dates). + let p = std::path::Path::new("/x/snapshot-2026-11-05-123456.tar.gz"); + let t = snapshot_stamp(p).expect("stamp parses"); + assert_eq!(t.format("%Y-%m-%d %H:%M:%S").to_string(), "2026-11-05 12:34:56"); + let legacy = std::path::Path::new("/x/snapshot-2026-10-01-000000-legacy.tar.gz"); + assert_eq!(snapshot_stamp(legacy).expect("legacy parses").format("%Y-%m-%d").to_string(), "2026-10-01"); + assert!(snapshot_stamp(std::path::Path::new("/x/notes.txt")).is_none()); } /// A box made from this image (`ANYBOT_BOX_NAME= cargo test --release -- --ignored @@ -1876,6 +2123,39 @@ mod tests { let _ = box_exec("xdotool key --clearmodifiers ctrl+d", None, false); } + /// Uses the real Docker: `cargo test --release -- --ignored box_batch_in_one_round_trip --nocapture`. + /// A planned sequence runs in one round trip: ASCII and Chinese typing and a submit land in the + /// front app, the note lists every step, and one picture comes back. + #[test] + #[ignore] + fn box_batch_in_one_round_trip() { + ensure_running().expect("box starts"); + let _ = dk(&["exec", "-u", "bot", container(), "rm", "-f", "/home/bot/batch.txt"]); + let (txt, img) = computer_run("computer_use_open_app", &serde_json::json!({"app": "終端機"})).unwrap(); + assert!(img.is_some(), "{txt}"); + let t = Instant::now(); + let (txt, img) = computer_run( + "computer_use_batch", + &serde_json::json!({"actions": [ + {"action": "typing", "value": "echo batch-"}, + {"action": "typing", "value": "中文"}, + {"action": "typing", "value": "ok > /home/bot/batch.txt", "submit": true} + ]}), + ) + .unwrap(); + println!("batch of 3: {} ms\n{txt}", t.elapsed().as_millis()); + assert!(img.is_some() && txt.contains("已依序做完 3 步"), "{txt}"); + std::thread::sleep(Duration::from_millis(600)); + let out = dk(&["exec", "-u", "bot", container(), "cat", "/home/bot/batch.txt"]).unwrap_or_default(); + println!("typed: {out}"); + assert_eq!(out.trim(), "batch-中文ok"); + // Too many, empty: said before anything runs. + assert!(computer_run("computer_use_batch", &serde_json::json!({"actions": []})).is_err()); + let many: Vec<_> = (0..9).map(|_| serde_json::json!({"action": "press_key", "key": "f15"})).collect(); + assert!(computer_run("computer_use_batch", &serde_json::json!({"actions": many})).is_err()); + let _ = box_exec("xdotool key --clearmodifiers ctrl+d", None, false); + } + /// Uses the real Docker: `cargo test --release -- --ignored box_end_to_end --nocapture`. #[test] #[ignore] diff --git a/src/computer.rs b/src/computer.rs index 493b012..462d5f0 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -12,10 +12,29 @@ use base64::Engine; use serde_json::{json, Value}; use std::sync::Mutex; -pub const NAMES: [&str; 9] = ["computer_use_screenshot", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app"]; +pub const NAMES: [&str; 10] = ["computer_use_screenshot", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app", "computer_use_batch"]; /// Wait after an action before taking the picture that comes back with it (menus open, pages -/// start to change; estimated). +/// start to change; estimated). The box waits for the screen to stop changing instead, up to +/// about this long. pub const SETTLE_MS: u64 = 500; +/// Most actions one `computer_use_batch` call may carry (Anthropic's computer_batch guidance: +/// short self-contained sequences, one picture at the end). +pub const MAX_BATCH: usize = 8; + +/// A batch step's short action name → the tool it runs. Screen reads, apps and permissions stay +/// single steps on purpose: a batch is for input you already planned from the current picture. +pub fn action_tool(a: &str) -> Result<&'static str, String> { + match a.trim().to_lowercase().as_str() { + "click" => Ok("computer_use_click"), + "typing" | "type" => Ok("computer_use_typing"), + "press_key" | "key" => Ok("computer_use_press_key"), + "scroll" => Ok("computer_use_scroll"), + "mouse_move" | "move" => Ok("computer_use_mouse_move"), + "drag" => Ok("computer_use_drag"), + "" => Err("batch 的每個動作都要有 action（click、typing、press_key、scroll、mouse_move、drag）".into()), + other => Err(format!("batch 裡沒有「{other}」這個動作（可用：click、typing、press_key、scroll、mouse_move、drag；要打開 App 請單獨用 computer_use_open_app）")), + } +} /// The person took control back from the bot (the floating bar): screen actions wait. pub static HELD_BY_PERSON: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); @@ -209,6 +228,7 @@ pub fn label(name: &str) -> &'static str { "computer_use_drag" => "拖曳", "computer_use_check_permissions" => "檢查螢幕權限", "computer_use_open_app" => "打開 App", + "computer_use_batch" => "依序操作", _ => "操作畫面", } } @@ -241,6 +261,7 @@ pub fn describe(name: &str, args: &str) -> String { "computer_use_drag" => format!("{} 個點", v["path"].as_array().map(|a| a.len()).unwrap_or(0)), "computer_use_check_permissions" => "螢幕錄製、輔助使用".into(), "computer_use_open_app" => v["app"].as_str().filter(|a| !a.is_empty()).unwrap_or("").to_string(), + "computer_use_batch" => format!("依序 {} 個動作", v["actions"].as_array().map(|a| a.len()).unwrap_or(0)), _ => String::new(), } } @@ -269,6 +290,8 @@ pub fn specs_for(computers: &[String], default: &str) -> Vec { "parameters": {"type": "object", "properties": {"path": {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "button": {"type": "string", "enum": ["left", "right"]}, "modifier_keys": mods}, "required": ["path"]}}), json!({"name": "computer_use_check_permissions", "description": "Check whether the screen can be seen and the mouse and keyboard used on that computer.", "parameters": {"type": "object", "properties": {}}}), + json!({"name": "computer_use_batch", "description": format!("Do a short planned sequence of steps in one call (max {MAX_BATCH}): each item is one action with its own arguments, e.g. {{\"action\":\"click\",\"ref\":\"12\"}}, {{\"action\":\"typing\",\"ref\":\"15\",\"value\":\"cat\"}}, {{\"action\":\"press_key\",\"key\":\"tab\"}}. Use it when the current picture already tells you every step (filling a form, a known shortcut chain): they run in order, the first step that fails stops the rest, and you get one picture at the end. Do not use it to explore, or where step 2 depends on what step 1's screen shows — act one step at a time there."), + "parameters": {"type": "object", "properties": {"actions": {"type": "array", "maxItems": MAX_BATCH, "items": {"type": "object", "properties": {"action": {"type": "string", "enum": ["click", "typing", "press_key", "scroll", "mouse_move", "drag"]}, "ref": rf.clone(), "x": {"type": "number"}, "y": {"type": "number"}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "count": {"type": "integer"}, "value": {"type": "string"}, "submit": {"type": "boolean"}, "key": {"type": "string"}, "hold_duration_ms": {"type": "integer"}, "direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "amount": {"type": "integer"}, "path": {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "modifier_keys": mods.clone()}, "required": ["action"]}}}, "required": ["actions"]}}), ]; // Always named: each computer goes by its own name (the instructions say which is which), // so the bot says whose screen it means and the person can tell where it worked. @@ -447,7 +470,7 @@ pub fn run_by(name: &str, args: &str, who: &str) -> (String, bool, Option bool { - matches!(name, "computer_use_click" | "computer_use_typing" | "computer_use_press_key" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_drag" | "computer_use_open_app") + matches!(name, "computer_use_click" | "computer_use_typing" | "computer_use_press_key" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_drag" | "computer_use_open_app" | "computer_use_batch") } /// Ask macOS for a permission (its own prompt, then 系統設定 at that pane) at most once every ten @@ -608,6 +631,35 @@ fn press_keys(keys: &str, hold: f64) -> Result<(), String> { Ok(()) } +/// Run a planned sequence of actions, then one picture (in the box the box's own batch runs; +/// here, the local screen). The first action that fails stops the rest — acting on a screen +/// that isn't where you think it is makes it worse (Anthropic's computer_batch rule). +pub fn batch(v: &Value, scale: (f64, f64)) -> Result { + let acts = v["actions"].as_array().ok_or("computer_use_batch 要給 actions：動作陣列，每個有 action（click、typing、press_key、scroll、mouse_move、drag）和它的參數")?; + if acts.is_empty() { + return Err("actions 是空的".into()); + } + if acts.len() > MAX_BATCH { + return Err(format!("一次最多 {MAX_BATCH} 個動作（{} 個太多了）", acts.len())); + } + let mut notes = Vec::new(); + for (i, a) in acts.iter().enumerate() { + let name = action_tool(a["action"].as_str().unwrap_or(""))?; + match act(name, a, scale) { + Ok(n) => notes.push(format!("{}. {n}", i + 1)), + Err(e) => { + let e = plain_words(e); + if notes.is_empty() { + return Err(format!("第 {} 步就失敗：{e}", i + 1)); + } + notes.push(format!("{}. 失敗：{e}（後面 {} 個沒有做）", i + 1, acts.len() - i - 1)); + break; + } + } + } + Ok(format!("已依序做完 {} 步：\n{}", notes.len(), notes.join("\n"))) +} + /// Do one mouse or keyboard step at the screen's own coordinates (`scale`: screen points per /// picture pixel). The bot's tools and a person tapping on the phone's picture both come here: /// one way of acting on a screen. @@ -805,7 +857,10 @@ fn run_inner(name: &str, v: &Value) -> Result<(String, Option), String> if name != "computer_use_open_app" { need_controls()?; } - let note = act(name, v, scale_here()).map_err(plain_words)?; + let note = match name { + "computer_use_batch" => batch(v, scale_here())?, + _ => act(name, v, scale_here()).map_err(plain_words)?, + }; Ok(observe(note)) }