Merge branch 'box-smooth-logins': bots do a planned sequence in one step (computer_use_batch); the box settles by its screen, not the clock; every bot's login outlives resets, prunes and rebuilds (snapshot generations, save-before-wipe, volume holder)
This commit is contained in:
王性驊
commit
deec774a56
2
Makefile
2
Makefile
|
|
@ -88,6 +88,8 @@ ifeq ($(UNAME),Darwin)
|
|||
@xcode-select -p >/dev/null 2>&1 && echo " ✓ Xcode Command Line Tools" || echo " ✗ Xcode Command Line Tools：make deps"
|
||||
endif
|
||||
@test -x $(BIN) && echo " ✓ 已建置 $(BIN)" || echo " · 還沒建置：make"
|
||||
@command -v docker >/dev/null 2>&1 && { docker volume inspect anybot-box-home >/dev/null 2>&1 && echo " ✓ Bot 的電腦的資料碟 anybot-box-home（檔案和登入都在裡面）" || echo " · 沒有 anybot-box-home 資料碟（Bot 的電腦還沒建立過，或被清除過——有快照會在下次建立時自動還原）"; } || true
|
||||
@d="$${HOME}/Library/Application Support/AnyBot/box/snapshots"; [ -d "$$d" ] || d="$${HOME}/.local/share/AnyBot/box/snapshots"; [ -d "$$d" ] && echo " ✓ 快照 $$(ls "$$d" 2>/dev/null | grep -c '^snapshot-') 份（留最近 7 天＋之前每週一份；重設前也會先存）" || echo " · 還沒有快照（Bot 的電腦每天自動存，重設前也會先存）"
|
||||
|
||||
install: app
|
||||
ifeq ($(UNAME),Darwin)
|
||||
|
|
|
|||
|
|
@ -4032,6 +4032,7 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu
|
|||
- 如果讀到的是登入頁：把登入頁開在那台電腦上，用 hand_over_computer 請使用者登入一次（之後會記住），他按「我完成了」再讀。絕對不要叫使用者去改瀏覽器的設定或選單（例如「允許來自 Apple 事件的 JavaScript」），那不需要。也不要說要從 Chrome 複製登入或 Cookie，沒有這個功能。\n\
|
||||
- AnyBot 瀏覽器有設定檔（profile），每個有自己的登入。使用者說要換 profile／設定檔／另一個帳號時，用 browser_profiles：list 看有哪些，use 切換，add 新增（新的要登入一次）。也可以在 browser_open / browser_read 帶 profile 參數。使用者想用自己平常的 Chrome（自己的 Chrome 設定檔、已經登入的帳號）時，用 profile「你的瀏覽器」：請使用者在 Chrome 切到想要的設定檔，再 browser_read。\n\
|
||||
- 你也能像人一樣操作螢幕（{screens}），用在檔案、指令和網頁工具做不到的事（例如操作其他 App）。先 computer_use_screenshot：你會拿到一張每個按鈕、連結、欄位、選單項目都框起來、標了編號的截圖，和同樣編號的清單。一律照編號操作（click／typing／scroll 的 ref，例如 ref 填 12），點下去就是那個東西的正中央；只有圖上沒有編號的東西（圖片裡的一個點、遊戲）才用 x、y。每個動作做完都會附上新的編號截圖，不要另外截圖。結果說「畫面沒有變化」就是沒點到：換一個編號或換方法，絕對不要重複同一步。要在某個 App 裡打字：先 computer_use_open_app，再 computer_use_typing（ref 是輸入框的編號；submit=true 順便按 return）\n\
|
||||
- 已經從目前畫面知道接下來的幾個動作時（填一張表：點欄位、打字、下一欄、打字、按 return；一組捷徑），用 computer_use_batch 一次做完：actions 是動作陣列，每個 {{\"action\":\"click\",\"ref\":\"12\"}} 這樣（最多 8 個）。會依序執行，最後只附一張截圖；第 2 步要看第 1 步的結果才知道怎麼做的，就拆開用單個動作。每次看截圖用一句話說畫面顯示什麼、這步成功與否。\n\
|
||||
{}\
|
||||
- 不要說「我沒有瀏覽器或工具」：先試這些工具。需要權限時 App 會自己跳出視窗問使用者；工具回報要開什麼設定時，照實轉告。",
|
||||
now.format("%Y-%m-%d %H:%M"),
|
||||
|
|
|
|||
|
|
@ -425,6 +425,11 @@ impl App {
|
|||
act = Act::Browser;
|
||||
}
|
||||
});
|
||||
// What a step on this box really costs (from the app's own timings).
|
||||
let (n, exec_ms, total_ms) = cloudbox::step_stats();
|
||||
if n > 0 {
|
||||
hint(ui, format!("最近 {n} 步畫面操作：在電腦上做平均 {exec_ms} ms，整步（含編號畫面）平均 {total_ms} ms"), p);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 更新 Bot 的電腦: two clicks (the first arms it for 3 s).
|
||||
|
|
@ -467,7 +472,7 @@ impl App {
|
|||
// 重設 Bot 的電腦: from the last snapshot (red, asks first).
|
||||
let snap = cloudbox::snapshot_time().map(|t| t.format("%m/%d %H:%M").to_string());
|
||||
let desc = match &snap { Some(t) => format!("上次快照：{t}"), None => t("還沒有快照") };
|
||||
sec_row_s(ui, p, "重設 Bot 的電腦", &desc, "電腦卡住時可重新開始。它會從你上次儲存的快照重建，最近的變更可能會遺失；沒有快照時會換一台全新的電腦。「儲存快照」把 Bot 的檔案備份到這台電腦，每天也會自動存一份。", |ui| {
|
||||
sec_row_s(ui, p, "重設 Bot 的電腦", &desc, "電腦卡住時可重新開始。它會從你上次儲存的快照重建，最近的變更可能會遺失；沒有快照時會換一台全新的電腦。「儲存快照」把 Bot 的檔案備份到這台電腦，每天也會自動存一份。重設或還原之前，會先把現在的電腦存成一份快照（存不下就不會動手），之後還能從它找回；快照保留最近 7 天，之前每週各留一份。", |ui| {
|
||||
if ui.add_enabled(!busy_rebuild, danger_button("重設", p)).clicked() {
|
||||
act = Act::Reset;
|
||||
}
|
||||
|
|
@ -682,7 +687,7 @@ impl App {
|
|||
let close = dialog(ctx, "reset-computer", "要重設 AnyBot 的電腦嗎？", 460.0, true, p, |ui| {
|
||||
dialog_body(ui, "reset-computer-body", 320.0, |ui| {
|
||||
let w = if working == 0 { t("所有執行中的工作都將被中斷。") } else { format!("{working} 個 Bot 正在工作，將會被中斷。") };
|
||||
ui.label(if has_snap { format!("從上次儲存的快照還原。這會關閉目前的電腦，並以最新版本啟動一台全新的電腦。{w}") } else { format!("還沒有快照：這會換一台全新的電腦，裡面的檔案和登入資訊都會清除。{w}") });
|
||||
ui.label(if has_snap { format!("從上次儲存的快照還原。這會關閉目前的電腦，並以最新版本啟動一台全新的電腦。目前電腦的內容會先存成快照，事後可以找回。{w}") } else { format!("還沒有快照：這會換一台全新的電腦，裡面的檔案和登入資訊都會清除。{w}") });
|
||||
ui.add_space(8.0);
|
||||
Frame::new().fill(p.danger.gamma_multiply(0.08)).corner_radius(8).inner_margin(Margin::same(10)).show(ui, |ui| {
|
||||
ui.add(egui::Label::new(RichText::new(t("警告：尚未同步到快照的工作（包括最近的檔案）將會遺失。已安裝的應用程式和套件會被移除。")).size(FONT_SM).color(p.danger)).wrap());
|
||||
|
|
|
|||
437
src/cloudbox.rs
437
src/cloudbox.rs
|
|
@ -19,6 +19,7 @@
|
|||
// through `docker exec`.
|
||||
|
||||
use base64::Engine;
|
||||
use chrono::Datelike;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::path::PathBuf;
|
||||
|
|
@ -393,12 +394,86 @@ fn maybe_scheduled_update() {
|
|||
}
|
||||
|
||||
pub fn snapshot_path() -> PathBuf {
|
||||
data_dir().join("snapshot.tar.gz")
|
||||
list_snapshots().pop().unwrap_or_else(|| snapshots_dir().join("snapshot.tar.gz"))
|
||||
}
|
||||
|
||||
fn snapshots_dir() -> PathBuf {
|
||||
data_dir().join("snapshots")
|
||||
}
|
||||
|
||||
/// Every saved snapshot, oldest first. Names carry the time (`snapshot-YYYY-MM-DD-HHMMSS.tar.gz`)
|
||||
/// so they sort by time and a copy keeps its stamp. The single old `snapshot.tar.gz` (from
|
||||
/// before generations were kept) is moved in, stamped by its file date.
|
||||
pub fn list_snapshots() -> Vec<PathBuf> {
|
||||
let dir = snapshots_dir();
|
||||
let _ = std::fs::create_dir_all(&dir);
|
||||
let legacy = data_dir().join("snapshot.tar.gz");
|
||||
if legacy.exists() {
|
||||
let name = std::fs::metadata(&legacy)
|
||||
.and_then(|m| m.modified())
|
||||
.ok()
|
||||
.map(|t| format!("snapshot-{}-legacy.tar.gz", chrono::DateTime::<chrono::Local>::from(t).format("%Y-%m-%d-%H%M%S")))
|
||||
.unwrap_or_else(|| "snapshot-0000-00-00-000000-legacy.tar.gz".into());
|
||||
let _ = std::fs::rename(&legacy, dir.join(name));
|
||||
}
|
||||
let mut v: Vec<PathBuf> = std::fs::read_dir(&dir)
|
||||
.map(|rd| {
|
||||
rd.flatten()
|
||||
.map(|e| e.path())
|
||||
.filter(|p| {
|
||||
let n = p.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
||||
n.starts_with("snapshot-") && n.ends_with(".tar.gz")
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
v.sort();
|
||||
v
|
||||
}
|
||||
|
||||
/// When a snapshot was saved (from its name; file dates don't survive copies).
|
||||
fn snapshot_stamp(p: &std::path::Path) -> Option<chrono::DateTime<chrono::Local>> {
|
||||
let stem = p.file_name()?.to_string_lossy().strip_suffix(".tar.gz")?.strip_prefix("snapshot-")?.trim_end_matches("-legacy").to_string();
|
||||
chrono::NaiveDateTime::parse_from_str(&stem, "%Y-%m-%d-%H%M%S")
|
||||
.ok()
|
||||
.and_then(|t| chrono::TimeZone::from_local_datetime(&chrono::Local, &t).single())
|
||||
.or_else(|| std::fs::metadata(p).ok().and_then(|m| m.modified().ok()).map(chrono::DateTime::<chrono::Local>::from))
|
||||
}
|
||||
|
||||
/// When the last snapshot was saved.
|
||||
pub fn snapshot_time() -> Option<chrono::DateTime<chrono::Local>> {
|
||||
std::fs::metadata(snapshot_path()).ok()?.modified().ok().map(chrono::DateTime::<chrono::Local>::from)
|
||||
snapshot_stamp(&snapshot_path())
|
||||
}
|
||||
|
||||
/// Snapshot generations: the last 7 days in full, then the newest one of each of the three
|
||||
/// weeks before. The home folder holds every bot's logins — one bad snapshot overwriting the
|
||||
/// only copy used to be one command away.
|
||||
fn prune_snapshots() {
|
||||
let list = list_snapshots();
|
||||
if list.len() <= 7 {
|
||||
return;
|
||||
}
|
||||
let now = chrono::Local::now();
|
||||
let mut weeks_kept: Vec<(i32, u32)> = vec![];
|
||||
let mut del = vec![];
|
||||
for p in list.iter().rev() {
|
||||
let Some(t) = snapshot_stamp(p) else { continue };
|
||||
let age = (now - t).num_days();
|
||||
if age <= 7 {
|
||||
continue;
|
||||
}
|
||||
if age <= 28 {
|
||||
let w = (t.iso_week().year(), t.iso_week().week());
|
||||
if !weeks_kept.contains(&w) {
|
||||
weeks_kept.push(w);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
del.push(p.clone());
|
||||
}
|
||||
for p in del {
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
}
|
||||
|
||||
fn resume_path() -> PathBuf {
|
||||
|
|
@ -615,9 +690,30 @@ fn create_container() -> Result<(), String> {
|
|||
log("正在開一台新的電腦…");
|
||||
// host.docker.internal → this Mac (for 網路流量改走你的電腦; Docker Desktop and OrbStack already have it).
|
||||
dk(&["run", "-d", "--name", container(), "--hostname", "bot-computer", "--shm-size", "1g", "--pids-limit", "4096", "--memory", "2g", "--cpus", "2", "--add-host", "host.docker.internal:host-gateway", "-p", &publish, "-v", &vol, "-e", &env, "-e", &vnc, "--label", "anybot.box=1", IMAGE]).map_err(|e| format!("開不了電腦：{}", tail(&e, 400)))?;
|
||||
ensure_holder();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn holder_name() -> String {
|
||||
format!("{}-home-holder", container())
|
||||
}
|
||||
|
||||
/// Keep the home volume attached to a container. A box removed by an update is a moment with no
|
||||
/// container using the volume — and `docker volume prune` (a command people run to reclaim
|
||||
/// disk) deletes volumes that are attached to nothing, logins and all. One sleeping holder
|
||||
/// keeps it always attached.
|
||||
fn ensure_holder() {
|
||||
if dk(&["volume", "inspect", "-f", "{{.Name}}", VOLUME]).is_err() {
|
||||
return;
|
||||
}
|
||||
if dk(&["inspect", "-f", "{{.State.Running}}", &holder_name()]).map(|s| s == "true").unwrap_or(false) {
|
||||
return;
|
||||
}
|
||||
let _ = dk(&["rm", "-f", &holder_name()]);
|
||||
let vol = format!("{VOLUME}:{HOME}");
|
||||
let _ = dk(&["run", "-d", "--name", &holder_name(), "--label", "anybot.box=1", "-v", &vol, IMAGE, "sleep", "infinity"]);
|
||||
}
|
||||
|
||||
fn wait_ready(ms: u64) -> Result<(), String> {
|
||||
let start = Instant::now();
|
||||
while start.elapsed() < Duration::from_millis(ms) {
|
||||
|
|
@ -712,7 +808,18 @@ pub fn ensure_running() -> Result<(), String> {
|
|||
Run::NoDocker(e) => Err(e),
|
||||
Run::Absent => {
|
||||
set_run(Run::Starting("建立中"));
|
||||
create_container().and_then(|_| wait_ready(BOOT_WAIT_MS))
|
||||
create_container().and_then(|_| {
|
||||
// A brand-new box on an empty volume (a wiped Docker, a new host): the saved
|
||||
// home brings the files and every bot's logins back by themselves.
|
||||
if snapshot_path().exists() && !home_has_files() {
|
||||
log("新的電腦家目錄是空的：從最近的快照還原檔案和登入…");
|
||||
let src = snapshot_path();
|
||||
if let Err(e) = copy_in_snapshot(&src) {
|
||||
log(&format!("自動還原沒有完成：{e}（電腦照常用，檔案以現在為準）。"));
|
||||
}
|
||||
}
|
||||
wait_ready(BOOT_WAIT_MS)
|
||||
})
|
||||
}
|
||||
Run::Asleep => {
|
||||
set_run(Run::Starting("喚醒中"));
|
||||
|
|
@ -780,6 +887,10 @@ pub fn start_background() {
|
|||
if quiet && due && run_state().is_running() {
|
||||
let _ = save_snapshot();
|
||||
}
|
||||
// (Only when a box exists on this Docker: the holder is its volume's seat-saver.)
|
||||
if matches!(probe(), Run::Asleep | Run::Running) {
|
||||
ensure_holder();
|
||||
}
|
||||
refresh();
|
||||
maybe_scheduled_update();
|
||||
update_when_idle(idle);
|
||||
|
|
@ -831,10 +942,13 @@ fn viewer_open() -> bool {
|
|||
|
||||
// ---------- snapshots (the durable store) ----------
|
||||
|
||||
/// Save the bot's home folder to this Mac. Returns its size in bytes.
|
||||
/// Save the bot's home folder to this Mac, as a new generation. Returns its size in bytes.
|
||||
pub fn save_snapshot() -> Result<u64, String> {
|
||||
let _ = std::fs::create_dir_all(data_dir());
|
||||
let tmp = data_dir().join("snapshot.tmp");
|
||||
let dir = snapshots_dir();
|
||||
let _ = std::fs::create_dir_all(&dir);
|
||||
let stamp = chrono::Local::now().format("%Y-%m-%d-%H%M%S");
|
||||
let tmp = dir.join(format!("snapshot-{stamp}.tmp"));
|
||||
let final_path = dir.join(format!("snapshot-{stamp}.tar.gz"));
|
||||
let file = std::fs::File::create(&tmp).map_err(|e| e.to_string())?;
|
||||
let mut child = docker()
|
||||
// Caches are left out: the browser's grow to hundreds of MB and come back by themselves.
|
||||
|
|
@ -873,15 +987,17 @@ pub fn save_snapshot() -> Result<u64, String> {
|
|||
let _ = std::fs::remove_file(&tmp);
|
||||
return Err("快照沒有存成功".into());
|
||||
}
|
||||
std::fs::rename(&tmp, snapshot_path()).map_err(|e| e.to_string())?;
|
||||
let n = std::fs::metadata(snapshot_path()).map(|m| m.len()).unwrap_or(0);
|
||||
log(&format!("已儲存快照（{}）。", human(n)));
|
||||
std::fs::rename(&tmp, &final_path).map_err(|e| e.to_string())?;
|
||||
let n = std::fs::metadata(&final_path).map(|m| m.len()).unwrap_or(0);
|
||||
prune_snapshots();
|
||||
log(&format!("已儲存快照（{}，共 {} 代）。", human(n), list_snapshots().len()));
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
fn copy_in_snapshot() -> Result<(), String> {
|
||||
/// Put a saved snapshot's files back into the box's home folder.
|
||||
fn copy_in_snapshot(src: &std::path::Path) -> Result<(), String> {
|
||||
use std::io::Write;
|
||||
let data = std::fs::read(snapshot_path()).map_err(|_| "還沒有快照可以還原".to_string())?;
|
||||
let data = std::fs::read(src).map_err(|_| "還沒有快照可以還原".to_string())?;
|
||||
let mut child = docker()
|
||||
.args(["exec", "-i", container(), "sh", "-c", &format!("tar -C {HOME} -xzf - && chown -R bot:bot {HOME} && echo '[box-copy-in] result outcome=hydrated' >> /tmp/anybot-copy-in.log")])
|
||||
.stdin(Stdio::piped())
|
||||
|
|
@ -1030,8 +1146,15 @@ fn run_rebuild(kind: Rebuild, phase: &dyn Fn(&str, &str)) -> Result<(), String>
|
|||
Ok(())
|
||||
}
|
||||
Rebuild::Reset => {
|
||||
// The box's home is the only copy of every bot's logins: never wipe it without a
|
||||
// snapshot of what's in it first — reset becomes undoable.
|
||||
if matches!(probe(), Run::Running) {
|
||||
phase("backing-up", "重設前：先存下現在的電腦");
|
||||
save_snapshot().map_err(|e| format!("重設取消了：存不下現在的狀態（{e}）。先處理好這個，重設才救得回來。"))?;
|
||||
}
|
||||
phase("cleaning-up", "拆掉舊的電腦");
|
||||
let _ = dk(&["rm", "-f", container()]);
|
||||
let _ = dk(&["rm", "-f", &holder_name()]);
|
||||
phase("wiping", "清除電腦裡的資料");
|
||||
let _ = dk(&["volume", "rm", "-f", VOLUME]);
|
||||
phase("creating", "開一台全新的電腦");
|
||||
|
|
@ -1042,8 +1165,16 @@ fn run_rebuild(kind: Rebuild, phase: &dyn Fn(&str, &str)) -> Result<(), String>
|
|||
wait_ready(BOOT_WAIT_MS)
|
||||
}
|
||||
Rebuild::Restore => {
|
||||
// Say which snapshot to restore before saving the current state (saving adds a
|
||||
// newer "latest" that restore must not pick up).
|
||||
let src = snapshot_path();
|
||||
if matches!(probe(), Run::Running) {
|
||||
phase("backing-up", "復原前：先存下現在的電腦");
|
||||
save_snapshot().map_err(|e| format!("復原取消了：存不下現在的狀態（{e}）。先處理好這個，最近的變更才不會無聲消失。"))?;
|
||||
}
|
||||
phase("cleaning-up", "拆掉舊的電腦和它的資料");
|
||||
let _ = dk(&["rm", "-f", container()]);
|
||||
let _ = dk(&["rm", "-f", &holder_name()]);
|
||||
let _ = dk(&["volume", "rm", "-f", VOLUME]);
|
||||
phase("creating", "從快照開一台新的電腦");
|
||||
if !image_exists() {
|
||||
|
|
@ -1052,7 +1183,7 @@ fn run_rebuild(kind: Rebuild, phase: &dyn Fn(&str, &str)) -> Result<(), String>
|
|||
create_container()?;
|
||||
wait_ready(BOOT_WAIT_MS)?;
|
||||
phase("restoring", "把快照裡的檔案放回去");
|
||||
copy_in_snapshot()
|
||||
copy_in_snapshot(&src)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1276,8 +1407,9 @@ fn box_scale() -> Result<(f64, f64), String> {
|
|||
/// A step on the box's screen as a shell script (for `sh -c` inside the box, as the bot user),
|
||||
/// the text it reads on stdin (typing), and what to tell the bot. Coordinates are picture pixels
|
||||
/// times `scale` (desktop pixels per picture pixel; 1 for a person's viewer, which shows the
|
||||
/// desktop at full size).
|
||||
pub fn box_script(name: &str, v: &Value, scale: (f64, f64)) -> Result<(String, Option<String>, String), String> {
|
||||
/// desktop at full size). `inline` puts typed text inside the script quoted (a batch runs its
|
||||
/// steps in one exec with one stdin, so steps can't ride on it).
|
||||
pub fn box_script(name: &str, v: &Value, scale: (f64, f64), inline: bool) -> Result<(String, Option<String>, String), String> {
|
||||
// A control's number from the latest picture: its centre on the desktop.
|
||||
let by_number = |p: &Value| -> Option<Result<((i64, i64), String), String>> {
|
||||
let r = match &p["ref"] {
|
||||
|
|
@ -1366,13 +1498,13 @@ pub fn box_script(name: &str, v: &Value, scale: (f64, f64)) -> Result<(String, O
|
|||
None => String::new(),
|
||||
};
|
||||
if !text.is_empty() {
|
||||
script.push_str(&format!(" && {}", type_script(&text)));
|
||||
script.push_str(&format!(" && {}", if inline { type_script_inline(&text) } else { type_script(&text) }));
|
||||
}
|
||||
if submit {
|
||||
script.push_str(" && sleep 0.06 && xdotool key --clearmodifiers Return");
|
||||
}
|
||||
let script = script.trim_start_matches(" && ").to_string();
|
||||
(script, Some(text.clone()).filter(|t| !t.is_empty()), format!("已打字（{} 個字）{field}{}。", text.chars().count(), if submit { "，再按了 return" } else { "" }))
|
||||
(script, if inline { None } else { Some(text.clone()).filter(|t| !t.is_empty()) }, format!("已打字（{} 個字）{field}{}。", text.chars().count(), if submit { "，再按了 return" } else { "" }))
|
||||
}
|
||||
"computer_use_press_key" => {
|
||||
let keys = v["key"].as_str().unwrap_or("");
|
||||
|
|
@ -1409,9 +1541,26 @@ fn type_script(text: &str) -> String {
|
|||
.to_string()
|
||||
}
|
||||
|
||||
/// The same, with the text quoted into the script (batch steps: one stdin for the whole chain).
|
||||
fn type_script_inline(text: &str) -> String {
|
||||
let q = crate::tools::sh_quote(text);
|
||||
if text.is_ascii() {
|
||||
return format!("xdotool type --clearmodifiers --delay {TYPE_DELAY_MS} -- {q}");
|
||||
}
|
||||
format!("printf '%s' {q} > /tmp/anybot-paste.txt && for s in clipboard primary; do setsid xclip -selection $s -i /tmp/anybot-paste.txt </dev/null >/dev/null 2>&1 & done; sleep 0.15; \
|
||||
case \"$(xprop -id $(xdotool getactivewindow) WM_CLASS 2>/dev/null | tr A-Z a-z)\" in *term*) k=shift+Insert;; *) k=ctrl+v;; esac; xdotool key --clearmodifiers $k")
|
||||
}
|
||||
|
||||
/// Wait for the screen to stop changing before the picture comes back: two matching probes mean
|
||||
/// it settled. A probe is two small grayscale captures (top-left, center: ~60 ms on the box), so
|
||||
/// a settled screen takes ~250 ms where every step used to pay a fixed 0.5 s — and a page that
|
||||
/// is still loading is waited for (up to ~0.9 s) instead of captured mid-change. Plain sleep
|
||||
/// when `import` is missing.
|
||||
const SETTLE_SH: &str = "{ probe() { { import -window root -crop 160x140+80+30 -depth 8 pgm:-; import -window root -crop 160x160+560+320 -depth 8 pgm:-; } 2>/dev/null | md5sum; }; if command -v import >/dev/null 2>&1; then a=$(probe); i=0; while [ $i -lt 5 ]; do sleep 0.1; b=$(probe); [ \"$a\" = \"$b\" ] && break; a=$b; i=$((i+1)); done; else sleep 0.35; fi; }";
|
||||
|
||||
/// Run a step's script in the box; with `picture`, wait for the screen to settle and return it.
|
||||
fn box_exec(script: &str, stdin: Option<&str>, picture: bool) -> Result<Option<image::RgbImage>, String> {
|
||||
let full = if picture { format!("{{ {script}; }} || exit $?; sleep {:.2}; import -window root png:- 2>/dev/null || true", crate::computer::SETTLE_MS as f64 / 1000.0) } else { script.to_string() };
|
||||
let full = if picture { format!("{{ {script}; }} || exit $?; {SETTLE_SH}; import -window root -quality 90 jpeg:- 2>/dev/null || true") } else { script.to_string() };
|
||||
// Always an stdin (closed at once when there's no text), as `docker exec -i` expects.
|
||||
let out = match dk_with_input(&["exec", "-i", "-u", "bot", "-e", "DISPLAY=:1", container(), "sh", "-c", &full], Some(stdin.unwrap_or("").as_bytes()), 30_000) {
|
||||
Ok(o) => o,
|
||||
|
|
@ -1498,10 +1647,14 @@ pub fn elements_of(v: &Value) -> Vec<crate::marks::Element> {
|
|||
/// controls numbered and the list, in one `docker exec`. After a step it also says when nothing
|
||||
/// changed, so the bot doesn't click the same empty place again and again.
|
||||
fn box_look(script: Option<(&str, Option<&str>)>, after: Option<String>) -> Result<(String, Option<String>), String> {
|
||||
let look = "anybot-a11y --budget 3 > /tmp/anybot-look.json 2>/dev/null || echo '{\"elements\":[]}' > /tmp/anybot-look.json; \
|
||||
wc -c < /tmp/anybot-look.json; cat /tmp/anybot-look.json; import -window root png:- 2>/dev/null";
|
||||
let t_all = Instant::now();
|
||||
// The picture is captured by a background `import` while the controls are read (the a11y
|
||||
// read is the slow part on web pages); a JPEG costs less to make and to ship than a PNG.
|
||||
let look = "( import -window root -quality 90 jpeg:- 2>/dev/null > /tmp/anybot-look.jpg & ip=$!; \
|
||||
sleep 0.18; anybot-a11y --budget 3 > /tmp/anybot-look.json 2>/dev/null || echo '{\"elements\":[]}' > /tmp/anybot-look.json; \
|
||||
wc -c < /tmp/anybot-look.json; cat /tmp/anybot-look.json; wait $ip; cat /tmp/anybot-look.jpg )";
|
||||
let full = match script {
|
||||
Some((s, _)) => format!("{{ {s}; }} || exit $?; sleep {:.2}; {look}", crate::computer::SETTLE_MS as f64 / 1000.0),
|
||||
Some((s, _)) => format!("{{ {s}; }} || exit $?; {SETTLE_SH}; {look}"),
|
||||
None => look.to_string(),
|
||||
};
|
||||
let input = script.and_then(|(_, i)| i).unwrap_or("");
|
||||
|
|
@ -1511,7 +1664,8 @@ fn box_look(script: Option<(&str, Option<&str>)>, after: Option<String>) -> Resu
|
|||
Err(e) if e.is_empty() => return Err("這一步沒有成功".into()),
|
||||
Err(e) => return Err(format!("xdotool：{e}")),
|
||||
};
|
||||
// "<bytes of the list>\n<list JSON><PNG>"
|
||||
let exec_ms = t_all.elapsed().as_millis() as u64;
|
||||
// "<bytes of the list>\n<list JSON><JPEG>"
|
||||
let nl = out.iter().position(|b| *b == b'\n').ok_or("這台電腦沒有回傳畫面")?;
|
||||
let n: usize = String::from_utf8_lossy(&out[..nl]).trim().parse().map_err(|_| "這台電腦回傳的畫面讀不懂")?;
|
||||
let json_end = (nl + 1 + n).min(out.len());
|
||||
|
|
@ -1543,9 +1697,32 @@ fn box_look(script: Option<(&str, Option<&str>)>, after: Option<String>) -> Resu
|
|||
} else {
|
||||
text.push_str(&format!("\n可以操作的東西（圖上同樣的編號，用 ref 點）：\n{}", lines.join("\n")));
|
||||
}
|
||||
// Step timing (for the 電腦 card): how long the box's side took vs. the whole step.
|
||||
{
|
||||
let total_ms = t_all.elapsed().as_millis() as u64;
|
||||
let mut s = STEP_TIMES.lock().unwrap_or_else(|e| e.into_inner());
|
||||
s.push((exec_ms, total_ms));
|
||||
let n = s.len();
|
||||
if n > 40 {
|
||||
s.drain(..n - 40);
|
||||
}
|
||||
}
|
||||
Ok((text, Some(base64::engine::general_purpose::STANDARD.encode(jpeg))))
|
||||
}
|
||||
|
||||
/// The box's latest steps, timed: (count, average docker-exec ms, average whole-step ms).
|
||||
/// Whole step = exec (xdotool + settle + 控件 + 截圖) + 這裡的編號繪製和壓縮.
|
||||
pub fn step_stats() -> (usize, u64, u64) {
|
||||
let s = STEP_TIMES.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if s.is_empty() {
|
||||
return (0, 0, 0);
|
||||
}
|
||||
let n = s.len() as u64;
|
||||
(s.len(), s.iter().map(|(a, _)| *a).sum::<u64>() / n, s.iter().map(|(_, b)| *b).sum::<u64>() / n)
|
||||
}
|
||||
|
||||
static STEP_TIMES: Mutex<Vec<(u64, u64)>> = Mutex::new(Vec::new());
|
||||
|
||||
/// Programs people name, as the box knows them.
|
||||
fn box_program(app: &str) -> Option<&'static str> {
|
||||
let a = app.trim().to_lowercase();
|
||||
|
|
@ -1601,10 +1778,51 @@ pub fn computer_run(name: &str, v: &Value) -> Result<(String, Option<String>), S
|
|||
let note = open_on_box(v["app"].as_str().unwrap_or(""))?;
|
||||
return box_look(Some(("true", None)), Some(note));
|
||||
}
|
||||
let (script, stdin, note) = box_script(name, v, box_scale()?)?;
|
||||
if name == "computer_use_batch" {
|
||||
return box_batch(v, box_scale()?);
|
||||
}
|
||||
let (script, stdin, note) = box_script(name, v, box_scale()?, false)?;
|
||||
box_look(Some((&script, stdin.as_deref())), Some(note))
|
||||
}
|
||||
|
||||
const BATCH_MARK: &str = "/tmp/anybot-batch.done";
|
||||
|
||||
/// Do a planned sequence of steps on the box in one round trip: the steps' scripts chained with
|
||||
/// `&&` (so the chain stops at the first failure), one marker line per step that ran, and one
|
||||
/// picture at the end — like computer_batch everywhere else, and one `docker exec` for all of
|
||||
/// them instead of one per step.
|
||||
fn box_batch(v: &Value, scale: (f64, f64)) -> Result<(String, Option<String>), String> {
|
||||
let acts = v["actions"].as_array().ok_or("computer_use_batch 要給 actions：動作陣列，每個有 action（click、typing、press_key、scroll、mouse_move、drag）和它的參數")?;
|
||||
if acts.is_empty() {
|
||||
return Err("actions 是空的".into());
|
||||
}
|
||||
if acts.len() > crate::computer::MAX_BATCH {
|
||||
return Err(format!("一次最多 {} 個動作（{} 個太多了）", crate::computer::MAX_BATCH, acts.len()));
|
||||
}
|
||||
let mut parts = Vec::new();
|
||||
let mut notes = Vec::new();
|
||||
for (i, a) in acts.iter().enumerate() {
|
||||
let name = crate::computer::action_tool(a["action"].as_str().unwrap_or(""))?;
|
||||
let (script, _, note) = box_script(name, a, scale, true)?;
|
||||
parts.push(format!("{{ {script} && echo x >> {BATCH_MARK}; }}"));
|
||||
notes.push(format!("{}. {note}", i + 1));
|
||||
}
|
||||
box_exec(&format!("rm -f {BATCH_MARK}; {} ; true", parts.join(" && ")), None, false)?;
|
||||
let done = dk(&["exec", container(), "sh", "-c", &format!("wc -l < {BATCH_MARK} 2>/dev/null || echo 0")])
|
||||
.ok()
|
||||
.and_then(|s| s.trim().parse::<usize>().ok())
|
||||
.unwrap_or(0)
|
||||
.min(acts.len());
|
||||
let _ = dk(&["exec", container(), "sh", "-c", &format!("rm -f {BATCH_MARK}")]);
|
||||
let note = if done == acts.len() {
|
||||
format!("已依序做完 {done} 步：\n{}", notes.join("\n"))
|
||||
} else {
|
||||
let ran: Vec<&str> = notes.iter().take(done).map(String::as_str).collect();
|
||||
format!("做完 {done}/{} 步，第 {} 步沒有成功（後面沒有做）：\n{}", acts.len(), done + 1, ran.join("\n"))
|
||||
};
|
||||
box_look(Some(("true", None)), Some(note))
|
||||
}
|
||||
|
||||
// ---------- text a person types in a viewer ----------
|
||||
|
||||
/// Text from a person's viewer (any language): typed or pasted into the window in front, like
|
||||
|
|
@ -1762,28 +1980,57 @@ mod tests {
|
|||
let s = (1.25, 1.25);
|
||||
// Picture pixels → desktop pixels; never `mousemove --sync` (it hangs when the pointer is
|
||||
// already there).
|
||||
let (sc, input, note) = box_script("computer_use_click", &serde_json::json!({"x": 512, "y": 320}), s).unwrap();
|
||||
let (sc, input, note) = box_script("computer_use_click", &serde_json::json!({"x": 512, "y": 320}), s, false).unwrap();
|
||||
assert_eq!(sc, "xdotool mousemove 640 400 sleep 0.03 click --repeat 1 --delay 80 1");
|
||||
assert!(input.is_none() && note.contains("(512, 320)"));
|
||||
// Keys held for a step are let go even when it fails.
|
||||
let (sc, _, _) = box_script("computer_use_click", &serde_json::json!({"x": 8, "y": 8, "modifier_keys": ["cmd", "shift"], "count": 2}), s).unwrap();
|
||||
let (sc, _, _) = box_script("computer_use_click", &serde_json::json!({"x": 8, "y": 8, "modifier_keys": ["cmd", "shift"], "count": 2}), s, false).unwrap();
|
||||
assert!(sc.starts_with("xdotool keydown ctrl keydown shift mousemove 10 10") && sc.ends_with("rc=$?; xdotool keyup shift keyup ctrl; [ $rc = 0 ]"), "{sc}");
|
||||
assert!(!sc.contains("--sync"));
|
||||
// Text goes in on stdin (no quoting to get wrong); return after it when asked.
|
||||
let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "echo 'hi' && ls", "submit": true}), s).unwrap();
|
||||
let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "echo 'hi' && ls", "submit": true}), s, false).unwrap();
|
||||
assert_eq!(input.as_deref(), Some("echo 'hi' && ls"));
|
||||
assert!(sc.starts_with("xdotool type --clearmodifiers --delay 12 --file -") && sc.ends_with("xdotool key --clearmodifiers Return"), "{sc}");
|
||||
let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "中文"}), s).unwrap();
|
||||
let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "中文"}), s, false).unwrap();
|
||||
assert!(sc.contains("xclip") && input.as_deref() == Some("中文"));
|
||||
// Keys by name, a character typed, several in order.
|
||||
let (sc, _, _) = box_script("computer_use_press_key", &serde_json::json!({"key": "slash @ cmd+a"}), s).unwrap();
|
||||
let (sc, _, _) = box_script("computer_use_press_key", &serde_json::json!({"key": "slash @ cmd+a"}), s, false).unwrap();
|
||||
assert_eq!(sc, "xdotool key --clearmodifiers slash sleep 0.02 type --clearmodifiers -- '@' sleep 0.02 key --clearmodifiers ctrl+a");
|
||||
// A drag walks its path in small steps and lets go at the end.
|
||||
let (sc, _, _) = box_script("computer_use_drag", &serde_json::json!({"path": [{"x": 0, "y": 0}, {"x": 48, "y": 0}]}), s).unwrap();
|
||||
let (sc, _, _) = box_script("computer_use_drag", &serde_json::json!({"path": [{"x": 0, "y": 0}, {"x": 48, "y": 0}]}), s, false).unwrap();
|
||||
assert!(sc.starts_with("xdotool mousemove 0 0 sleep 0.03 mousedown 1") && sc.contains("mousemove 60 0") && sc.ends_with("mouseup 1"), "{sc}");
|
||||
// No app numbers on this desktop: said, not guessed.
|
||||
assert!(box_script("computer_use_click", &serde_json::json!({"ref": "e3"}), s).is_err());
|
||||
assert!(box_script("computer_use_scroll", &serde_json::json!({"direction": "sideways"}), s).is_err());
|
||||
assert!(box_script("computer_use_click", &serde_json::json!({"ref": "e3"}), s, false).is_err());
|
||||
assert!(box_script("computer_use_scroll", &serde_json::json!({"direction": "sideways"}), s, false).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_steps_carry_their_text_inline() {
|
||||
let s = (1.0, 1.0);
|
||||
// A batch runs its steps in one exec with one stdin: the text rides in the script, quoted.
|
||||
let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "it's a 'test'"}), s, true).unwrap();
|
||||
assert!(input.is_none(), "no stdin for a batch step");
|
||||
assert!(sc.starts_with("xdotool type --clearmodifiers --delay 12 -- 'it'\\''s a '\\''test'\\'''"), "{sc}");
|
||||
// Chinese in a batch: the paste path, text written inline.
|
||||
let (sc, input, _) = box_script("computer_use_typing", &serde_json::json!({"value": "中文", "submit": true}), s, true).unwrap();
|
||||
assert!(input.is_none() && sc.contains("printf '%s' '中文' > /tmp/anybot-paste.txt") && sc.ends_with("xdotool key --clearmodifiers Return"), "{sc}");
|
||||
// Batch action names, and the ones a batch must not carry.
|
||||
assert_eq!(crate::computer::action_tool("Click").unwrap(), "computer_use_click");
|
||||
assert_eq!(crate::computer::action_tool("typing").unwrap(), "computer_use_typing");
|
||||
assert!(crate::computer::action_tool("screenshot").is_err());
|
||||
assert!(crate::computer::action_tool("open_app").is_err());
|
||||
assert!(crate::computer::action_tool("").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_names_carry_their_time() {
|
||||
// Names sort by time, and the stamp is readable from the name (copies lose file dates).
|
||||
let p = std::path::Path::new("/x/snapshot-2026-11-05-123456.tar.gz");
|
||||
let t = snapshot_stamp(p).expect("stamp parses");
|
||||
assert_eq!(t.format("%Y-%m-%d %H:%M:%S").to_string(), "2026-11-05 12:34:56");
|
||||
let legacy = std::path::Path::new("/x/snapshot-2026-10-01-000000-legacy.tar.gz");
|
||||
assert_eq!(snapshot_stamp(legacy).expect("legacy parses").format("%Y-%m-%d").to_string(), "2026-10-01");
|
||||
assert!(snapshot_stamp(std::path::Path::new("/x/notes.txt")).is_none());
|
||||
}
|
||||
|
||||
/// A box made from this image (`ANYBOT_BOX_NAME=<test box> cargo test --release -- --ignored
|
||||
|
|
@ -1876,6 +2123,136 @@ mod tests {
|
|||
let _ = box_exec("xdotool key --clearmodifiers ctrl+d", None, false);
|
||||
}
|
||||
|
||||
/// Uses the real Docker: `cargo test --release -- --ignored box_batch_in_one_round_trip --nocapture`.
|
||||
/// A planned sequence runs in one round trip: ASCII and Chinese typing and a submit land in the
|
||||
/// front app, the note lists every step, and one picture comes back.
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn box_batch_in_one_round_trip() {
|
||||
ensure_running().expect("box starts");
|
||||
let _ = dk(&["exec", "-u", "bot", container(), "rm", "-f", "/home/bot/batch.txt"]);
|
||||
let (txt, img) = computer_run("computer_use_open_app", &serde_json::json!({"app": "終端機"})).unwrap();
|
||||
assert!(img.is_some(), "{txt}");
|
||||
let t = Instant::now();
|
||||
let (txt, img) = computer_run(
|
||||
"computer_use_batch",
|
||||
&serde_json::json!({"actions": [
|
||||
{"action": "typing", "value": "echo batch-"},
|
||||
{"action": "typing", "value": "中文"},
|
||||
{"action": "typing", "value": "ok > /home/bot/batch.txt", "submit": true}
|
||||
]}),
|
||||
)
|
||||
.unwrap();
|
||||
println!("batch of 3: {} ms\n{txt}", t.elapsed().as_millis());
|
||||
assert!(img.is_some() && txt.contains("已依序做完 3 步"), "{txt}");
|
||||
std::thread::sleep(Duration::from_millis(600));
|
||||
let out = dk(&["exec", "-u", "bot", container(), "cat", "/home/bot/batch.txt"]).unwrap_or_default();
|
||||
println!("typed: {out}");
|
||||
assert_eq!(out.trim(), "batch-中文ok");
|
||||
// Too many, empty: said before anything runs.
|
||||
assert!(computer_run("computer_use_batch", &serde_json::json!({"actions": []})).is_err());
|
||||
let many: Vec<_> = (0..9).map(|_| serde_json::json!({"action": "press_key", "key": "f15"})).collect();
|
||||
assert!(computer_run("computer_use_batch", &serde_json::json!({"actions": many})).is_err());
|
||||
let _ = box_exec("xdotool key --clearmodifiers ctrl+d", None, false);
|
||||
}
|
||||
|
||||
/// Uses the real Docker: `cargo test --release -- --ignored box_real_usage_drill --nocapture`.
|
||||
/// The whole screen path, drilled on a page that reports what really happened: a form filled
|
||||
/// and sent by one batch, a bare-x/y click and a drag landing on a canvas by the page's own
|
||||
/// record, scrolling revealing a field that is then clicked and typed by number. Timed.
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn box_real_usage_drill() {
|
||||
ensure_running().expect("box starts");
|
||||
let cancel = AtomicBool::new(false);
|
||||
let page = r##"<!doctype html><meta charset=utf-8><title>drill</title><body style="margin:0;font:16px sans-serif"><div style="padding:20px"><input id=q1 aria-label="Name" style="width:200px;font-size:16px"> <input id=q2 aria-label="Email" style="width:220px;font-size:16px"> <label>Agree <input type=checkbox id=ck></label> <button id=go onclick="document.getElementById('res').textContent='SUB:'+q1.value+':'+q2.value+':'+ck.checked">Send</button><div id=res>idle</div><canvas id=cv width=400 height=100 style="border:1px solid #777;display:block;margin-top:12px"></canvas><div id=dragx></div><div style="height:1200px">keep scrolling down</div><input id=q3 aria-label="Bottom field" style="width:200px;font-size:16px"></div><script>cv.addEventListener('click',e=>{document.title='cv:'+Math.round(e.offsetX)+','+Math.round(e.offsetY)});let down=false;cv.addEventListener('mousedown',()=>{down=true;dragx.textContent=''});window.addEventListener('mousemove',e=>{if(down)dragx.textContent=String(Math.round(e.clientX))});window.addEventListener('mouseup',()=>{down=false});</script>"##;
|
||||
dk_with_input(&["exec", "-i", "-u", "bot", container(), "sh", "-c", "mkdir -p /tmp/anybot-page && cat > /tmp/anybot-page/index.html && (cd /tmp/anybot-page && setsid python3 -m http.server 8765 --bind 127.0.0.1 >/dev/null 2>&1 &) ; sleep 0.5"], Some(page.as_bytes()), 10_000).expect("page served");
|
||||
// A clean slate: every earlier drill left its page (and its state) in a tab. Closing all
|
||||
// pages closes the window too; browser_open starts the browser again with our page.
|
||||
let close_pages = "python3 -c \"import json,urllib.request\nfor t in json.load(urllib.request.urlopen('http://127.0.0.1:9222/json/list')):\n if t.get('type')=='page':\n try: urllib.request.urlopen('http://127.0.0.1:9222/json/close/'+t['id'])\n except Exception: pass\"";
|
||||
let _ = dk(&["exec", "-u", "bot", container(), "sh", "-c", close_pages]);
|
||||
std::thread::sleep(Duration::from_millis(600));
|
||||
let opened = crate::botbrowser::in_box(|| crate::web::run_full("browser_open", r#"{"url":"http://127.0.0.1:8765/"}"#, &cancel));
|
||||
assert!(opened.1, "{}", opened.0);
|
||||
std::thread::sleep(Duration::from_millis(1500));
|
||||
let js = |e: &str| crate::botbrowser::in_box(|| crate::botbrowser::eval_current(None, e)).map(|(_, v)| v.as_str().unwrap_or("").to_string()).unwrap_or_default();
|
||||
assert_eq!(js("document.title"), "drill", "the drill page is up");
|
||||
let number = |txt: &str, what: &str| -> String {
|
||||
txt.lines().find(|l| l.starts_with('[') && l.contains(what)).and_then(|l| l.trim().strip_prefix('[')).and_then(|l| l.split(']').next()).unwrap_or_else(|| panic!("no number for {what} in:\n{txt}")).to_string()
|
||||
};
|
||||
let mut times: Vec<(String, u64)> = vec![];
|
||||
macro_rules! step {
|
||||
($name:expr, $tool:expr, $args:expr) => {{
|
||||
let t0 = Instant::now();
|
||||
let r = computer_run($tool, &$args).unwrap();
|
||||
times.push((format!("{} ({})", $name, $tool), t0.elapsed().as_millis() as u64));
|
||||
r
|
||||
}};
|
||||
}
|
||||
// 1. One look: the numbered picture.
|
||||
let (txt, img) = step!("look", "computer_use_screenshot", serde_json::json!({}));
|
||||
if let Some(b) = &img {
|
||||
let _ = std::fs::write(std::env::temp_dir().join("anybot-drill-1.jpg"), base64::engine::general_purpose::STANDARD.decode(b).unwrap_or_default());
|
||||
}
|
||||
// 2. The form, all six steps in ONE batch call: clicks by number, Chinese and ASCII
|
||||
// typing, and the Send button — the page says what arrived.
|
||||
let batch = serde_json::json!({"actions": [
|
||||
{"action": "click", "ref": number(&txt, "Name")},
|
||||
{"action": "typing", "value": "王小明"},
|
||||
{"action": "click", "ref": number(&txt, "Email")},
|
||||
{"action": "typing", "value": "a@b.c"},
|
||||
{"action": "click", "ref": number(&txt, "Agree")},
|
||||
{"action": "click", "ref": number(&txt, "Send")}
|
||||
]});
|
||||
let (btxt, bimg) = step!("fill+send form", "computer_use_batch", batch);
|
||||
assert!(btxt.contains("已依序做完 6 步"), "{btxt}");
|
||||
assert_eq!(js("document.getElementById('res').textContent"), "SUB:王小明:a@b.c:true", "the batch really filled and sent the form");
|
||||
if let Some(b) = &bimg {
|
||||
let _ = std::fs::write(std::env::temp_dir().join("anybot-drill-2.jpg"), base64::engine::general_purpose::STANDARD.decode(b).unwrap_or_default());
|
||||
}
|
||||
// 3. A click at bare x,y (no number on a canvas): the canvas records where it landed.
|
||||
// The page knows CSS pixels; the pointer lives on the desktop — add the window's own
|
||||
// place and toolbar height (what the page's viewport sits under).
|
||||
let geo: Value = serde_json::from_str(&js("JSON.stringify({sx:screenX,sy:screenY,ow:outerWidth,oh:outerHeight,iw:innerWidth,ih:innerHeight})")).unwrap();
|
||||
println!("window: {geo}");
|
||||
let off_x = geo["sx"].as_f64().unwrap() + (geo["ow"].as_f64().unwrap() - geo["iw"].as_f64().unwrap()) / 2.0;
|
||||
let off_y = geo["sy"].as_f64().unwrap() + (geo["oh"].as_f64().unwrap() - geo["ih"].as_f64().unwrap()) - (geo["ow"].as_f64().unwrap() - geo["iw"].as_f64().unwrap()) / 2.0;
|
||||
let rect: Value = serde_json::from_str(&js("JSON.stringify(document.getElementById('cv').getBoundingClientRect())")).unwrap();
|
||||
let (sx, sy) = box_scale().unwrap();
|
||||
let (cx, cy) = ((off_x + rect["left"].as_f64().unwrap() + 200.0) / sx, (off_y + rect["top"].as_f64().unwrap() + 50.0) / sy);
|
||||
println!("rect: {rect} off=({off_x},{off_y}) scale=({sx},{sy}) picture=({cx},{cy})");
|
||||
step!("canvas dot at (200,50)", "computer_use_click", serde_json::json!({"x": cx.round() as i64, "y": cy.round() as i64}));
|
||||
let title = js("document.title");
|
||||
println!("canvas saw: {title}");
|
||||
let mut xy = title.trim_start_matches("cv:").split(',').map(|s| s.parse::<f64>().unwrap());
|
||||
let (got_x, got_y) = (xy.next().unwrap(), xy.next().unwrap());
|
||||
assert!((got_x - 200.0).abs() <= 15.0 && (got_y - 50.0).abs() <= 15.0, "canvas click off: {got_x},{got_y}");
|
||||
// 4. A drag across the canvas: the page records the last pointer position it felt.
|
||||
let y = (off_y + rect["top"].as_f64().unwrap() + 50.0) / sy;
|
||||
let (x1, x2) = ((off_x + rect["left"].as_f64().unwrap() + 20.0) / sx, (off_x + rect["left"].as_f64().unwrap() + 320.0) / sx);
|
||||
step!("drag 20→320", "computer_use_drag", serde_json::json!({"path": [{"x": x1.round() as i64, "y": y.round() as i64}, {"x": x2.round() as i64, "y": y.round() as i64}]}));
|
||||
let dragx = js("document.getElementById('dragx').textContent").parse::<f64>().unwrap_or(-1.0);
|
||||
println!("drag ended at clientX {dragx} (wanted ~{})", rect["left"].as_f64().unwrap() + 320.0);
|
||||
assert!((dragx - (rect["left"].as_f64().unwrap() + 320.0)).abs() <= 20.0, "drag ended at {dragx}");
|
||||
// 5. Scrolling reveals the field at the bottom; it gets a number, and clicking that
|
||||
// number focuses it for typing.
|
||||
let (stxt, _) = step!("scroll down", "computer_use_scroll", serde_json::json!({"direction": "down", "amount": 10}));
|
||||
let scroll_y: f64 = js("String(window.scrollY)").parse().unwrap_or(0.0);
|
||||
assert!(scroll_y > 100.0, "the page scrolled: {scroll_y}");
|
||||
let (ctxt, _) = step!("bottom click", "computer_use_click", serde_json::json!({"ref": number(&stxt, "Bottom field")}));
|
||||
step!("bottom type", "computer_use_typing", serde_json::json!({"ref": number(&ctxt, "Bottom field"), "value": "end"}));
|
||||
assert_eq!(js("document.getElementById('q3').value"), "end", "the field at the bottom got the typing");
|
||||
// 6. Same spot twice: the second says so, in words.
|
||||
let (again, _) = computer_run("computer_use_click", &serde_json::json!({"x": 700, "y": 550})).unwrap();
|
||||
let (again2, _) = computer_run("computer_use_click", &serde_json::json!({"x": 700, "y": 550})).unwrap();
|
||||
println!("twice at (700,550): first「{}」second「{}」", again.lines().next().unwrap_or(""), again2.lines().find(|l| l.contains("完全一樣")).unwrap_or("(said nothing)"));
|
||||
println!("\n--- step times ---");
|
||||
for (n, ms) in × {
|
||||
println!("{ms:>6} ms {n}");
|
||||
}
|
||||
println!("total of {} timed steps: {} ms", times.len(), times.iter().map(|(_, m)| m).sum::<u64>());
|
||||
}
|
||||
|
||||
/// Uses the real Docker: `cargo test --release -- --ignored box_end_to_end --nocapture`.
|
||||
#[test]
|
||||
#[ignore]
|
||||
|
|
|
|||
|
|
@ -12,10 +12,29 @@ use base64::Engine;
|
|||
use serde_json::{json, Value};
|
||||
use std::sync::Mutex;
|
||||
|
||||
pub const NAMES: [&str; 9] = ["computer_use_screenshot", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app"];
|
||||
pub const NAMES: [&str; 10] = ["computer_use_screenshot", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app", "computer_use_batch"];
|
||||
/// Wait after an action before taking the picture that comes back with it (menus open, pages
|
||||
/// start to change; estimated).
|
||||
/// start to change; estimated). The box waits for the screen to stop changing instead, up to
|
||||
/// about this long.
|
||||
pub const SETTLE_MS: u64 = 500;
|
||||
/// Most actions one `computer_use_batch` call may carry (Anthropic's computer_batch guidance:
|
||||
/// short self-contained sequences, one picture at the end).
|
||||
pub const MAX_BATCH: usize = 8;
|
||||
|
||||
/// A batch step's short action name → the tool it runs. Screen reads, apps and permissions stay
|
||||
/// single steps on purpose: a batch is for input you already planned from the current picture.
|
||||
pub fn action_tool(a: &str) -> Result<&'static str, String> {
|
||||
match a.trim().to_lowercase().as_str() {
|
||||
"click" => Ok("computer_use_click"),
|
||||
"typing" | "type" => Ok("computer_use_typing"),
|
||||
"press_key" | "key" => Ok("computer_use_press_key"),
|
||||
"scroll" => Ok("computer_use_scroll"),
|
||||
"mouse_move" | "move" => Ok("computer_use_mouse_move"),
|
||||
"drag" => Ok("computer_use_drag"),
|
||||
"" => Err("batch 的每個動作都要有 action（click、typing、press_key、scroll、mouse_move、drag）".into()),
|
||||
other => Err(format!("batch 裡沒有「{other}」這個動作（可用：click、typing、press_key、scroll、mouse_move、drag；要打開 App 請單獨用 computer_use_open_app）")),
|
||||
}
|
||||
}
|
||||
|
||||
/// The person took control back from the bot (the floating bar): screen actions wait.
|
||||
pub static HELD_BY_PERSON: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
|
||||
|
|
@ -209,6 +228,7 @@ pub fn label(name: &str) -> &'static str {
|
|||
"computer_use_drag" => "拖曳",
|
||||
"computer_use_check_permissions" => "檢查螢幕權限",
|
||||
"computer_use_open_app" => "打開 App",
|
||||
"computer_use_batch" => "依序操作",
|
||||
_ => "操作畫面",
|
||||
}
|
||||
}
|
||||
|
|
@ -241,6 +261,7 @@ pub fn describe(name: &str, args: &str) -> String {
|
|||
"computer_use_drag" => format!("{} 個點", v["path"].as_array().map(|a| a.len()).unwrap_or(0)),
|
||||
"computer_use_check_permissions" => "螢幕錄製、輔助使用".into(),
|
||||
"computer_use_open_app" => v["app"].as_str().filter(|a| !a.is_empty()).unwrap_or("").to_string(),
|
||||
"computer_use_batch" => format!("依序 {} 個動作", v["actions"].as_array().map(|a| a.len()).unwrap_or(0)),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
|
@ -269,6 +290,8 @@ pub fn specs_for(computers: &[String], default: &str) -> Vec<Value> {
|
|||
"parameters": {"type": "object", "properties": {"path": {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "button": {"type": "string", "enum": ["left", "right"]}, "modifier_keys": mods}, "required": ["path"]}}),
|
||||
json!({"name": "computer_use_check_permissions", "description": "Check whether the screen can be seen and the mouse and keyboard used on that computer.",
|
||||
"parameters": {"type": "object", "properties": {}}}),
|
||||
json!({"name": "computer_use_batch", "description": format!("Do a short planned sequence of steps in one call (max {MAX_BATCH}): each item is one action with its own arguments, e.g. {{\"action\":\"click\",\"ref\":\"12\"}}, {{\"action\":\"typing\",\"ref\":\"15\",\"value\":\"cat\"}}, {{\"action\":\"press_key\",\"key\":\"tab\"}}. Use it when the current picture already tells you every step (filling a form, a known shortcut chain): they run in order, the first step that fails stops the rest, and you get one picture at the end. Do not use it to explore, or where step 2 depends on what step 1's screen shows — act one step at a time there."),
|
||||
"parameters": {"type": "object", "properties": {"actions": {"type": "array", "maxItems": MAX_BATCH, "items": {"type": "object", "properties": {"action": {"type": "string", "enum": ["click", "typing", "press_key", "scroll", "mouse_move", "drag"]}, "ref": rf.clone(), "x": {"type": "number"}, "y": {"type": "number"}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "count": {"type": "integer"}, "value": {"type": "string"}, "submit": {"type": "boolean"}, "key": {"type": "string"}, "hold_duration_ms": {"type": "integer"}, "direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "amount": {"type": "integer"}, "path": {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "modifier_keys": mods.clone()}, "required": ["action"]}}}, "required": ["actions"]}}),
|
||||
];
|
||||
// Always named: each computer goes by its own name (the instructions say which is which),
|
||||
// so the bot says whose screen it means and the person can tell where it worked.
|
||||
|
|
@ -447,7 +470,7 @@ pub fn run_by(name: &str, args: &str, who: &str) -> (String, bool, Option<String
|
|||
|
||||
/// Steps that use the mouse or keyboard (and come back with the screen after them).
|
||||
pub fn moves_things(name: &str) -> bool {
|
||||
matches!(name, "computer_use_click" | "computer_use_typing" | "computer_use_press_key" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_drag" | "computer_use_open_app")
|
||||
matches!(name, "computer_use_click" | "computer_use_typing" | "computer_use_press_key" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_drag" | "computer_use_open_app" | "computer_use_batch")
|
||||
}
|
||||
|
||||
/// Ask macOS for a permission (its own prompt, then 系統設定 at that pane) at most once every ten
|
||||
|
|
@ -608,6 +631,35 @@ fn press_keys(keys: &str, hold: f64) -> Result<(), String> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Run a planned sequence of actions, then one picture (in the box the box's own batch runs;
|
||||
/// here, the local screen). The first action that fails stops the rest — acting on a screen
|
||||
/// that isn't where you think it is makes it worse (Anthropic's computer_batch rule).
|
||||
pub fn batch(v: &Value, scale: (f64, f64)) -> Result<String, String> {
|
||||
let acts = v["actions"].as_array().ok_or("computer_use_batch 要給 actions：動作陣列，每個有 action（click、typing、press_key、scroll、mouse_move、drag）和它的參數")?;
|
||||
if acts.is_empty() {
|
||||
return Err("actions 是空的".into());
|
||||
}
|
||||
if acts.len() > MAX_BATCH {
|
||||
return Err(format!("一次最多 {MAX_BATCH} 個動作（{} 個太多了）", acts.len()));
|
||||
}
|
||||
let mut notes = Vec::new();
|
||||
for (i, a) in acts.iter().enumerate() {
|
||||
let name = action_tool(a["action"].as_str().unwrap_or(""))?;
|
||||
match act(name, a, scale) {
|
||||
Ok(n) => notes.push(format!("{}. {n}", i + 1)),
|
||||
Err(e) => {
|
||||
let e = plain_words(e);
|
||||
if notes.is_empty() {
|
||||
return Err(format!("第 {} 步就失敗：{e}", i + 1));
|
||||
}
|
||||
notes.push(format!("{}. 失敗：{e}（後面 {} 個沒有做）", i + 1, acts.len() - i - 1));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(format!("已依序做完 {} 步：\n{}", notes.len(), notes.join("\n")))
|
||||
}
|
||||
|
||||
/// Do one mouse or keyboard step at the screen's own coordinates (`scale`: screen points per
|
||||
/// picture pixel). The bot's tools and a person tapping on the phone's picture both come here:
|
||||
/// one way of acting on a screen.
|
||||
|
|
@ -805,7 +857,10 @@ fn run_inner(name: &str, v: &Value) -> Result<(String, Option<String>), String>
|
|||
if name != "computer_use_open_app" {
|
||||
need_controls()?;
|
||||
}
|
||||
let note = act(name, v, scale_here()).map_err(plain_words)?;
|
||||
let note = match name {
|
||||
"computer_use_batch" => batch(v, scale_here())?,
|
||||
_ => act(name, v, scale_here()).map_err(plain_words)?,
|
||||
};
|
||||
Ok(observe(note))
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue