From 1be088474e3c62ec5a8fa853014cc9fa6ba16112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=80=A7=E9=A9=8A?= Date: Mon, 9 Nov 2026 11:25:17 +0800 Subject: [PATCH] The box really replaces a one-line field's text; a turn that keeps going wrong stops handing steps to the quick model; a missed name says what's close MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The one-line-field check got the step's name as 「#14（輸入框…）」 on the box and never matched, so the select-and-paste never ran there and addresses still came out mangled. It reads both forms now, with a test on the script the box runs. - Trouble twice in a turn and the rest of the turn is the bot's own model's (a quick model guessed its way through Mousepad's save dialog for 79 steps). - A name that isn't on the screen comes back with the names close to it ([12]「New Window」…), so the step is put right at once. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS --- src/agent.rs | 19 ++++++++++++++++++- src/cloudbox.rs | 19 ++++++++++++++++++- src/computer.rs | 2 +- src/marks.rs | 25 ++++++++++++++++++++++++- 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/src/agent.rs b/src/agent.rs index 20f0382..614329e 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -764,6 +764,10 @@ struct Replay { /// Quick steps in a row before the bot's own model looks at the work again (estimated). pub const QUICK_RUN_MAX: usize = 4; +/// Trouble this many times in a turn and the rest of it is the bot's own model's: the quick model +/// guessing its way through an unfamiliar dialog (names that aren't there, unnamed items) only +/// made it longer each time it got the step back. +pub const QUICK_TROUBLE_MAX: usize = 2; /// Steps that only look (nothing on any computer changes): a turn of only these needs no check /// at the end. @@ -785,13 +789,15 @@ pub struct StepPlan { quick_run: usize, /// The way the last acting step went (commands, the screen, a web page). last_route: Option<&'static str>, + /// Steps that went wrong this turn. + troubles: usize, /// Something on a computer may have changed this turn (not only looking). pub changed: bool, } impl StepPlan { pub fn new(quick: Option) -> Self { - StepPlan { quick, main_next: true, quick_run: 0, last_route: None, changed: false } + StepPlan { quick, main_next: true, quick_run: 0, last_route: None, troubles: 0, changed: false } } /// The quick model for this step, or None for the bot's own. @@ -810,6 +816,10 @@ impl StepPlan { pub fn after(&mut self, trouble: bool) { if trouble { self.main_next = true; + self.troubles += 1; + if self.troubles >= QUICK_TROUBLE_MAX && self.quick.take().is_some() { + eprintln!("AnyBot：這一回合出了 {} 次問題，剩下的步驟改用 Bot 自己的模型", self.troubles); + } } } @@ -2039,6 +2049,13 @@ mod tests { assert_eq!(p.pick().as_deref(), Some("q")); p.after(true); assert_eq!(p.pick(), None, "after trouble: the bot's own model"); + // Trouble twice: the rest of the turn is the bot's own model's. + let mut twice = StepPlan::new(Some("q".into())); + twice.pick(); + twice.after(true); + twice.pick(); + twice.after(true); + assert!(twice.quick_is_off() && twice.pick().is_none() && twice.pick().is_none()); for _ in 0..QUICK_RUN_MAX { assert_eq!(p.pick().as_deref(), Some("q")); } diff --git a/src/cloudbox.rs b/src/cloudbox.rs index a1f6ec5..d64015e 100644 --- a/src/cloudbox.rs +++ b/src/cloudbox.rs @@ -1826,7 +1826,9 @@ pub fn box_script(name: &str, v: &Value, scale: (f64, f64)) -> Result<(String, S /// Typing into this control replaces what it holds: a one-line field (輸入框), unless the step /// asks to add to it. A document's text area is left alone (selecting all would replace a file). pub fn replaces_field(what: &str, v: &Value) -> bool { - what.starts_with("輸入框") && !v["append"].as_bool().unwrap_or(false) + // As a step names it (「#14（輸入框「Address and search bar」：…）」) or bare. + let desc = what.strip_prefix('#').and_then(|r| r.split_once('（')).map(|(_, d)| d).unwrap_or(what); + desc.starts_with("輸入框") && !v["append"].as_bool().unwrap_or(false) } fn type_script(text: &str) -> String { @@ -2328,6 +2330,21 @@ pub fn shutdown() { mod tests { use super::*; + #[test] + fn typing_replaces_a_one_line_field_however_it_is_named() { + let none = serde_json::json!({}); + assert!(replaces_field("#14（輸入框「Address and search bar」：about:blank（游標在這））", &none)); + assert!(replaces_field("輸入框「Name」", &none)); + assert!(!replaces_field("#3（文字「Untitled」）", &none), "a text area keeps its text"); + assert!(!replaces_field("#14（輸入框「Search」）", &serde_json::json!({"append": true}))); + // What the box runs: the field clicked, its text selected, the address pasted in one go. + let els = vec![crate::marks::Element { kind: "輸入框".into(), name: "Address and search bar".into(), value: "about:blank".into(), x: 100.0, y: 50.0, w: 400.0, h: 30.0, active: true, flags: vec![], window: "Chromium".into() }]; + let (_, _, spots) = crate::marks::number_keeping(&els, false, &mut Default::default()); + *BOX_SPOTS.lock().unwrap() = Some(spots); + let (script, _) = box_script("computer_use_typing", &serde_json::json!({"ref": "1", "value": "chrome://settings/search", "submit": true}), (1.0, 1.0)).unwrap(); + assert!(script.contains("click 1") && script.contains("ctrl+a") && script.contains("xclip") && script.contains("ctrl+v") && !script.contains("xdotool type"), "{script}"); + } + #[test] fn a_command_asking_for_sudo_is_recognized() { for c in ["sudo apt-get install -y tree", "cd x && sudo make install", "ls; sudo -n true", "(sudo id)", "echo ok | sudo tee /etc/x"] { diff --git a/src/computer.rs b/src/computer.rs index 2e52f33..e3cec90 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -863,7 +863,7 @@ pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result { if let Some((x, y, what)) = &at { mac::click(*x, *y, "left", 1, 0); std::thread::sleep(std::time::Duration::from_millis(150)); - if crate::cloudbox::replaces_field(what.split_once('（').map(|(_, w)| w).unwrap_or(what), v) { + if crate::cloudbox::replaces_field(what, v) { press_keys("cmd+a", KEY_HOLD_SECS)?; std::thread::sleep(std::time::Duration::from_millis(50)); } diff --git a/src/marks.rs b/src/marks.rs index 49b07cb..54c79dd 100644 --- a/src/marks.rs +++ b/src/marks.rs @@ -59,7 +59,27 @@ impl Spots { hits.sort(); match hits.as_slice() { [n] => Ok(*n), - [] => Err(format!("最新的畫面裡沒有叫「{want}」的東西：先看一下畫面（computer_use_screenshot），用清單上的名字或編號。")), + [] => { + // Names that come close (one inside the other, ignoring case), so the step can be + // put right at once instead of guessed again. + let low = want.to_lowercase(); + let mut near: Vec<(usize, String)> = self + .by_number + .iter() + .filter_map(|(n, (_, what))| name_of(what).filter(|nm| !nm.is_empty()).map(|nm| (*n, nm))) + .filter(|(_, nm)| { + let l = nm.to_lowercase(); + !low.is_empty() && (l.contains(&low) || low.contains(&l)) + }) + .collect(); + near.sort(); + near.truncate(6); + if near.is_empty() { + Err(format!("最新的畫面裡沒有叫「{want}」的東西：照最新清單上的名字或編號點，不要猜名字。")) + } else { + Err(format!("最新的畫面裡沒有叫「{want}」的東西。名字相近的有：{}。", near.iter().map(|(n, nm)| format!("[{n}]「{nm}」")).collect::>().join("、"))) + } + } many => Err(format!("最新的畫面裡有 {} 個叫「{want}」的東西（編號 {}）：請用編號指定是哪一個。", many.len(), many.iter().map(|n| n.to_string()).collect::>().join("、"))), } } @@ -342,6 +362,9 @@ mod tests { assert_eq!(spots.named("4").unwrap(), 1); assert_eq!(spots.named("「C」").unwrap(), 3, "C is not AC"); assert!(spots.named("Save").unwrap_err().contains("沒有叫「Save」")); + // A near miss names what is there. + let near = spots.named("自動").unwrap_err(); + assert!(near.contains("名字相近") && near.contains("[4]「自動更新」"), "{near}"); let mut v = serde_json::json!({"actions": [{"action": "click", "name": "4"}, {"action": "click", "ref": "2"}, {"action": "typing", "value": "x"}]}); names_to_numbers(&mut v, Some(&spots)).unwrap(); assert_eq!(v["actions"][0]["ref"], "1");