The box really replaces a one-line field's text; a turn that keeps going wrong stops handing steps to the quick model; a missed name says what's close

- The one-line-field check got the step's name as 「#14(輸入框…)」 on the box and never
  matched, so the select-and-paste never ran there and addresses still came out mangled. It
  reads both forms now, with a test on the script the box runs.
- Trouble twice in a turn and the rest of the turn is the bot's own model's (a quick model
  guessed its way through Mousepad's save dialog for 79 steps).
- A name that isn't on the screen comes back with the names close to it ([12]「New Window」…),
  so the step is put right at once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
This commit is contained in:
王性驊 2026-11-09 11:25:17 +08:00
parent 93ead114cd
commit 1be088474e
4 changed files with 61 additions and 4 deletions

View File

@ -764,6 +764,10 @@ struct Replay {
/// Quick steps in a row before the bot's own model looks at the work again (estimated). /// Quick steps in a row before the bot's own model looks at the work again (estimated).
pub const QUICK_RUN_MAX: usize = 4; pub const QUICK_RUN_MAX: usize = 4;
/// Trouble this many times in a turn and the rest of it is the bot's own model's: the quick model
/// guessing its way through an unfamiliar dialog (names that aren't there, unnamed items) only
/// made it longer each time it got the step back.
pub const QUICK_TROUBLE_MAX: usize = 2;
/// Steps that only look (nothing on any computer changes): a turn of only these needs no check /// Steps that only look (nothing on any computer changes): a turn of only these needs no check
/// at the end. /// at the end.
@ -785,13 +789,15 @@ pub struct StepPlan {
quick_run: usize, quick_run: usize,
/// The way the last acting step went (commands, the screen, a web page). /// The way the last acting step went (commands, the screen, a web page).
last_route: Option<&'static str>, last_route: Option<&'static str>,
/// Steps that went wrong this turn.
troubles: usize,
/// Something on a computer may have changed this turn (not only looking). /// Something on a computer may have changed this turn (not only looking).
pub changed: bool, pub changed: bool,
} }
impl StepPlan { impl StepPlan {
pub fn new(quick: Option<String>) -> Self { pub fn new(quick: Option<String>) -> Self {
StepPlan { quick, main_next: true, quick_run: 0, last_route: None, changed: false } StepPlan { quick, main_next: true, quick_run: 0, last_route: None, troubles: 0, changed: false }
} }
/// The quick model for this step, or None for the bot's own. /// The quick model for this step, or None for the bot's own.
@ -810,6 +816,10 @@ impl StepPlan {
pub fn after(&mut self, trouble: bool) { pub fn after(&mut self, trouble: bool) {
if trouble { if trouble {
self.main_next = true; self.main_next = true;
self.troubles += 1;
if self.troubles >= QUICK_TROUBLE_MAX && self.quick.take().is_some() {
eprintln!("AnyBot:這一回合出了 {} 次問題,剩下的步驟改用 Bot 自己的模型", self.troubles);
}
} }
} }
@ -2039,6 +2049,13 @@ mod tests {
assert_eq!(p.pick().as_deref(), Some("q")); assert_eq!(p.pick().as_deref(), Some("q"));
p.after(true); p.after(true);
assert_eq!(p.pick(), None, "after trouble: the bot's own model"); assert_eq!(p.pick(), None, "after trouble: the bot's own model");
// Trouble twice: the rest of the turn is the bot's own model's.
let mut twice = StepPlan::new(Some("q".into()));
twice.pick();
twice.after(true);
twice.pick();
twice.after(true);
assert!(twice.quick_is_off() && twice.pick().is_none() && twice.pick().is_none());
for _ in 0..QUICK_RUN_MAX { for _ in 0..QUICK_RUN_MAX {
assert_eq!(p.pick().as_deref(), Some("q")); assert_eq!(p.pick().as_deref(), Some("q"));
} }

View File

@ -1826,7 +1826,9 @@ pub fn box_script(name: &str, v: &Value, scale: (f64, f64)) -> Result<(String, S
/// Typing into this control replaces what it holds: a one-line field (輸入框), unless the step /// Typing into this control replaces what it holds: a one-line field (輸入框), unless the step
/// asks to add to it. A document's text area is left alone (selecting all would replace a file). /// asks to add to it. A document's text area is left alone (selecting all would replace a file).
pub fn replaces_field(what: &str, v: &Value) -> bool { pub fn replaces_field(what: &str, v: &Value) -> bool {
what.starts_with("輸入框") && !v["append"].as_bool().unwrap_or(false) // As a step names it (「#14(輸入框「Address and search bar」:…)」) or bare.
let desc = what.strip_prefix('#').and_then(|r| r.split_once('(')).map(|(_, d)| d).unwrap_or(what);
desc.starts_with("輸入框") && !v["append"].as_bool().unwrap_or(false)
} }
fn type_script(text: &str) -> String { fn type_script(text: &str) -> String {
@ -2328,6 +2330,21 @@ pub fn shutdown() {
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn typing_replaces_a_one_line_field_however_it_is_named() {
let none = serde_json::json!({});
assert!(replaces_field("#14(輸入框「Address and search bar」:about:blank(游標在這))", &none));
assert!(replaces_field("輸入框「Name」", &none));
assert!(!replaces_field("#3(文字「Untitled」)", &none), "a text area keeps its text");
assert!(!replaces_field("#14(輸入框「Search」)", &serde_json::json!({"append": true})));
// What the box runs: the field clicked, its text selected, the address pasted in one go.
let els = vec![crate::marks::Element { kind: "輸入框".into(), name: "Address and search bar".into(), value: "about:blank".into(), x: 100.0, y: 50.0, w: 400.0, h: 30.0, active: true, flags: vec![], window: "Chromium".into() }];
let (_, _, spots) = crate::marks::number_keeping(&els, false, &mut Default::default());
*BOX_SPOTS.lock().unwrap() = Some(spots);
let (script, _) = box_script("computer_use_typing", &serde_json::json!({"ref": "1", "value": "chrome://settings/search", "submit": true}), (1.0, 1.0)).unwrap();
assert!(script.contains("click 1") && script.contains("ctrl+a") && script.contains("xclip") && script.contains("ctrl+v") && !script.contains("xdotool type"), "{script}");
}
#[test] #[test]
fn a_command_asking_for_sudo_is_recognized() { fn a_command_asking_for_sudo_is_recognized() {
for c in ["sudo apt-get install -y tree", "cd x && sudo make install", "ls; sudo -n true", "(sudo id)", "echo ok | sudo tee /etc/x"] { for c in ["sudo apt-get install -y tree", "cd x && sudo make install", "ls; sudo -n true", "(sudo id)", "echo ok | sudo tee /etc/x"] {

View File

@ -863,7 +863,7 @@ pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result<String, String> {
if let Some((x, y, what)) = &at { if let Some((x, y, what)) = &at {
mac::click(*x, *y, "left", 1, 0); mac::click(*x, *y, "left", 1, 0);
std::thread::sleep(std::time::Duration::from_millis(150)); std::thread::sleep(std::time::Duration::from_millis(150));
if crate::cloudbox::replaces_field(what.split_once('(').map(|(_, w)| w).unwrap_or(what), v) { if crate::cloudbox::replaces_field(what, v) {
press_keys("cmd+a", KEY_HOLD_SECS)?; press_keys("cmd+a", KEY_HOLD_SECS)?;
std::thread::sleep(std::time::Duration::from_millis(50)); std::thread::sleep(std::time::Duration::from_millis(50));
} }

View File

@ -59,7 +59,27 @@ impl Spots {
hits.sort(); hits.sort();
match hits.as_slice() { match hits.as_slice() {
[n] => Ok(*n), [n] => Ok(*n),
[] => Err(format!("最新的畫面裡沒有叫「{want}」的東西:先看一下畫面(computer_use_screenshot),用清單上的名字或編號。")), [] => {
// Names that come close (one inside the other, ignoring case), so the step can be
// put right at once instead of guessed again.
let low = want.to_lowercase();
let mut near: Vec<(usize, String)> = self
.by_number
.iter()
.filter_map(|(n, (_, what))| name_of(what).filter(|nm| !nm.is_empty()).map(|nm| (*n, nm)))
.filter(|(_, nm)| {
let l = nm.to_lowercase();
!low.is_empty() && (l.contains(&low) || low.contains(&l))
})
.collect();
near.sort();
near.truncate(6);
if near.is_empty() {
Err(format!("最新的畫面裡沒有叫「{want}」的東西:照最新清單上的名字或編號點,不要猜名字。"))
} else {
Err(format!("最新的畫面裡沒有叫「{want}」的東西。名字相近的有:{}。", near.iter().map(|(n, nm)| format!("[{n}]「{nm}」")).collect::<Vec<_>>().join("、")))
}
}
many => Err(format!("最新的畫面裡有 {} 個叫「{want}」的東西(編號 {}):請用編號指定是哪一個。", many.len(), many.iter().map(|n| n.to_string()).collect::<Vec<_>>().join("、"))), many => Err(format!("最新的畫面裡有 {} 個叫「{want}」的東西(編號 {}):請用編號指定是哪一個。", many.len(), many.iter().map(|n| n.to_string()).collect::<Vec<_>>().join("、"))),
} }
} }
@ -342,6 +362,9 @@ mod tests {
assert_eq!(spots.named("4").unwrap(), 1); assert_eq!(spots.named("4").unwrap(), 1);
assert_eq!(spots.named("「C」").unwrap(), 3, "C is not AC"); assert_eq!(spots.named("「C」").unwrap(), 3, "C is not AC");
assert!(spots.named("Save").unwrap_err().contains("沒有叫「Save」")); assert!(spots.named("Save").unwrap_err().contains("沒有叫「Save」"));
// A near miss names what is there.
let near = spots.named("自動").unwrap_err();
assert!(near.contains("名字相近") && near.contains("[4]「自動更新」"), "{near}");
let mut v = serde_json::json!({"actions": [{"action": "click", "name": "4"}, {"action": "click", "ref": "2"}, {"action": "typing", "value": "x"}]}); let mut v = serde_json::json!({"actions": [{"action": "click", "name": "4"}, {"action": "click", "ref": "2"}, {"action": "typing", "value": "x"}]});
names_to_numbers(&mut v, Some(&spots)).unwrap(); names_to_numbers(&mut v, Some(&spots)).unwrap();
assert_eq!(v["actions"][0]["ref"], "1"); assert_eq!(v["actions"][0]["ref"], "1");