The box really replaces a one-line field's text; a turn that keeps going wrong stops handing steps to the quick model; a missed name says what's close
- The one-line-field check got the step's name as 「#14(輸入框…)」 on the box and never matched, so the select-and-paste never ran there and addresses still came out mangled. It reads both forms now, with a test on the script the box runs. - Trouble twice in a turn and the rest of the turn is the bot's own model's (a quick model guessed its way through Mousepad's save dialog for 79 steps). - A name that isn't on the screen comes back with the names close to it ([12]「New Window」…), so the step is put right at once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
This commit is contained in:
parent
93ead114cd
commit
1be088474e
19
src/agent.rs
19
src/agent.rs
|
|
@ -764,6 +764,10 @@ struct Replay {
|
|||
|
||||
/// Quick steps in a row before the bot's own model looks at the work again (estimated).
|
||||
pub const QUICK_RUN_MAX: usize = 4;
|
||||
/// Trouble this many times in a turn and the rest of it is the bot's own model's: the quick model
|
||||
/// guessing its way through an unfamiliar dialog (names that aren't there, unnamed items) only
|
||||
/// made it longer each time it got the step back.
|
||||
pub const QUICK_TROUBLE_MAX: usize = 2;
|
||||
|
||||
/// Steps that only look (nothing on any computer changes): a turn of only these needs no check
|
||||
/// at the end.
|
||||
|
|
@ -785,13 +789,15 @@ pub struct StepPlan {
|
|||
quick_run: usize,
|
||||
/// The way the last acting step went (commands, the screen, a web page).
|
||||
last_route: Option<&'static str>,
|
||||
/// Steps that went wrong this turn.
|
||||
troubles: usize,
|
||||
/// Something on a computer may have changed this turn (not only looking).
|
||||
pub changed: bool,
|
||||
}
|
||||
|
||||
impl StepPlan {
|
||||
pub fn new(quick: Option<String>) -> Self {
|
||||
StepPlan { quick, main_next: true, quick_run: 0, last_route: None, changed: false }
|
||||
StepPlan { quick, main_next: true, quick_run: 0, last_route: None, troubles: 0, changed: false }
|
||||
}
|
||||
|
||||
/// The quick model for this step, or None for the bot's own.
|
||||
|
|
@ -810,6 +816,10 @@ impl StepPlan {
|
|||
pub fn after(&mut self, trouble: bool) {
|
||||
if trouble {
|
||||
self.main_next = true;
|
||||
self.troubles += 1;
|
||||
if self.troubles >= QUICK_TROUBLE_MAX && self.quick.take().is_some() {
|
||||
eprintln!("AnyBot:這一回合出了 {} 次問題,剩下的步驟改用 Bot 自己的模型", self.troubles);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2039,6 +2049,13 @@ mod tests {
|
|||
assert_eq!(p.pick().as_deref(), Some("q"));
|
||||
p.after(true);
|
||||
assert_eq!(p.pick(), None, "after trouble: the bot's own model");
|
||||
// Trouble twice: the rest of the turn is the bot's own model's.
|
||||
let mut twice = StepPlan::new(Some("q".into()));
|
||||
twice.pick();
|
||||
twice.after(true);
|
||||
twice.pick();
|
||||
twice.after(true);
|
||||
assert!(twice.quick_is_off() && twice.pick().is_none() && twice.pick().is_none());
|
||||
for _ in 0..QUICK_RUN_MAX {
|
||||
assert_eq!(p.pick().as_deref(), Some("q"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1826,7 +1826,9 @@ pub fn box_script(name: &str, v: &Value, scale: (f64, f64)) -> Result<(String, S
|
|||
/// Typing into this control replaces what it holds: a one-line field (輸入框), unless the step
|
||||
/// asks to add to it. A document's text area is left alone (selecting all would replace a file).
|
||||
pub fn replaces_field(what: &str, v: &Value) -> bool {
|
||||
what.starts_with("輸入框") && !v["append"].as_bool().unwrap_or(false)
|
||||
// As a step names it (「#14(輸入框「Address and search bar」:…)」) or bare.
|
||||
let desc = what.strip_prefix('#').and_then(|r| r.split_once('(')).map(|(_, d)| d).unwrap_or(what);
|
||||
desc.starts_with("輸入框") && !v["append"].as_bool().unwrap_or(false)
|
||||
}
|
||||
|
||||
fn type_script(text: &str) -> String {
|
||||
|
|
@ -2328,6 +2330,21 @@ pub fn shutdown() {
|
|||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn typing_replaces_a_one_line_field_however_it_is_named() {
|
||||
let none = serde_json::json!({});
|
||||
assert!(replaces_field("#14(輸入框「Address and search bar」:about:blank(游標在這))", &none));
|
||||
assert!(replaces_field("輸入框「Name」", &none));
|
||||
assert!(!replaces_field("#3(文字「Untitled」)", &none), "a text area keeps its text");
|
||||
assert!(!replaces_field("#14(輸入框「Search」)", &serde_json::json!({"append": true})));
|
||||
// What the box runs: the field clicked, its text selected, the address pasted in one go.
|
||||
let els = vec![crate::marks::Element { kind: "輸入框".into(), name: "Address and search bar".into(), value: "about:blank".into(), x: 100.0, y: 50.0, w: 400.0, h: 30.0, active: true, flags: vec![], window: "Chromium".into() }];
|
||||
let (_, _, spots) = crate::marks::number_keeping(&els, false, &mut Default::default());
|
||||
*BOX_SPOTS.lock().unwrap() = Some(spots);
|
||||
let (script, _) = box_script("computer_use_typing", &serde_json::json!({"ref": "1", "value": "chrome://settings/search", "submit": true}), (1.0, 1.0)).unwrap();
|
||||
assert!(script.contains("click 1") && script.contains("ctrl+a") && script.contains("xclip") && script.contains("ctrl+v") && !script.contains("xdotool type"), "{script}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_command_asking_for_sudo_is_recognized() {
|
||||
for c in ["sudo apt-get install -y tree", "cd x && sudo make install", "ls; sudo -n true", "(sudo id)", "echo ok | sudo tee /etc/x"] {
|
||||
|
|
|
|||
|
|
@ -863,7 +863,7 @@ pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result<String, String> {
|
|||
if let Some((x, y, what)) = &at {
|
||||
mac::click(*x, *y, "left", 1, 0);
|
||||
std::thread::sleep(std::time::Duration::from_millis(150));
|
||||
if crate::cloudbox::replaces_field(what.split_once('(').map(|(_, w)| w).unwrap_or(what), v) {
|
||||
if crate::cloudbox::replaces_field(what, v) {
|
||||
press_keys("cmd+a", KEY_HOLD_SECS)?;
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
}
|
||||
|
|
|
|||
25
src/marks.rs
25
src/marks.rs
|
|
@ -59,7 +59,27 @@ impl Spots {
|
|||
hits.sort();
|
||||
match hits.as_slice() {
|
||||
[n] => Ok(*n),
|
||||
[] => Err(format!("最新的畫面裡沒有叫「{want}」的東西:先看一下畫面(computer_use_screenshot),用清單上的名字或編號。")),
|
||||
[] => {
|
||||
// Names that come close (one inside the other, ignoring case), so the step can be
|
||||
// put right at once instead of guessed again.
|
||||
let low = want.to_lowercase();
|
||||
let mut near: Vec<(usize, String)> = self
|
||||
.by_number
|
||||
.iter()
|
||||
.filter_map(|(n, (_, what))| name_of(what).filter(|nm| !nm.is_empty()).map(|nm| (*n, nm)))
|
||||
.filter(|(_, nm)| {
|
||||
let l = nm.to_lowercase();
|
||||
!low.is_empty() && (l.contains(&low) || low.contains(&l))
|
||||
})
|
||||
.collect();
|
||||
near.sort();
|
||||
near.truncate(6);
|
||||
if near.is_empty() {
|
||||
Err(format!("最新的畫面裡沒有叫「{want}」的東西:照最新清單上的名字或編號點,不要猜名字。"))
|
||||
} else {
|
||||
Err(format!("最新的畫面裡沒有叫「{want}」的東西。名字相近的有:{}。", near.iter().map(|(n, nm)| format!("[{n}]「{nm}」")).collect::<Vec<_>>().join("、")))
|
||||
}
|
||||
}
|
||||
many => Err(format!("最新的畫面裡有 {} 個叫「{want}」的東西(編號 {}):請用編號指定是哪一個。", many.len(), many.iter().map(|n| n.to_string()).collect::<Vec<_>>().join("、"))),
|
||||
}
|
||||
}
|
||||
|
|
@ -342,6 +362,9 @@ mod tests {
|
|||
assert_eq!(spots.named("4").unwrap(), 1);
|
||||
assert_eq!(spots.named("「C」").unwrap(), 3, "C is not AC");
|
||||
assert!(spots.named("Save").unwrap_err().contains("沒有叫「Save」"));
|
||||
// A near miss names what is there.
|
||||
let near = spots.named("自動").unwrap_err();
|
||||
assert!(near.contains("名字相近") && near.contains("[4]「自動更新」"), "{near}");
|
||||
let mut v = serde_json::json!({"actions": [{"action": "click", "name": "4"}, {"action": "click", "ref": "2"}, {"action": "typing", "value": "x"}]});
|
||||
names_to_numbers(&mut v, Some(&spots)).unwrap();
|
||||
assert_eq!(v["actions"][0]["ref"], "1");
|
||||
|
|
|
|||
Loading…
Reference in New Issue