daniel.w
/
AnyBot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Drilled the box's screen path end to end on the real box, and fixed what the drill caught 

box_real_usage_drill (ignored) drives the whole path on a page that reports what really landed:
one batch fills and sends a form (six steps, Chinese typing, 1.6 s), a bare-x/y click on a canvas
lands 1 px off its target, a drag ends where it was pointed, scrolling reveals a field that is
then clicked and typed by number, and the same spot clicked twice says so. Every step is timed:
look 477 ms, steps 650-880 ms.

Caught: the a11y tree is exposed after the screen settles (measured 0.2-0.35 s behind a scroll),
so a list read in parallel with the picture can miss a control the picture already shows.
box_look now starts the capture in the background and gives the a11y read 0.18 s before running;
the parallel capture pays for it, and a scrolled-in field is in the list every run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
王性驊 2026-11-06 09:58:07 +08:00
parent cbf43ce2d4
commit a3e71aa22e
1 changed files with 98 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
src/cloudbox.rs
View File

@ -1651,7 +1651,7 @@ fn box_look(script: Option<(&str, Option<&str>)>, after: Option<String>) -> Resu
// The picture is captured by a background `import` while the controls are read (the a11y
// read is the slow part on web pages); a JPEG costs less to make and to ship than a PNG.
let look = "( import -window root -quality 90 jpeg:- 2>/dev/null > /tmp/anybot-look.jpg & ip=$!; \
anybot-a11y --budget 3 > /tmp/anybot-look.json 2>/dev/null || echo '{\"elements\":[]}' > /tmp/anybot-look.json; \
sleep 0.18; anybot-a11y --budget 3 > /tmp/anybot-look.json 2>/dev/null || echo '{\"elements\":[]}' > /tmp/anybot-look.json; \
wc -c < /tmp/anybot-look.json; cat /tmp/anybot-look.json; wait $ip; cat /tmp/anybot-look.jpg )";
let full = match script {
Some((s, _)) => format!("{{ {s}; }} || exit $?; {SETTLE_SH}; {look}"),
@ -2156,6 +2156,103 @@ mod tests {
let _ = box_exec("xdotool key --clearmodifiers ctrl+d", None, false);
}
/// Uses the real Docker: `cargo test --release -- --ignored box_real_usage_drill --nocapture`.
/// The whole screen path, drilled on a page that reports what really happened: a form filled
/// and sent by one batch, a bare-x/y click and a drag landing on a canvas by the page's own
/// record, scrolling revealing a field that is then clicked and typed by number. Timed.
#[test]
#[ignore]
fn box_real_usage_drill() {
ensure_running().expect("box starts");
let cancel = AtomicBool::new(false);
let page = r##"<!doctype html><meta charset=utf-8><title>drill</title><body style="margin:0;font:16px sans-serif"><div style="padding:20px"><input id=q1 aria-label="Name" style="width:200px;font-size:16px"> <input id=q2 aria-label="Email" style="width:220px;font-size:16px"> <label>Agree <input type=checkbox id=ck></label> <button id=go onclick="document.getElementById('res').textContent='SUB:'+q1.value+':'+q2.value+':'+ck.checked">Send</button><div id=res>idle</div><canvas id=cv width=400 height=100 style="border:1px solid #777;display:block;margin-top:12px"></canvas><div id=dragx></div><div style="height:1200px">keep scrolling down</div><input id=q3 aria-label="Bottom field" style="width:200px;font-size:16px"></div><script>cv.addEventListener('click',e=>{document.title='cv:'+Math.round(e.offsetX)+','+Math.round(e.offsetY)});let down=false;cv.addEventListener('mousedown',()=>{down=true;dragx.textContent=''});window.addEventListener('mousemove',e=>{if(down)dragx.textContent=String(Math.round(e.clientX))});window.addEventListener('mouseup',()=>{down=false});</script>"##;
dk_with_input(&["exec", "-i", "-u", "bot", container(), "sh", "-c", "mkdir -p /tmp/anybot-page && cat > /tmp/anybot-page/index.html && (cd /tmp/anybot-page && setsid python3 -m http.server 8765 --bind 127.0.0.1 >/dev/null 2>&1 &) ; sleep 0.5"], Some(page.as_bytes()), 10_000).expect("page served");
// A clean slate: every earlier drill left its page (and its state) in a tab. Closing all
// pages closes the window too; browser_open starts the browser again with our page.
let close_pages = "python3 -c \"import json,urllib.request\nfor t in json.load(urllib.request.urlopen('http://127.0.0.1:9222/json/list')):\n if t.get('type')=='page':\n try: urllib.request.urlopen('http://127.0.0.1:9222/json/close/'+t['id'])\n except Exception: pass\"";
let _ = dk(&["exec", "-u", "bot", container(), "sh", "-c", close_pages]);
std::thread::sleep(Duration::from_millis(600));
let opened = crate::botbrowser::in_box(|| crate::web::run_full("browser_open", r#"{"url":"http://127.0.0.1:8765/"}"#, &cancel));
assert!(opened.1, "{}", opened.0);
std::thread::sleep(Duration::from_millis(1500));
let js = |e: &str| crate::botbrowser::in_box(|| crate::botbrowser::eval_current(None, e)).map(|(_, v)| v.as_str().unwrap_or("").to_string()).unwrap_or_default();
assert_eq!(js("document.title"), "drill", "the drill page is up");
let number = |txt: &str, what: &str| -> String {
txt.lines().find(|l| l.starts_with('[') && l.contains(what)).and_then(|l| l.trim().strip_prefix('[')).and_then(|l| l.split(']').next()).unwrap_or_else(|| panic!("no number for {what} in:\n{txt}")).to_string()
};
let mut times: Vec<(String, u64)> = vec![];
macro_rules! step {
($name:expr, $tool:expr, $args:expr) => {{
let t0 = Instant::now();
let r = computer_run($tool, &$args).unwrap();
times.push((format!("{} ({})", $name, $tool), t0.elapsed().as_millis() as u64));
r
}};
}
// 1. One look: the numbered picture.
let (txt, img) = step!("look", "computer_use_screenshot", serde_json::json!({}));
if let Some(b) = &img {
let _ = std::fs::write(std::env::temp_dir().join("anybot-drill-1.jpg"), base64::engine::general_purpose::STANDARD.decode(b).unwrap_or_default());
}
// 2. The form, all six steps in ONE batch call: clicks by number, Chinese and ASCII
// typing, and the Send button — the page says what arrived.
let batch = serde_json::json!({"actions": [
{"action": "click", "ref": number(&txt, "Name")},
{"action": "typing", "value": "王小明"},
{"action": "click", "ref": number(&txt, "Email")},
{"action": "typing", "value": "a@b.c"},
{"action": "click", "ref": number(&txt, "Agree")},
{"action": "click", "ref": number(&txt, "Send")}
]});
let (btxt, bimg) = step!("fill+send form", "computer_use_batch", batch);
assert!(btxt.contains("已依序做完 6 步"), "{btxt}");
assert_eq!(js("document.getElementById('res').textContent"), "SUB:王小明:a@b.c:true", "the batch really filled and sent the form");
if let Some(b) = &bimg {
let _ = std::fs::write(std::env::temp_dir().join("anybot-drill-2.jpg"), base64::engine::general_purpose::STANDARD.decode(b).unwrap_or_default());
}
// 3. A click at bare x,y (no number on a canvas): the canvas records where it landed.
// The page knows CSS pixels; the pointer lives on the desktop — add the window's own
// place and toolbar height (what the page's viewport sits under).
let geo: Value = serde_json::from_str(&js("JSON.stringify({sx:screenX,sy:screenY,ow:outerWidth,oh:outerHeight,iw:innerWidth,ih:innerHeight})")).unwrap();
println!("window: {geo}");
let off_x = geo["sx"].as_f64().unwrap() + (geo["ow"].as_f64().unwrap() - geo["iw"].as_f64().unwrap()) / 2.0;
let off_y = geo["sy"].as_f64().unwrap() + (geo["oh"].as_f64().unwrap() - geo["ih"].as_f64().unwrap()) - (geo["ow"].as_f64().unwrap() - geo["iw"].as_f64().unwrap()) / 2.0;
let rect: Value = serde_json::from_str(&js("JSON.stringify(document.getElementById('cv').getBoundingClientRect())")).unwrap();
let (sx, sy) = box_scale().unwrap();
let (cx, cy) = ((off_x + rect["left"].as_f64().unwrap() + 200.0) / sx, (off_y + rect["top"].as_f64().unwrap() + 50.0) / sy);
println!("rect: {rect} off=({off_x},{off_y}) scale=({sx},{sy}) picture=({cx},{cy})");
step!("canvas dot at (200,50)", "computer_use_click", serde_json::json!({"x": cx.round() as i64, "y": cy.round() as i64}));
let title = js("document.title");
println!("canvas saw: {title}");
let mut xy = title.trim_start_matches("cv:").split(',').map(|s| s.parse::<f64>().unwrap());
let (got_x, got_y) = (xy.next().unwrap(), xy.next().unwrap());
assert!((got_x - 200.0).abs() <= 15.0 && (got_y - 50.0).abs() <= 15.0, "canvas click off: {got_x},{got_y}");
// 4. A drag across the canvas: the page records the last pointer position it felt.
let y = (off_y + rect["top"].as_f64().unwrap() + 50.0) / sy;
let (x1, x2) = ((off_x + rect["left"].as_f64().unwrap() + 20.0) / sx, (off_x + rect["left"].as_f64().unwrap() + 320.0) / sx);
step!("drag 20→320", "computer_use_drag", serde_json::json!({"path": [{"x": x1.round() as i64, "y": y.round() as i64}, {"x": x2.round() as i64, "y": y.round() as i64}]}));
let dragx = js("document.getElementById('dragx').textContent").parse::<f64>().unwrap_or(-1.0);
println!("drag ended at clientX {dragx} (wanted ~{})", rect["left"].as_f64().unwrap() + 320.0);
assert!((dragx - (rect["left"].as_f64().unwrap() + 320.0)).abs() <= 20.0, "drag ended at {dragx}");
// 5. Scrolling reveals the field at the bottom; it gets a number, and clicking that
// number focuses it for typing.
let (stxt, _) = step!("scroll down", "computer_use_scroll", serde_json::json!({"direction": "down", "amount": 10}));
let scroll_y: f64 = js("String(window.scrollY)").parse().unwrap_or(0.0);
assert!(scroll_y > 100.0, "the page scrolled: {scroll_y}");
let (ctxt, _) = step!("bottom click", "computer_use_click", serde_json::json!({"ref": number(&stxt, "Bottom field")}));
step!("bottom type", "computer_use_typing", serde_json::json!({"ref": number(&ctxt, "Bottom field"), "value": "end"}));
assert_eq!(js("document.getElementById('q3').value"), "end", "the field at the bottom got the typing");
// 6. Same spot twice: the second says so, in words.
let (again, _) = computer_run("computer_use_click", &serde_json::json!({"x": 700, "y": 550})).unwrap();
let (again2, _) = computer_run("computer_use_click", &serde_json::json!({"x": 700, "y": 550})).unwrap();
println!("twice at (700,550): first「{}」second「{}」", again.lines().next().unwrap_or(""), again2.lines().find(|l| l.contains("完全一樣")).unwrap_or("(said nothing)"));
println!("\n--- step times ---");
for (n, ms) in &times {
println!("{ms:>6} ms {n}");
}
println!("total of {} timed steps: {} ms", times.len(), times.iter().map(|(_, m)| m).sum::<u64>());
}
/// Uses the real Docker: `cargo test --release -- --ignored box_end_to_end --nocapture`.
#[test]
#[ignore]