diff --git a/src/cloudbox.rs b/src/cloudbox.rs index ed919eb..5abf7f4 100644 --- a/src/cloudbox.rs +++ b/src/cloudbox.rs @@ -1651,7 +1651,7 @@ fn box_look(script: Option<(&str, Option<&str>)>, after: Option) -> Resu // The picture is captured by a background `import` while the controls are read (the a11y // read is the slow part on web pages); a JPEG costs less to make and to ship than a PNG. let look = "( import -window root -quality 90 jpeg:- 2>/dev/null > /tmp/anybot-look.jpg & ip=$!; \ - anybot-a11y --budget 3 > /tmp/anybot-look.json 2>/dev/null || echo '{\"elements\":[]}' > /tmp/anybot-look.json; \ + sleep 0.18; anybot-a11y --budget 3 > /tmp/anybot-look.json 2>/dev/null || echo '{\"elements\":[]}' > /tmp/anybot-look.json; \ wc -c < /tmp/anybot-look.json; cat /tmp/anybot-look.json; wait $ip; cat /tmp/anybot-look.jpg )"; let full = match script { Some((s, _)) => format!("{{ {s}; }} || exit $?; {SETTLE_SH}; {look}"), @@ -2156,6 +2156,103 @@ mod tests { let _ = box_exec("xdotool key --clearmodifiers ctrl+d", None, false); } + /// Uses the real Docker: `cargo test --release -- --ignored box_real_usage_drill --nocapture`. + /// The whole screen path, drilled on a page that reports what really happened: a form filled + /// and sent by one batch, a bare-x/y click and a drag landing on a canvas by the page's own + /// record, scrolling revealing a field that is then clicked and typed by number. Timed. + #[test] + #[ignore] + fn box_real_usage_drill() { + ensure_running().expect("box starts"); + let cancel = AtomicBool::new(false); + let page = r##"drill
idle
keep scrolling down
"##; + dk_with_input(&["exec", "-i", "-u", "bot", container(), "sh", "-c", "mkdir -p /tmp/anybot-page && cat > /tmp/anybot-page/index.html && (cd /tmp/anybot-page && setsid python3 -m http.server 8765 --bind 127.0.0.1 >/dev/null 2>&1 &) ; sleep 0.5"], Some(page.as_bytes()), 10_000).expect("page served"); + // A clean slate: every earlier drill left its page (and its state) in a tab. Closing all + // pages closes the window too; browser_open starts the browser again with our page. + let close_pages = "python3 -c \"import json,urllib.request\nfor t in json.load(urllib.request.urlopen('http://127.0.0.1:9222/json/list')):\n if t.get('type')=='page':\n try: urllib.request.urlopen('http://127.0.0.1:9222/json/close/'+t['id'])\n except Exception: pass\""; + let _ = dk(&["exec", "-u", "bot", container(), "sh", "-c", close_pages]); + std::thread::sleep(Duration::from_millis(600)); + let opened = crate::botbrowser::in_box(|| crate::web::run_full("browser_open", r#"{"url":"http://127.0.0.1:8765/"}"#, &cancel)); + assert!(opened.1, "{}", opened.0); + std::thread::sleep(Duration::from_millis(1500)); + let js = |e: &str| crate::botbrowser::in_box(|| crate::botbrowser::eval_current(None, e)).map(|(_, v)| v.as_str().unwrap_or("").to_string()).unwrap_or_default(); + assert_eq!(js("document.title"), "drill", "the drill page is up"); + let number = |txt: &str, what: &str| -> String { + txt.lines().find(|l| l.starts_with('[') && l.contains(what)).and_then(|l| l.trim().strip_prefix('[')).and_then(|l| l.split(']').next()).unwrap_or_else(|| panic!("no number for {what} in:\n{txt}")).to_string() + }; + let mut times: Vec<(String, u64)> = vec![]; + macro_rules! step { + ($name:expr, $tool:expr, $args:expr) => {{ + let t0 = Instant::now(); + let r = computer_run($tool, &$args).unwrap(); + times.push((format!("{} ({})", $name, $tool), t0.elapsed().as_millis() as u64)); + r + }}; + } + // 1. One look: the numbered picture. + let (txt, img) = step!("look", "computer_use_screenshot", serde_json::json!({})); + if let Some(b) = &img { + let _ = std::fs::write(std::env::temp_dir().join("anybot-drill-1.jpg"), base64::engine::general_purpose::STANDARD.decode(b).unwrap_or_default()); + } + // 2. The form, all six steps in ONE batch call: clicks by number, Chinese and ASCII + // typing, and the Send button — the page says what arrived. + let batch = serde_json::json!({"actions": [ + {"action": "click", "ref": number(&txt, "Name")}, + {"action": "typing", "value": "王小明"}, + {"action": "click", "ref": number(&txt, "Email")}, + {"action": "typing", "value": "a@b.c"}, + {"action": "click", "ref": number(&txt, "Agree")}, + {"action": "click", "ref": number(&txt, "Send")} + ]}); + let (btxt, bimg) = step!("fill+send form", "computer_use_batch", batch); + assert!(btxt.contains("已依序做完 6 步"), "{btxt}"); + assert_eq!(js("document.getElementById('res').textContent"), "SUB:王小明:a@b.c:true", "the batch really filled and sent the form"); + if let Some(b) = &bimg { + let _ = std::fs::write(std::env::temp_dir().join("anybot-drill-2.jpg"), base64::engine::general_purpose::STANDARD.decode(b).unwrap_or_default()); + } + // 3. A click at bare x,y (no number on a canvas): the canvas records where it landed. + // The page knows CSS pixels; the pointer lives on the desktop — add the window's own + // place and toolbar height (what the page's viewport sits under). + let geo: Value = serde_json::from_str(&js("JSON.stringify({sx:screenX,sy:screenY,ow:outerWidth,oh:outerHeight,iw:innerWidth,ih:innerHeight})")).unwrap(); + println!("window: {geo}"); + let off_x = geo["sx"].as_f64().unwrap() + (geo["ow"].as_f64().unwrap() - geo["iw"].as_f64().unwrap()) / 2.0; + let off_y = geo["sy"].as_f64().unwrap() + (geo["oh"].as_f64().unwrap() - geo["ih"].as_f64().unwrap()) - (geo["ow"].as_f64().unwrap() - geo["iw"].as_f64().unwrap()) / 2.0; + let rect: Value = serde_json::from_str(&js("JSON.stringify(document.getElementById('cv').getBoundingClientRect())")).unwrap(); + let (sx, sy) = box_scale().unwrap(); + let (cx, cy) = ((off_x + rect["left"].as_f64().unwrap() + 200.0) / sx, (off_y + rect["top"].as_f64().unwrap() + 50.0) / sy); + println!("rect: {rect} off=({off_x},{off_y}) scale=({sx},{sy}) picture=({cx},{cy})"); + step!("canvas dot at (200,50)", "computer_use_click", serde_json::json!({"x": cx.round() as i64, "y": cy.round() as i64})); + let title = js("document.title"); + println!("canvas saw: {title}"); + let mut xy = title.trim_start_matches("cv:").split(',').map(|s| s.parse::().unwrap()); + let (got_x, got_y) = (xy.next().unwrap(), xy.next().unwrap()); + assert!((got_x - 200.0).abs() <= 15.0 && (got_y - 50.0).abs() <= 15.0, "canvas click off: {got_x},{got_y}"); + // 4. A drag across the canvas: the page records the last pointer position it felt. + let y = (off_y + rect["top"].as_f64().unwrap() + 50.0) / sy; + let (x1, x2) = ((off_x + rect["left"].as_f64().unwrap() + 20.0) / sx, (off_x + rect["left"].as_f64().unwrap() + 320.0) / sx); + step!("drag 20→320", "computer_use_drag", serde_json::json!({"path": [{"x": x1.round() as i64, "y": y.round() as i64}, {"x": x2.round() as i64, "y": y.round() as i64}]})); + let dragx = js("document.getElementById('dragx').textContent").parse::().unwrap_or(-1.0); + println!("drag ended at clientX {dragx} (wanted ~{})", rect["left"].as_f64().unwrap() + 320.0); + assert!((dragx - (rect["left"].as_f64().unwrap() + 320.0)).abs() <= 20.0, "drag ended at {dragx}"); + // 5. Scrolling reveals the field at the bottom; it gets a number, and clicking that + // number focuses it for typing. + let (stxt, _) = step!("scroll down", "computer_use_scroll", serde_json::json!({"direction": "down", "amount": 10})); + let scroll_y: f64 = js("String(window.scrollY)").parse().unwrap_or(0.0); + assert!(scroll_y > 100.0, "the page scrolled: {scroll_y}"); + let (ctxt, _) = step!("bottom click", "computer_use_click", serde_json::json!({"ref": number(&stxt, "Bottom field")})); + step!("bottom type", "computer_use_typing", serde_json::json!({"ref": number(&ctxt, "Bottom field"), "value": "end"})); + assert_eq!(js("document.getElementById('q3').value"), "end", "the field at the bottom got the typing"); + // 6. Same spot twice: the second says so, in words. + let (again, _) = computer_run("computer_use_click", &serde_json::json!({"x": 700, "y": 550})).unwrap(); + let (again2, _) = computer_run("computer_use_click", &serde_json::json!({"x": 700, "y": 550})).unwrap(); + println!("twice at (700,550): first「{}」second「{}」", again.lines().next().unwrap_or(""), again2.lines().find(|l| l.contains("完全一樣")).unwrap_or("(said nothing)")); + println!("\n--- step times ---"); + for (n, ms) in × { + println!("{ms:>6} ms {n}"); + } + println!("total of {} timed steps: {} ms", times.len(), times.iter().map(|(_, m)| m).sum::()); + } + /// Uses the real Docker: `cargo test --release -- --ignored box_end_to_end --nocapture`. #[test] #[ignore]