// What a bot sees of a screen: the picture with every control it can use boxed and numbered, // and the same numbers in a list ("[12] 連結「Profile」"). It acts by number; the click lands on // that control's centre. Guessing coordinates from a picture is where clicks went wrong (a // model took an empty strip for a menu item and clicked it ten times); a number can't miss. // // The controls come from the system's accessibility tree: macOS AX and Linux AT-SPI through // agent-desktop on a person's computer (desktop.rs), AT-SPI in the bots' own computer // (box/a11y). Web pages are in those trees too (Chromium with renderer accessibility on). use std::collections::HashMap; /// One control on the screen, in screen units (the mouse's: points on a Mac, pixels elsewhere). #[derive(Clone, Debug, PartialEq)] pub struct Element { /// 按鈕, 連結, 輸入框… (already in words). pub kind: String, pub name: String, pub value: String, pub x: f64, pub y: f64, pub w: f64, pub h: f64, /// Something to click or type into (gets a number), not just text. pub active: bool, pub flags: Vec<&'static str>, pub window: String, } impl Element { pub fn centre(&self) -> (f64, f64) { (self.x + self.w / 2.0, self.y + self.h / 2.0) } } /// Most controls numbered on one picture (more is clutter the model can't use). pub const MAX_MARKS: usize = 150; /// The numbered controls of the latest look at a screen: number → (centre, words). #[derive(Clone, Debug, Default)] pub struct Spots { pub by_number: HashMap, } impl Spots { /// "12", "e12", "[12]" → the control's centre and what it is. pub fn get(&self, r: &str) -> Result<((f64, f64), String), String> { let n = number_of(r).ok_or_else(|| format!("「{}」不是編號:請用截圖上的編號(例如 12)", r.trim()))?; self.by_number.get(&n).cloned().ok_or_else(|| format!("最新的畫面裡沒有編號 {n}:先用 computer(action=screenshot)看一次畫面再點。")) } } impl Spots { /// The numbered control called `name` in the latest look (its name as listed: 「4」, 「Close」, /// 「自動更新」), so a step can say which button the way a person would. Ok(number) or why not. pub fn named(&self, name: &str) -> Result { let want = name.trim().trim_matches(['「', '」']); let name_of = |what: &str| what.split_once('「').and_then(|(_, r)| r.split_once('」')).map(|(n, _)| n.to_string()); let mut hits: Vec = self.by_number.iter().filter(|(_, (_, what))| name_of(what).as_deref() == Some(want)).map(|(n, _)| *n).collect(); hits.sort(); match hits.as_slice() { [n] => Ok(*n), [] => { // Names that come close (one inside the other, ignoring case), so the step can be // put right at once instead of guessed again. let low = want.to_lowercase(); let mut near: Vec<(usize, String)> = self .by_number .iter() .filter_map(|(n, (_, what))| name_of(what).filter(|nm| !nm.is_empty()).map(|nm| (*n, nm))) .filter(|(_, nm)| { let l = nm.to_lowercase(); !low.is_empty() && (l.contains(&low) || low.contains(&l)) }) .collect(); near.sort(); near.truncate(6); if near.is_empty() { Err(format!("最新的畫面裡沒有叫「{want}」的東西:照最新清單上的名字或編號點,不要猜名字。")) } else { Err(format!("最新的畫面裡沒有叫「{want}」的東西。名字相近的有:{}。", near.iter().map(|(n, nm)| format!("[{n}]「{nm}」")).collect::>().join("、"))) } } many => Err(format!("最新的畫面裡有 {} 個叫「{want}」的東西(編號 {}):請用編號指定是哪一個。", many.len(), many.iter().map(|n| n.to_string()).collect::>().join("、"))), } } } /// A step that names its control (`name`, no number, no x/y) gets that control's number from /// the latest look; so does every action of a batch. pub fn names_to_numbers(v: &mut serde_json::Value, spots: Option<&Spots>) -> Result<(), String> { let fix = |p: &mut serde_json::Value| -> Result<(), String> { // ("name" in the screen tools; "text" too, the word the browser tools use.) let named = ["name", "text"].iter().find_map(|k| p[*k].as_str().map(str::trim).filter(|t| !t.is_empty()).map(String::from)); if let Some(t) = named { // A name wins over coordinates sent along with it (x: 0, y: 0 from a model filling // every field): the click went to the screen's corner. if p["ref"].is_null() || p["ref"].as_str().is_some_and(|r| r.trim().is_empty()) { let s = spots.ok_or("先用 computer_use_screenshot 看一下畫面,再照清單上的名字或編號操作。")?; p["ref"] = serde_json::Value::String(s.named(&t)?.to_string()); } } Ok(()) }; if let Some(actions) = v["actions"].as_array_mut() { for a in actions.iter_mut() { fix(a)?; } return Ok(()); } fix(v) } pub fn number_of(r: &str) -> Option { let t = r.trim().trim_matches(['[', ']']).trim_start_matches(['e', 'E', '#']); t.parse().ok() } /// What the last look's numbers meant, so the next look keeps them: a control that is still /// there keeps its number (the 「7」 key stays #141 after 「1」 was pressed). Renumbering every /// look broke steps planned from one picture: a bot sending five clicks at once had the second /// to fifth land on whatever took those numbers after the first click. #[derive(Default, Debug, Clone)] pub struct Numbering { /// (window, kind, name, position bucket) → number. exact: HashMap<(String, String, String, i64, i64), usize>, /// (window, kind, name) → numbers (moved a little, still the same control when only one). loose: HashMap<(String, String, String), Vec>, windows: Vec, top: usize, } /// Position buckets for "the same control" (screen units; a list scrolled a little stays put). const SAME_SPOT: f64 = 24.0; /// Numbers grow while windows stay; past this they start again from 1. const MAX_NUMBER: usize = 600; /// Order (reading order, by window) and number the controls; the list's lines and the spots. /// `all_text`: also list plain text (when there is no picture to read it from). #[cfg(test)] pub fn number(elements: &[Element], all_text: bool) -> (Vec<(usize, Element)>, Vec, Spots) { number_keeping(elements, all_text, &mut Numbering::default()) } /// Same, keeping the numbers of controls that were there in the last look (`memory`, updated). /// Starts again from 1 when none of the last look's windows is still there. pub fn number_keeping(elements: &[Element], all_text: bool, memory: &mut Numbering) -> (Vec<(usize, Element)>, Vec, Spots) { let mut seen = std::collections::HashSet::new(); let mut els: Vec<&Element> = elements.iter().filter(|e| e.w >= 1.0 && e.h >= 1.0 && seen.insert((e.kind.clone(), e.name.clone(), e.x as i64, e.y as i64, e.w as i64, e.h as i64))).collect(); let mut windows: Vec = vec![]; for e in &els { if !windows.contains(&e.window) { windows.push(e.window.clone()); } } els.sort_by(|a, b| { let wa = windows.iter().position(|w| *w == a.window).unwrap_or(0); let wb = windows.iter().position(|w| *w == b.window).unwrap_or(0); (wa, (a.y / 12.0) as i64, a.x as i64).cmp(&(wb, (b.y / 12.0) as i64, b.x as i64)) }); // Which controls get numbers (reading order, at most MAX_MARKS), then which numbers: the // last look's where the same control is still there, new ones above the highest. let marked: Vec<&Element> = els.iter().copied().filter(|e| e.active).take(MAX_MARKS).collect(); let fresh = memory.top > MAX_NUMBER || !windows.iter().any(|w| memory.windows.contains(w)); if fresh { *memory = Numbering::default(); } let exact_of = |e: &Element| (e.window.clone(), e.kind.clone(), e.name.clone(), (e.centre().0 / SAME_SPOT).round() as i64, (e.centre().1 / SAME_SPOT).round() as i64); let loose_of = |e: &Element| (e.window.clone(), e.kind.clone(), e.name.clone()); let mut taken: std::collections::HashSet = std::collections::HashSet::new(); let mut assigned: Vec> = marked.iter().map(|e| memory.exact.get(&exact_of(e)).copied()).collect(); for n in assigned.iter().flatten() { taken.insert(*n); } // Moved a little: the one control of that kind and name in that window, both times. let mut loose_now: HashMap<(String, String, String), usize> = HashMap::new(); for e in &marked { *loose_now.entry(loose_of(e)).or_default() += 1; } for (i, e) in marked.iter().enumerate() { if assigned[i].is_none() && loose_now.get(&loose_of(e)) == Some(&1) { if let Some([n]) = memory.loose.get(&loose_of(e)).map(Vec::as_slice) { if !taken.contains(n) { assigned[i] = Some(*n); taken.insert(*n); } } } } let mut top = memory.top.max(taken.iter().copied().max().unwrap_or(0)); let mut next = Numbering { windows: windows.clone(), ..Default::default() }; let mut number_of_el: HashMap<*const Element, usize> = HashMap::new(); for (i, e) in marked.iter().enumerate() { let n = assigned[i].unwrap_or_else(|| { top += 1; top }); number_of_el.insert(*e as *const Element, n); next.exact.insert(exact_of(e), n); next.loose.entry(loose_of(e)).or_default().push(n); } next.top = top; *memory = next; let mut numbered = vec![]; let mut lines = vec![]; let mut spots = Spots::default(); let mut window = None; for e in els { if !e.active && !all_text { continue; } if windows.len() > 1 && window.as_ref() != Some(&e.window) && !e.window.is_empty() { window = Some(e.window.clone()); lines.push(format!("## {}", e.window)); } let what = describe(e); if let Some(&n) = number_of_el.get(&(e as *const Element)) { spots.by_number.insert(n, (e.centre(), what.clone())); numbered.push((n, e.clone())); lines.push(format!("[{n}] {what}")); } else if !e.active { lines.push(format!(" {what}")); } } (numbered, lines, spots) } fn describe(e: &Element) -> String { let short = |s: &str, n: usize| -> String { let c: Vec = s.chars().collect(); if c.len() > n { format!("{}…", c[..n].iter().collect::()) } else { s.to_string() } }; let mut t = e.kind.clone(); if !e.name.is_empty() { t.push_str(&format!("「{}」", short(&e.name, 60))); } if !e.value.is_empty() && e.value != e.name { t.push_str(&format!(":{}", short(&e.value, 80))); } if !e.flags.is_empty() { t.push_str(&format!("({})", e.flags.join("、"))); } t } /// A fingerprint of what the controls are and where (to say "nothing changed"). /// What came and what went between two looks at the screen (multiset of (kind, name)). /// The quick model reads this to know what its last step did, without re-reading the whole /// numbered list (the list is long; the change is usually two lines). pub fn diff(prev: &[Element], now: &[Element]) -> Vec { fn key(e: &Element) -> (&str, &str) { (e.kind.as_str(), e.name.as_str()) } let mut counts: HashMap<(&str, &str), i32> = HashMap::new(); for e in prev { *counts.entry(key(e)).or_default() += 1; } for e in now { *counts.entry(key(e)).or_default() -= 1; } let mut lines = Vec::new(); // Order: as they appear in the current picture (adds), then as they were listed (removes). let mut shown: std::collections::HashSet<(&str, &str)> = std::collections::HashSet::new(); for e in now { let k = key(e); if counts.get(&k) == Some(&-1) && shown.insert(k) { lines.push(format!("+ {}「{}」", e.kind, e.name)); } } for e in prev { let k = key(e); if counts.get(&k) == Some(&1) && shown.insert(k) { lines.push(format!("- {}「{}」", e.kind, e.name)); } } // A dialog that appeared entirely is the case that matters: show it in full. lines.truncate(12); lines } pub fn fingerprint(elements: &[Element]) -> u64 { use std::hash::{Hash, Hasher}; let mut h = std::collections::hash_map::DefaultHasher::new(); for e in elements { (e.kind.as_str(), e.name.as_str(), e.value.as_str(), e.x as i64, e.y as i64, e.w as i64, e.h as i64, e.flags.len()).hash(&mut h); } h.finish() } /// A coarse fingerprint of a picture (16×10 grey cells), for "nothing changed" when there is no /// tree to compare. pub fn picture_fingerprint(img: &image::RgbImage) -> u64 { use std::hash::{Hash, Hasher}; let small = image::imageops::resize(img, 16, 10, image::imageops::FilterType::Triangle); let mut h = std::collections::hash_map::DefaultHasher::new(); for p in small.pixels() { // Quantised, so a blinking caret or a clock's second doesn't count as a change. ((p[0] as u16 + p[1] as u16 + p[2] as u16) / 24).hash(&mut h); } h.finish() } const COLOURS: [[u8; 3]; 6] = [[220, 38, 38], [37, 99, 235], [22, 163, 74], [192, 38, 211], [234, 88, 12], [8, 145, 178]]; /// Box and number each control on the picture. `to_pic` maps screen units to picture pixels. pub fn draw(img: &mut image::RgbImage, numbered: &[(usize, Element)], to_pic: (f64, f64)) { let (iw, ih) = (img.width() as i64, img.height() as i64); for (n, e) in numbered { let c = COLOURS[n % COLOURS.len()]; let (x0, y0) = ((e.x / to_pic.0).round() as i64, (e.y / to_pic.1).round() as i64); let (x1, y1) = (((e.x + e.w) / to_pic.0).round() as i64, ((e.y + e.h) / to_pic.1).round() as i64); rect_outline(img, x0, y0, x1, y1, c); // The number: white digits on the control's colour, at its top-left corner (inside the // picture). let label = n.to_string(); let (lw, lh) = (label.len() as i64 * (DIGIT_W + 1) * SCALE + 3, DIGIT_H * SCALE + 4); let lx = x0.clamp(0, (iw - lw).max(0)); let ly = (y0 - lh).clamp(0, (ih - lh).max(0)); let ly = if ly + lh > y0 && y0 >= lh { y0 - lh } else { ly }; fill(img, lx, ly, lx + lw, ly + lh, c); let mut cx = lx + 2; for ch in label.bytes() { digit(img, cx, ly + 2, ch - b'0', [255, 255, 255]); cx += (DIGIT_W + 1) * SCALE; } } } fn put(img: &mut image::RgbImage, x: i64, y: i64, c: [u8; 3]) { if x >= 0 && y >= 0 && (x as u32) < img.width() && (y as u32) < img.height() { img.put_pixel(x as u32, y as u32, image::Rgb(c)); } } fn fill(img: &mut image::RgbImage, x0: i64, y0: i64, x1: i64, y1: i64, c: [u8; 3]) { for y in y0..y1 { for x in x0..x1 { put(img, x, y, c); } } } fn rect_outline(img: &mut image::RgbImage, x0: i64, y0: i64, x1: i64, y1: i64, c: [u8; 3]) { for x in x0..=x1 { put(img, x, y0, c); put(img, x, y1, c); } for y in y0..=y1 { put(img, x0, y, c); put(img, x1, y, c); } } /// A 3×5 digit font, each pixel drawn SCALE×SCALE. const DIGIT_W: i64 = 3; const DIGIT_H: i64 = 5; const SCALE: i64 = 2; const DIGITS: [[u8; 5]; 10] = [ [0b111, 0b101, 0b101, 0b101, 0b111], [0b010, 0b110, 0b010, 0b010, 0b111], [0b111, 0b001, 0b111, 0b100, 0b111], [0b111, 0b001, 0b111, 0b001, 0b111], [0b101, 0b101, 0b111, 0b001, 0b001], [0b111, 0b100, 0b111, 0b001, 0b111], [0b111, 0b100, 0b111, 0b101, 0b111], [0b111, 0b001, 0b010, 0b010, 0b010], [0b111, 0b101, 0b111, 0b101, 0b111], [0b111, 0b101, 0b111, 0b001, 0b111], ]; fn digit(img: &mut image::RgbImage, x: i64, y: i64, d: u8, c: [u8; 3]) { let rows = DIGITS[(d as usize).min(9)]; for (r, bits) in rows.iter().enumerate() { for col in 0..DIGIT_W { if bits >> (DIGIT_W - 1 - col) & 1 == 1 { fill(img, x + col * SCALE, y + r as i64 * SCALE, x + (col + 1) * SCALE, y + (r as i64 + 1) * SCALE, c); } } } } #[cfg(test)] mod tests { use super::*; fn mel(kind: &str, name: &str) -> Element { Element { kind: kind.into(), name: name.into(), value: String::new(), x: 0.0, y: 0.0, w: 10.0, h: 10.0, active: true, flags: vec![], window: String::new() } } #[test] fn the_change_between_two_looks_is_two_lines() { let before = vec![mel("按鈕", "Save"), mel("輸入框", "標題")]; let after = vec![mel("按鈕", "Save"), mel("按鈕", "OK"), mel("按鈕", "Cancel")]; assert_eq!(diff(&before, &after), ["+ 按鈕「OK」", "+ 按鈕「Cancel」", "- 輸入框「標題」"]); // Same controls twice = a multiset: one of two identical buttons leaving is a change. let two = vec![mel("按鈕", "x"), mel("按鈕", "x")]; assert_eq!(diff(&two, &[mel("按鈕", "x")]), ["- 按鈕「x」"]); assert!(diff(&before, &before).is_empty()); } fn el(kind: &str, name: &str, x: f64, y: f64, active: bool) -> Element { Element { kind: kind.into(), name: name.into(), value: String::new(), x, y, w: 40.0, h: 20.0, active, flags: vec![], window: "W".into() } } /// A control still there keeps its number from look to look, even when other controls come /// and go before it in reading order; a new one gets a number above the highest; a look at a /// different window starts from 1. #[test] fn a_control_can_be_named_instead_of_numbered() { let els = vec![el("按鈕", "4", 10.0, 10.0, true), el("按鈕", "AC", 60.0, 10.0, true), el("按鈕", "C", 110.0, 10.0, true), el("勾選框", "自動更新", 10.0, 60.0, true)]; let (_, _, spots) = number(&els, false); assert_eq!(spots.named("4").unwrap(), 1); assert_eq!(spots.named("「C」").unwrap(), 3, "C is not AC"); assert!(spots.named("Save").unwrap_err().contains("沒有叫「Save」")); // A near miss names what is there. let near = spots.named("自動").unwrap_err(); assert!(near.contains("名字相近") && near.contains("[4]「自動更新」"), "{near}"); let mut v = serde_json::json!({"actions": [{"action": "click", "name": "4"}, {"action": "click", "ref": "2"}, {"action": "typing", "value": "x"}]}); names_to_numbers(&mut v, Some(&spots)).unwrap(); assert_eq!(v["actions"][0]["ref"], "1"); assert_eq!(v["actions"][1]["ref"], "2"); assert!(v["actions"][2]["ref"].is_null()); let mut one = serde_json::json!({"name": "自動更新"}); names_to_numbers(&mut one, Some(&spots)).unwrap(); assert_eq!(one["ref"], "4"); // A name with x: 0, y: 0 beside it: the name. let mut both = serde_json::json!({"name": "AC", "x": 0.0, "y": 0.0}); names_to_numbers(&mut both, Some(&spots)).unwrap(); assert_eq!(both["ref"], "2"); } #[test] fn numbers_stay_with_their_controls() { let mut m = Numbering::default(); let keys = |extra: bool| { let mut v = vec![]; if extra { v.push(el("選單", "說明", 5.0, 5.0, true)); } for (i, k) in ["7", "8", "9", "*"].iter().enumerate() { v.push(el("按鈕", k, 100.0 + 50.0 * i as f64, 200.0, true)); } v }; let (first, _, _) = number_keeping(&keys(false), false, &mut m); let n_of = |list: &[(usize, Element)], name: &str| list.iter().find(|(_, e)| e.name == name).map(|(n, _)| *n).unwrap(); assert_eq!(n_of(&first, "9"), 3); // A menu appears first in reading order: the keys keep their numbers, the menu is new. let (second, lines, spots) = number_keeping(&keys(true), false, &mut m); assert_eq!(n_of(&second, "9"), 3); assert_eq!(n_of(&second, "說明"), 5); assert_eq!(spots.get("3").unwrap().1, "按鈕「9」"); assert_eq!(lines[0], "[5] 選單「說明」"); // Moved a little (a list scrolled): still the same control. let mut moved = keys(true); for e in moved.iter_mut() { e.y += 30.0; } let (third, _, _) = number_keeping(&moved, false, &mut m); assert_eq!(n_of(&third, "9"), 3); // Another window altogether: from 1 again. let other: Vec = keys(false).into_iter().map(|e| Element { window: "Other".into(), ..e }).collect(); let (fourth, _, _) = number_keeping(&other, false, &mut m); assert_eq!(n_of(&fourth, "7"), 1); } #[test] fn controls_are_numbered_in_reading_order() { let els = vec![el("按鈕", "下", 10.0, 300.0, true), el("連結", "Profile", 20.0, 100.0, true), el("文字", "標題", 0.0, 50.0, false), el("連結", "Profile", 20.0, 100.0, true)]; let (numbered, lines, spots) = number(&els, false); // Reading order, duplicates once, plain text left out when there is a picture. assert_eq!(lines, vec!["[1] 連結「Profile」", "[2] 按鈕「下」"]); assert_eq!(numbered.len(), 2); assert_eq!(spots.get("1").unwrap(), ((40.0, 110.0), "連結「Profile」".to_string())); assert_eq!(spots.get("e2").unwrap().0, (30.0, 310.0)); assert!(spots.get("3").is_err() && spots.get("ok").is_err()); // Without a picture the text is listed too (not numbered). let (_, lines, _) = number(&els, true); assert_eq!(lines, vec![" 文字「標題」", "[1] 連結「Profile」", "[2] 按鈕「下」"]); } #[test] fn numbers_are_drawn_on_the_picture() { let mut img = image::RgbImage::from_pixel(200, 100, image::Rgb([255, 255, 255])); let els = vec![el("按鈕", "OK", 100.0, 60.0, true)]; let (numbered, _, _) = number(&els, false); // Screen twice the picture's size: the box lands at (50, 30)–(70, 40). draw(&mut img, &numbered, (2.0, 2.0)); assert_eq!(img.get_pixel(50, 35).0, COLOURS[1]); assert_eq!(img.get_pixel(60, 30).0, COLOURS[1]); // The label sits above the box with white digits in it. let label_has_white = (16..30).any(|y| (50..62).any(|x| img.get_pixel(x, y).0 == [255, 255, 255])); let label_has_colour = (16..30).any(|y| (50..62).any(|x| img.get_pixel(x, y).0 == COLOURS[1])); assert!(label_has_white && label_has_colour); } #[test] fn a_still_screen_has_the_same_fingerprint() { let a = vec![el("按鈕", "OK", 1.0, 2.0, true)]; let mut b = a.clone(); assert_eq!(fingerprint(&a), fingerprint(&b)); b[0].name = "Cancel".into(); assert_ne!(fingerprint(&a), fingerprint(&b)); let img = image::RgbImage::from_pixel(64, 40, image::Rgb([10, 20, 30])); let mut img2 = img.clone(); img2.put_pixel(0, 0, image::Rgb([12, 20, 30])); assert_eq!(picture_fingerprint(&img), picture_fingerprint(&img2)); } }