diff --git a/src/cloudbox.rs b/src/cloudbox.rs index d490457..baa5591 100644 --- a/src/cloudbox.rs +++ b/src/cloudbox.rs @@ -2092,6 +2092,17 @@ fn box_look(script: Option<&str>, after: Option, pointer: bool) -> Resul } else { text.push_str(&format!("\n可以操作的東西(圖上同樣的編號,用 ref 點):\n{}", lines.join("\n"))); } + // What moved since the last look (same signal the Mac path gives the quick model). + static LAST_ELS: Mutex> = Mutex::new(Vec::new()); + { + let cur: Vec = numbered.iter().map(|(_, e)| e.clone()).collect(); + let mut last = LAST_ELS.lock().unwrap_or_else(|e| e.into_inner()); + let d = crate::marks::diff(&last, &cur); + *last = cur; + if !d.is_empty() { + text.push_str(&format!("\n跟上一張圖相比:\n{}", d.join("\n"))); + } + } // Step timing (for the 電腦 card): how long the box's side took vs. the whole step. { let total_ms = t_all.elapsed().as_millis() as u64; diff --git a/src/computer.rs b/src/computer.rs index c511067..07a8f6f 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -1169,6 +1169,17 @@ fn look(after: Option, pointer: bool) -> Result<(String, Option) text.push_str(&format!("\n讀不到最前面「{front}」視窗裡的按鈕{why}:下面只有選單列。視窗裡的東西照圖上的位置點 x、y,或等一下再截一次圖。\n{}", lines.join("\n"))); } else { text.push_str(&format!("\n「{app}」上可以操作的東西(圖上同樣的編號,用 ref 點):\n{}", lines.join("\n"))); + // What moved since the last look (the quick model's step-effect signal). + static LAST_ELS: Mutex> = Mutex::new(Vec::new()); + { + let cur: Vec = numbered.iter().map(|(_, e)| e.clone()).collect(); + let mut last = LAST_ELS.lock().unwrap_or_else(|e| e.into_inner()); + let d = crate::marks::diff(&last, &cur); + *last = cur; + if !d.is_empty() { + text.push_str(&format!("\n跟上一張圖相比:\n{}", d.join("\n"))); + } + } } text.push_str(a11y_note); Ok((text, img_b64)) diff --git a/src/marks.rs b/src/marks.rs index 4b0e0ce..532104c 100644 --- a/src/marks.rs +++ b/src/marks.rs @@ -243,6 +243,40 @@ fn describe(e: &Element) -> String { } /// A fingerprint of what the controls are and where (to say "nothing changed"). +/// What came and what went between two looks at the screen (multiset of (kind, name)). +/// The quick model reads this to know what its last step did, without re-reading the whole +/// numbered list (the list is long; the change is usually two lines). +pub fn diff(prev: &[Element], now: &[Element]) -> Vec { + fn key(e: &Element) -> (&str, &str) { + (e.kind.as_str(), e.name.as_str()) + } + let mut counts: HashMap<(&str, &str), i32> = HashMap::new(); + for e in prev { + *counts.entry(key(e)).or_default() += 1; + } + for e in now { + *counts.entry(key(e)).or_default() -= 1; + } + let mut lines = Vec::new(); + // Order: as they appear in the current picture (adds), then as they were listed (removes). + let mut shown: std::collections::HashSet<(&str, &str)> = std::collections::HashSet::new(); + for e in now { + let k = key(e); + if counts.get(&k) == Some(&-1) && shown.insert(k) { + lines.push(format!("+ {}「{}」", e.kind, e.name)); + } + } + for e in prev { + let k = key(e); + if counts.get(&k) == Some(&1) && shown.insert(k) { + lines.push(format!("- {}「{}」", e.kind, e.name)); + } + } + // A dialog that appeared entirely is the case that matters: show it in full. + lines.truncate(12); + lines +} + pub fn fingerprint(elements: &[Element]) -> u64 { use std::hash::{Hash, Hasher}; let mut h = std::collections::hash_map::DefaultHasher::new(); @@ -348,6 +382,22 @@ fn digit(img: &mut image::RgbImage, x: i64, y: i64, d: u8, c: [u8; 3]) { mod tests { use super::*; + fn mel(kind: &str, name: &str) -> Element { + Element { kind: kind.into(), name: name.into(), value: String::new(), x: 0.0, y: 0.0, w: 10.0, h: 10.0, active: true, flags: vec![], window: String::new() } + } + + #[test] + fn the_change_between_two_looks_is_two_lines() { + let before = vec![mel("按鈕", "Save"), mel("輸入框", "標題")]; + let after = vec![mel("按鈕", "Save"), mel("按鈕", "OK"), mel("按鈕", "Cancel")]; + assert_eq!(diff(&before, &after), ["+ 按鈕「OK」", "+ 按鈕「Cancel」", "- 輸入框「標題」"]); + // Same controls twice = a multiset: one of two identical buttons leaving is a change. + let two = vec![mel("按鈕", "x"), mel("按鈕", "x")]; + assert_eq!(diff(&two, &[mel("按鈕", "x")]), ["- 按鈕「x」"]); + assert!(diff(&before, &before).is_empty()); + } + + fn el(kind: &str, name: &str, x: f64, y: f64, active: bool) -> Element { Element { kind: kind.into(), name: name.into(), value: String::new(), x, y, w: 40.0, h: 20.0, active, flags: vec![], window: "W".into() } }