510 lines
23 KiB
Rust
510 lines
23 KiB
Rust
// What a bot sees of a screen: the picture with every control it can use boxed and numbered,
|
||
// and the same numbers in a list ("[12] 連結「Profile」"). It acts by number; the click lands on
|
||
// that control's centre. Guessing coordinates from a picture is where clicks went wrong (a
|
||
// model took an empty strip for a menu item and clicked it ten times); a number can't miss.
|
||
//
|
||
// The controls come from the system's accessibility tree: macOS AX and Linux AT-SPI through
|
||
// agent-desktop on a person's computer (desktop.rs), AT-SPI in the bots' own computer
|
||
// (box/a11y). Web pages are in those trees too (Chromium with renderer accessibility on).
|
||
|
||
use std::collections::HashMap;
|
||
|
||
/// One control on the screen, in screen units (the mouse's: points on a Mac, pixels elsewhere).
|
||
#[derive(Clone, Debug, PartialEq)]
|
||
pub struct Element {
|
||
/// 按鈕, 連結, 輸入框… (already in words).
|
||
pub kind: String,
|
||
pub name: String,
|
||
pub value: String,
|
||
pub x: f64,
|
||
pub y: f64,
|
||
pub w: f64,
|
||
pub h: f64,
|
||
/// Something to click or type into (gets a number), not just text.
|
||
pub active: bool,
|
||
pub flags: Vec<&'static str>,
|
||
pub window: String,
|
||
}
|
||
|
||
impl Element {
|
||
pub fn centre(&self) -> (f64, f64) {
|
||
(self.x + self.w / 2.0, self.y + self.h / 2.0)
|
||
}
|
||
}
|
||
|
||
/// Most controls numbered on one picture (more is clutter the model can't use).
|
||
pub const MAX_MARKS: usize = 150;
|
||
|
||
/// The numbered controls of the latest look at a screen: number → (centre, words).
|
||
#[derive(Clone, Debug, Default)]
|
||
pub struct Spots {
|
||
pub by_number: HashMap<usize, ((f64, f64), String)>,
|
||
}
|
||
|
||
impl Spots {
|
||
/// "12", "e12", "[12]" → the control's centre and what it is.
|
||
pub fn get(&self, r: &str) -> Result<((f64, f64), String), String> {
|
||
let n = number_of(r).ok_or_else(|| format!("「{}」不是編號：請用截圖上的編號（例如 12）", r.trim()))?;
|
||
self.by_number.get(&n).cloned().ok_or_else(|| format!("最新的畫面裡沒有編號 {n}：先用 computer（action=screenshot）看一次畫面再點。"))
|
||
}
|
||
}
|
||
|
||
impl Spots {
|
||
/// The numbered control called `name` in the latest look (its name as listed: 「4」, 「Close」,
|
||
/// 「自動更新」), so a step can say which button the way a person would. Ok(number) or why not.
|
||
pub fn named(&self, name: &str) -> Result<usize, String> {
|
||
let want = name.trim().trim_matches(['「', '」']);
|
||
let name_of = |what: &str| what.split_once('「').and_then(|(_, r)| r.split_once('」')).map(|(n, _)| n.to_string());
|
||
let mut hits: Vec<usize> = self.by_number.iter().filter(|(_, (_, what))| name_of(what).as_deref() == Some(want)).map(|(n, _)| *n).collect();
|
||
hits.sort();
|
||
match hits.as_slice() {
|
||
[n] => Ok(*n),
|
||
[] => {
|
||
// Names that come close (one inside the other, ignoring case), so the step can be
|
||
// put right at once instead of guessed again.
|
||
let low = want.to_lowercase();
|
||
let mut near: Vec<(usize, String)> = self
|
||
.by_number
|
||
.iter()
|
||
.filter_map(|(n, (_, what))| name_of(what).filter(|nm| !nm.is_empty()).map(|nm| (*n, nm)))
|
||
.filter(|(_, nm)| {
|
||
let l = nm.to_lowercase();
|
||
!low.is_empty() && (l.contains(&low) || low.contains(&l))
|
||
})
|
||
.collect();
|
||
near.sort();
|
||
near.truncate(6);
|
||
if near.is_empty() {
|
||
Err(format!("最新的畫面裡沒有叫「{want}」的東西：照最新清單上的名字或編號點，不要猜名字。"))
|
||
} else {
|
||
Err(format!("最新的畫面裡沒有叫「{want}」的東西。名字相近的有：{}。", near.iter().map(|(n, nm)| format!("[{n}]「{nm}」")).collect::<Vec<_>>().join("、")))
|
||
}
|
||
}
|
||
many => Err(format!("最新的畫面裡有 {} 個叫「{want}」的東西（編號 {}）：請用編號指定是哪一個。", many.len(), many.iter().map(|n| n.to_string()).collect::<Vec<_>>().join("、"))),
|
||
}
|
||
}
|
||
}
|
||
|
||
/// A step that names its control (`name`, no number, no x/y) gets that control's number from
|
||
/// the latest look; so does every action of a batch.
|
||
pub fn names_to_numbers(v: &mut serde_json::Value, spots: Option<&Spots>) -> Result<(), String> {
|
||
let fix = |p: &mut serde_json::Value| -> Result<(), String> {
|
||
// ("name" in the screen tools; "text" too, the word the browser tools use.)
|
||
let named = ["name", "text"].iter().find_map(|k| p[*k].as_str().map(str::trim).filter(|t| !t.is_empty()).map(String::from));
|
||
if let Some(t) = named {
|
||
// A name wins over coordinates sent along with it (x: 0, y: 0 from a model filling
|
||
// every field): the click went to the screen's corner.
|
||
if p["ref"].is_null() || p["ref"].as_str().is_some_and(|r| r.trim().is_empty()) {
|
||
let s = spots.ok_or("先用 computer_use_screenshot 看一下畫面，再照清單上的名字或編號操作。")?;
|
||
p["ref"] = serde_json::Value::String(s.named(&t)?.to_string());
|
||
}
|
||
}
|
||
Ok(())
|
||
};
|
||
if let Some(actions) = v["actions"].as_array_mut() {
|
||
for a in actions.iter_mut() {
|
||
fix(a)?;
|
||
}
|
||
return Ok(());
|
||
}
|
||
fix(v)
|
||
}
|
||
|
||
pub fn number_of(r: &str) -> Option<usize> {
|
||
let t = r.trim().trim_matches(['[', ']']).trim_start_matches(['e', 'E', '#']);
|
||
t.parse().ok()
|
||
}
|
||
|
||
/// What the last look's numbers meant, so the next look keeps them: a control that is still
|
||
/// there keeps its number (the 「7」 key stays #141 after 「1」 was pressed). Renumbering every
|
||
/// look broke steps planned from one picture: a bot sending five clicks at once had the second
|
||
/// to fifth land on whatever took those numbers after the first click.
|
||
#[derive(Default, Debug, Clone)]
|
||
pub struct Numbering {
|
||
/// (window, kind, name, position bucket) → number.
|
||
exact: HashMap<(String, String, String, i64, i64), usize>,
|
||
/// (window, kind, name) → numbers (moved a little, still the same control when only one).
|
||
loose: HashMap<(String, String, String), Vec<usize>>,
|
||
windows: Vec<String>,
|
||
top: usize,
|
||
}
|
||
|
||
/// Position buckets for "the same control" (screen units; a list scrolled a little stays put).
|
||
const SAME_SPOT: f64 = 24.0;
|
||
/// Numbers grow while windows stay; past this they start again from 1.
|
||
const MAX_NUMBER: usize = 600;
|
||
|
||
/// Order (reading order, by window) and number the controls; the list's lines and the spots.
|
||
/// `all_text`: also list plain text (when there is no picture to read it from).
|
||
#[cfg(test)]
|
||
pub fn number(elements: &[Element], all_text: bool) -> (Vec<(usize, Element)>, Vec<String>, Spots) {
|
||
number_keeping(elements, all_text, &mut Numbering::default())
|
||
}
|
||
|
||
/// Same, keeping the numbers of controls that were there in the last look (`memory`, updated).
|
||
/// Starts again from 1 when none of the last look's windows is still there.
|
||
pub fn number_keeping(elements: &[Element], all_text: bool, memory: &mut Numbering) -> (Vec<(usize, Element)>, Vec<String>, Spots) {
|
||
let mut seen = std::collections::HashSet::new();
|
||
let mut els: Vec<&Element> = elements.iter().filter(|e| e.w >= 1.0 && e.h >= 1.0 && seen.insert((e.kind.clone(), e.name.clone(), e.x as i64, e.y as i64, e.w as i64, e.h as i64))).collect();
|
||
let mut windows: Vec<String> = vec![];
|
||
for e in &els {
|
||
if !windows.contains(&e.window) {
|
||
windows.push(e.window.clone());
|
||
}
|
||
}
|
||
els.sort_by(|a, b| {
|
||
let wa = windows.iter().position(|w| *w == a.window).unwrap_or(0);
|
||
let wb = windows.iter().position(|w| *w == b.window).unwrap_or(0);
|
||
(wa, (a.y / 12.0) as i64, a.x as i64).cmp(&(wb, (b.y / 12.0) as i64, b.x as i64))
|
||
});
|
||
// Which controls get numbers (reading order, at most MAX_MARKS), then which numbers: the
|
||
// last look's where the same control is still there, new ones above the highest.
|
||
let marked: Vec<&Element> = els.iter().copied().filter(|e| e.active).take(MAX_MARKS).collect();
|
||
let fresh = memory.top > MAX_NUMBER || !windows.iter().any(|w| memory.windows.contains(w));
|
||
if fresh {
|
||
*memory = Numbering::default();
|
||
}
|
||
let exact_of = |e: &Element| (e.window.clone(), e.kind.clone(), e.name.clone(), (e.centre().0 / SAME_SPOT).round() as i64, (e.centre().1 / SAME_SPOT).round() as i64);
|
||
let loose_of = |e: &Element| (e.window.clone(), e.kind.clone(), e.name.clone());
|
||
let mut taken: std::collections::HashSet<usize> = std::collections::HashSet::new();
|
||
let mut assigned: Vec<Option<usize>> = marked.iter().map(|e| memory.exact.get(&exact_of(e)).copied()).collect();
|
||
for n in assigned.iter().flatten() {
|
||
taken.insert(*n);
|
||
}
|
||
// Moved a little: the one control of that kind and name in that window, both times.
|
||
let mut loose_now: HashMap<(String, String, String), usize> = HashMap::new();
|
||
for e in &marked {
|
||
*loose_now.entry(loose_of(e)).or_default() += 1;
|
||
}
|
||
for (i, e) in marked.iter().enumerate() {
|
||
if assigned[i].is_none() && loose_now.get(&loose_of(e)) == Some(&1) {
|
||
if let Some([n]) = memory.loose.get(&loose_of(e)).map(Vec::as_slice) {
|
||
if !taken.contains(n) {
|
||
assigned[i] = Some(*n);
|
||
taken.insert(*n);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
let mut top = memory.top.max(taken.iter().copied().max().unwrap_or(0));
|
||
let mut next = Numbering { windows: windows.clone(), ..Default::default() };
|
||
let mut number_of_el: HashMap<*const Element, usize> = HashMap::new();
|
||
for (i, e) in marked.iter().enumerate() {
|
||
let n = assigned[i].unwrap_or_else(|| {
|
||
top += 1;
|
||
top
|
||
});
|
||
number_of_el.insert(*e as *const Element, n);
|
||
next.exact.insert(exact_of(e), n);
|
||
next.loose.entry(loose_of(e)).or_default().push(n);
|
||
}
|
||
next.top = top;
|
||
*memory = next;
|
||
let mut numbered = vec![];
|
||
let mut lines = vec![];
|
||
let mut spots = Spots::default();
|
||
let mut window = None;
|
||
for e in els {
|
||
if !e.active && !all_text {
|
||
continue;
|
||
}
|
||
if windows.len() > 1 && window.as_ref() != Some(&e.window) && !e.window.is_empty() {
|
||
window = Some(e.window.clone());
|
||
lines.push(format!("## {}", e.window));
|
||
}
|
||
let what = describe(e);
|
||
if let Some(&n) = number_of_el.get(&(e as *const Element)) {
|
||
spots.by_number.insert(n, (e.centre(), what.clone()));
|
||
numbered.push((n, e.clone()));
|
||
lines.push(format!("[{n}] {what}"));
|
||
} else if !e.active {
|
||
lines.push(format!(" {what}"));
|
||
}
|
||
}
|
||
(numbered, lines, spots)
|
||
}
|
||
|
||
fn describe(e: &Element) -> String {
|
||
let short = |s: &str, n: usize| -> String {
|
||
let c: Vec<char> = s.chars().collect();
|
||
if c.len() > n { format!("{}…", c[..n].iter().collect::<String>()) } else { s.to_string() }
|
||
};
|
||
let mut t = e.kind.clone();
|
||
if !e.name.is_empty() {
|
||
t.push_str(&format!("「{}」", short(&e.name, 60)));
|
||
}
|
||
if !e.value.is_empty() && e.value != e.name {
|
||
t.push_str(&format!("：{}", short(&e.value, 80)));
|
||
}
|
||
if !e.flags.is_empty() {
|
||
t.push_str(&format!("（{}）", e.flags.join("、")));
|
||
}
|
||
t
|
||
}
|
||
|
||
/// A fingerprint of what the controls are and where (to say "nothing changed").
|
||
/// What came and what went between two looks at the screen (multiset of (kind, name)).
|
||
/// The quick model reads this to know what its last step did, without re-reading the whole
|
||
/// numbered list (the list is long; the change is usually two lines).
|
||
pub fn diff(prev: &[Element], now: &[Element]) -> Vec<String> {
|
||
fn key(e: &Element) -> (&str, &str) {
|
||
(e.kind.as_str(), e.name.as_str())
|
||
}
|
||
let mut counts: HashMap<(&str, &str), i32> = HashMap::new();
|
||
for e in prev {
|
||
*counts.entry(key(e)).or_default() += 1;
|
||
}
|
||
for e in now {
|
||
*counts.entry(key(e)).or_default() -= 1;
|
||
}
|
||
let mut lines = Vec::new();
|
||
// Order: as they appear in the current picture (adds), then as they were listed (removes).
|
||
let mut shown: std::collections::HashSet<(&str, &str)> = std::collections::HashSet::new();
|
||
for e in now {
|
||
let k = key(e);
|
||
if counts.get(&k) == Some(&-1) && shown.insert(k) {
|
||
lines.push(format!("＋ {}「{}」", e.kind, e.name));
|
||
}
|
||
}
|
||
for e in prev {
|
||
let k = key(e);
|
||
if counts.get(&k) == Some(&1) && shown.insert(k) {
|
||
lines.push(format!("－ {}「{}」", e.kind, e.name));
|
||
}
|
||
}
|
||
// A dialog that appeared entirely is the case that matters: show it in full.
|
||
lines.truncate(12);
|
||
lines
|
||
}
|
||
|
||
pub fn fingerprint(elements: &[Element]) -> u64 {
|
||
use std::hash::{Hash, Hasher};
|
||
let mut h = std::collections::hash_map::DefaultHasher::new();
|
||
for e in elements {
|
||
(e.kind.as_str(), e.name.as_str(), e.value.as_str(), e.x as i64, e.y as i64, e.w as i64, e.h as i64, e.flags.len()).hash(&mut h);
|
||
}
|
||
h.finish()
|
||
}
|
||
|
||
/// A coarse fingerprint of a picture (16×10 grey cells), for "nothing changed" when there is no
|
||
/// tree to compare.
|
||
pub fn picture_fingerprint(img: &image::RgbImage) -> u64 {
|
||
use std::hash::{Hash, Hasher};
|
||
let small = image::imageops::resize(img, 16, 10, image::imageops::FilterType::Triangle);
|
||
let mut h = std::collections::hash_map::DefaultHasher::new();
|
||
for p in small.pixels() {
|
||
// Quantised, so a blinking caret or a clock's second doesn't count as a change.
|
||
((p[0] as u16 + p[1] as u16 + p[2] as u16) / 24).hash(&mut h);
|
||
}
|
||
h.finish()
|
||
}
|
||
|
||
const COLOURS: [[u8; 3]; 6] = [[220, 38, 38], [37, 99, 235], [22, 163, 74], [192, 38, 211], [234, 88, 12], [8, 145, 178]];
|
||
|
||
/// Box and number each control on the picture. `to_pic` maps screen units to picture pixels.
|
||
pub fn draw(img: &mut image::RgbImage, numbered: &[(usize, Element)], to_pic: (f64, f64)) {
|
||
let (iw, ih) = (img.width() as i64, img.height() as i64);
|
||
for (n, e) in numbered {
|
||
let c = COLOURS[n % COLOURS.len()];
|
||
let (x0, y0) = ((e.x / to_pic.0).round() as i64, (e.y / to_pic.1).round() as i64);
|
||
let (x1, y1) = (((e.x + e.w) / to_pic.0).round() as i64, ((e.y + e.h) / to_pic.1).round() as i64);
|
||
rect_outline(img, x0, y0, x1, y1, c);
|
||
// The number: white digits on the control's colour, at its top-left corner (inside the
|
||
// picture).
|
||
let label = n.to_string();
|
||
let (lw, lh) = (label.len() as i64 * (DIGIT_W + 1) * SCALE + 3, DIGIT_H * SCALE + 4);
|
||
let lx = x0.clamp(0, (iw - lw).max(0));
|
||
let ly = (y0 - lh).clamp(0, (ih - lh).max(0));
|
||
let ly = if ly + lh > y0 && y0 >= lh { y0 - lh } else { ly };
|
||
fill(img, lx, ly, lx + lw, ly + lh, c);
|
||
let mut cx = lx + 2;
|
||
for ch in label.bytes() {
|
||
digit(img, cx, ly + 2, ch - b'0', [255, 255, 255]);
|
||
cx += (DIGIT_W + 1) * SCALE;
|
||
}
|
||
}
|
||
}
|
||
|
||
fn put(img: &mut image::RgbImage, x: i64, y: i64, c: [u8; 3]) {
|
||
if x >= 0 && y >= 0 && (x as u32) < img.width() && (y as u32) < img.height() {
|
||
img.put_pixel(x as u32, y as u32, image::Rgb(c));
|
||
}
|
||
}
|
||
|
||
fn fill(img: &mut image::RgbImage, x0: i64, y0: i64, x1: i64, y1: i64, c: [u8; 3]) {
|
||
for y in y0..y1 {
|
||
for x in x0..x1 {
|
||
put(img, x, y, c);
|
||
}
|
||
}
|
||
}
|
||
|
||
fn rect_outline(img: &mut image::RgbImage, x0: i64, y0: i64, x1: i64, y1: i64, c: [u8; 3]) {
|
||
for x in x0..=x1 {
|
||
put(img, x, y0, c);
|
||
put(img, x, y1, c);
|
||
}
|
||
for y in y0..=y1 {
|
||
put(img, x0, y, c);
|
||
put(img, x1, y, c);
|
||
}
|
||
}
|
||
|
||
/// A 3×5 digit font, each pixel drawn SCALE×SCALE.
|
||
const DIGIT_W: i64 = 3;
|
||
const DIGIT_H: i64 = 5;
|
||
const SCALE: i64 = 2;
|
||
const DIGITS: [[u8; 5]; 10] = [
|
||
[0b111, 0b101, 0b101, 0b101, 0b111],
|
||
[0b010, 0b110, 0b010, 0b010, 0b111],
|
||
[0b111, 0b001, 0b111, 0b100, 0b111],
|
||
[0b111, 0b001, 0b111, 0b001, 0b111],
|
||
[0b101, 0b101, 0b111, 0b001, 0b001],
|
||
[0b111, 0b100, 0b111, 0b001, 0b111],
|
||
[0b111, 0b100, 0b111, 0b101, 0b111],
|
||
[0b111, 0b001, 0b010, 0b010, 0b010],
|
||
[0b111, 0b101, 0b111, 0b101, 0b111],
|
||
[0b111, 0b101, 0b111, 0b001, 0b111],
|
||
];
|
||
|
||
fn digit(img: &mut image::RgbImage, x: i64, y: i64, d: u8, c: [u8; 3]) {
|
||
let rows = DIGITS[(d as usize).min(9)];
|
||
for (r, bits) in rows.iter().enumerate() {
|
||
for col in 0..DIGIT_W {
|
||
if bits >> (DIGIT_W - 1 - col) & 1 == 1 {
|
||
fill(img, x + col * SCALE, y + r as i64 * SCALE, x + (col + 1) * SCALE, y + (r as i64 + 1) * SCALE, c);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
fn mel(kind: &str, name: &str) -> Element {
|
||
Element { kind: kind.into(), name: name.into(), value: String::new(), x: 0.0, y: 0.0, w: 10.0, h: 10.0, active: true, flags: vec![], window: String::new() }
|
||
}
|
||
|
||
#[test]
|
||
fn the_change_between_two_looks_is_two_lines() {
|
||
let before = vec![mel("按鈕", "Save"), mel("輸入框", "標題")];
|
||
let after = vec![mel("按鈕", "Save"), mel("按鈕", "OK"), mel("按鈕", "Cancel")];
|
||
assert_eq!(diff(&before, &after), ["＋ 按鈕「OK」", "＋ 按鈕「Cancel」", "－ 輸入框「標題」"]);
|
||
// Same controls twice = a multiset: one of two identical buttons leaving is a change.
|
||
let two = vec![mel("按鈕", "x"), mel("按鈕", "x")];
|
||
assert_eq!(diff(&two, &[mel("按鈕", "x")]), ["－ 按鈕「x」"]);
|
||
assert!(diff(&before, &before).is_empty());
|
||
}
|
||
|
||
|
||
fn el(kind: &str, name: &str, x: f64, y: f64, active: bool) -> Element {
|
||
Element { kind: kind.into(), name: name.into(), value: String::new(), x, y, w: 40.0, h: 20.0, active, flags: vec![], window: "W".into() }
|
||
}
|
||
|
||
/// A control still there keeps its number from look to look, even when other controls come
|
||
/// and go before it in reading order; a new one gets a number above the highest; a look at a
|
||
/// different window starts from 1.
|
||
#[test]
|
||
fn a_control_can_be_named_instead_of_numbered() {
|
||
let els = vec![el("按鈕", "4", 10.0, 10.0, true), el("按鈕", "AC", 60.0, 10.0, true), el("按鈕", "C", 110.0, 10.0, true), el("勾選框", "自動更新", 10.0, 60.0, true)];
|
||
let (_, _, spots) = number(&els, false);
|
||
assert_eq!(spots.named("4").unwrap(), 1);
|
||
assert_eq!(spots.named("「C」").unwrap(), 3, "C is not AC");
|
||
assert!(spots.named("Save").unwrap_err().contains("沒有叫「Save」"));
|
||
// A near miss names what is there.
|
||
let near = spots.named("自動").unwrap_err();
|
||
assert!(near.contains("名字相近") && near.contains("[4]「自動更新」"), "{near}");
|
||
let mut v = serde_json::json!({"actions": [{"action": "click", "name": "4"}, {"action": "click", "ref": "2"}, {"action": "typing", "value": "x"}]});
|
||
names_to_numbers(&mut v, Some(&spots)).unwrap();
|
||
assert_eq!(v["actions"][0]["ref"], "1");
|
||
assert_eq!(v["actions"][1]["ref"], "2");
|
||
assert!(v["actions"][2]["ref"].is_null());
|
||
let mut one = serde_json::json!({"name": "自動更新"});
|
||
names_to_numbers(&mut one, Some(&spots)).unwrap();
|
||
assert_eq!(one["ref"], "4");
|
||
// A name with x: 0, y: 0 beside it: the name.
|
||
let mut both = serde_json::json!({"name": "AC", "x": 0.0, "y": 0.0});
|
||
names_to_numbers(&mut both, Some(&spots)).unwrap();
|
||
assert_eq!(both["ref"], "2");
|
||
}
|
||
|
||
#[test]
|
||
fn numbers_stay_with_their_controls() {
|
||
let mut m = Numbering::default();
|
||
let keys = |extra: bool| {
|
||
let mut v = vec![];
|
||
if extra {
|
||
v.push(el("選單", "說明", 5.0, 5.0, true));
|
||
}
|
||
for (i, k) in ["7", "8", "9", "*"].iter().enumerate() {
|
||
v.push(el("按鈕", k, 100.0 + 50.0 * i as f64, 200.0, true));
|
||
}
|
||
v
|
||
};
|
||
let (first, _, _) = number_keeping(&keys(false), false, &mut m);
|
||
let n_of = |list: &[(usize, Element)], name: &str| list.iter().find(|(_, e)| e.name == name).map(|(n, _)| *n).unwrap();
|
||
assert_eq!(n_of(&first, "9"), 3);
|
||
// A menu appears first in reading order: the keys keep their numbers, the menu is new.
|
||
let (second, lines, spots) = number_keeping(&keys(true), false, &mut m);
|
||
assert_eq!(n_of(&second, "9"), 3);
|
||
assert_eq!(n_of(&second, "說明"), 5);
|
||
assert_eq!(spots.get("3").unwrap().1, "按鈕「9」");
|
||
assert_eq!(lines[0], "[5] 選單「說明」");
|
||
// Moved a little (a list scrolled): still the same control.
|
||
let mut moved = keys(true);
|
||
for e in moved.iter_mut() {
|
||
e.y += 30.0;
|
||
}
|
||
let (third, _, _) = number_keeping(&moved, false, &mut m);
|
||
assert_eq!(n_of(&third, "9"), 3);
|
||
// Another window altogether: from 1 again.
|
||
let other: Vec<Element> = keys(false).into_iter().map(|e| Element { window: "Other".into(), ..e }).collect();
|
||
let (fourth, _, _) = number_keeping(&other, false, &mut m);
|
||
assert_eq!(n_of(&fourth, "7"), 1);
|
||
}
|
||
|
||
#[test]
|
||
fn controls_are_numbered_in_reading_order() {
|
||
let els = vec![el("按鈕", "下", 10.0, 300.0, true), el("連結", "Profile", 20.0, 100.0, true), el("文字", "標題", 0.0, 50.0, false), el("連結", "Profile", 20.0, 100.0, true)];
|
||
let (numbered, lines, spots) = number(&els, false);
|
||
// Reading order, duplicates once, plain text left out when there is a picture.
|
||
assert_eq!(lines, vec!["[1] 連結「Profile」", "[2] 按鈕「下」"]);
|
||
assert_eq!(numbered.len(), 2);
|
||
assert_eq!(spots.get("1").unwrap(), ((40.0, 110.0), "連結「Profile」".to_string()));
|
||
assert_eq!(spots.get("e2").unwrap().0, (30.0, 310.0));
|
||
assert!(spots.get("3").is_err() && spots.get("ok").is_err());
|
||
// Without a picture the text is listed too (not numbered).
|
||
let (_, lines, _) = number(&els, true);
|
||
assert_eq!(lines, vec![" 文字「標題」", "[1] 連結「Profile」", "[2] 按鈕「下」"]);
|
||
}
|
||
|
||
#[test]
|
||
fn numbers_are_drawn_on_the_picture() {
|
||
let mut img = image::RgbImage::from_pixel(200, 100, image::Rgb([255, 255, 255]));
|
||
let els = vec![el("按鈕", "OK", 100.0, 60.0, true)];
|
||
let (numbered, _, _) = number(&els, false);
|
||
// Screen twice the picture's size: the box lands at (50, 30)–(70, 40).
|
||
draw(&mut img, &numbered, (2.0, 2.0));
|
||
assert_eq!(img.get_pixel(50, 35).0, COLOURS[1]);
|
||
assert_eq!(img.get_pixel(60, 30).0, COLOURS[1]);
|
||
// The label sits above the box with white digits in it.
|
||
let label_has_white = (16..30).any(|y| (50..62).any(|x| img.get_pixel(x, y).0 == [255, 255, 255]));
|
||
let label_has_colour = (16..30).any(|y| (50..62).any(|x| img.get_pixel(x, y).0 == COLOURS[1]));
|
||
assert!(label_has_white && label_has_colour);
|
||
}
|
||
|
||
#[test]
|
||
fn a_still_screen_has_the_same_fingerprint() {
|
||
let a = vec![el("按鈕", "OK", 1.0, 2.0, true)];
|
||
let mut b = a.clone();
|
||
assert_eq!(fingerprint(&a), fingerprint(&b));
|
||
b[0].name = "Cancel".into();
|
||
assert_ne!(fingerprint(&a), fingerprint(&b));
|
||
let img = image::RgbImage::from_pixel(64, 40, image::Rgb([10, 20, 30]));
|
||
let mut img2 = img.clone();
|
||
img2.put_pixel(0, 0, image::Rgb([12, 20, 30]));
|
||
assert_eq!(picture_fingerprint(&img), picture_fingerprint(&img2));
|
||
}
|
||
}
|