// Accessibility WRITE (macOS): press a control the accessibility tree names, set a focused // text field's value. A control pressed through accessibility does not care about Retina // scaling, a window covering part of the screen, or the picture's coordinates being one // pixel off — the failures the bench kept catching when the model clicked by number. // Everything here answers an error and the caller falls back to the pixel path (CGEvent), // so an app with a thin accessibility tree behaves exactly as before. // // Behaviour spec: press = AXPress on the element (AXPick for menu items); a pressable element // is a leaf-ish role in PRESSABLE_ROLES whose AXTitle/AXDescription/AXValue matches the name // (trim, case-insensitive; exact first, then contains); point press = the deepest pressable // element whose AXPosition/AXSize frame contains the point, else the deepest element under // the point. Typing = AXValue on the app's AXFocusedUIElement when its role is editable. /// Roles a name click may press (anything else under the name is not a click target). pub const PRESSABLE_ROLES: [&str; 9] = [ "AXButton", "AXCheckBox", "AXRadioButton", "AXMenuItem", "AXPopUpButton", "AXMenuButton", "AXMenu", "AXTabButton", "AXLink", ]; /// Roles text may be set on. pub const EDITABLE_ROLES: [&str; 5] = ["AXTextField", "AXTextArea", "AXComboBox", "AXSearchField", "AXPasswordTextField"]; /// Name matching: exact (trimmed, case-insensitive) beats contains. pub fn name_match(hay: &str, want: &str) -> u8 { let (h, w) = (hay.trim().to_lowercase(), want.trim().to_lowercase()); if h.is_empty() || w.is_empty() { return 0; } if h == w { 2 } else if h.contains(&w) { 1 } else { 0 } } /// The text a field ends up holding (accessibility set replaces the whole value). pub fn merged_text(old: &str, new: &str, append: bool) -> String { if append { format!("{old}{new}") } else { new.to_string() } } #[cfg(target_os = "macos")] mod mac { use super::{name_match, merged_text, EDITABLE_ROLES, PRESSABLE_ROLES}; use std::ffi::{c_char, c_void, CString}; use std::os::raw::c_void as raw_void; type CFTypeRef = *const raw_void; #[link(name = "ApplicationServices", kind = "framework")] extern "C" { fn AXUIElementCreateApplication(pid: i32) -> CFTypeRef; fn AXUIElementCopyAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: *mut CFTypeRef) -> i32; fn AXUIElementSetAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: CFTypeRef) -> i32; fn AXUIElementPerformAction(el: CFTypeRef, action: CFTypeRef) -> i32; fn AXUIElementSetMessagingTimeout(el: CFTypeRef, secs: f32) -> i32; fn AXValueGetValue(value: CFTypeRef, kind: u32, out: *mut c_void) -> bool; } #[link(name = "CoreFoundation", kind = "framework")] extern "C" { fn CFStringCreateWithCString(alloc: CFTypeRef, s: *const c_char, enc: u32) -> CFTypeRef; fn CFRelease(x: CFTypeRef); fn CFRetain(x: CFTypeRef) -> CFTypeRef; fn CFGetTypeID(x: CFTypeRef) -> usize; fn CFStringGetTypeID() -> usize; fn CFArrayGetTypeID() -> usize; fn CFArrayGetCount(a: CFTypeRef) -> isize; fn CFArrayGetValueAtIndex(a: CFTypeRef, i: isize) -> CFTypeRef; fn CFStringGetLength(s: CFTypeRef) -> isize; fn CFStringGetMaximumSizeForEncoding(len: isize, enc: u32) -> isize; fn CFStringGetCString(s: CFTypeRef, buf: *mut c_char, size: isize, enc: u32) -> bool; } const UTF8: u32 = 0x0800_0100; const AX_VALUE_POINT: u32 = 1; const MAX_NODES: usize = 20_000; const MAX_DEPTH: usize = 80; /// An app that answers slowly (Electron building its tree) must not hang the step. const TIMEOUT_SECS: f32 = 2.0; struct Cf(CFTypeRef); impl Drop for Cf { fn drop(&mut self) { if !self.0.is_null() { unsafe { CFRelease(self.0) } } } } fn cfstr(s: &str) -> Cf { let c = CString::new(s).unwrap_or_default(); Cf(unsafe { CFStringCreateWithCString(std::ptr::null(), c.as_ptr(), UTF8) }) } fn string_of(v: CFTypeRef) -> Option { unsafe { if v.is_null() || CFGetTypeID(v) != CFStringGetTypeID() { return None; } let size = CFStringGetMaximumSizeForEncoding(CFStringGetLength(v), UTF8) + 1; let mut buf = vec![0u8; size.max(1) as usize]; if !CFStringGetCString(v, buf.as_mut_ptr() as *mut c_char, size, UTF8) { return None; } let end = buf.iter().position(|b| *b == 0).unwrap_or(buf.len()); Some(String::from_utf8_lossy(&buf[..end]).into_owned()) } } fn attr(el: CFTypeRef, name: &str) -> Option { let key = cfstr(name); let mut out: CFTypeRef = std::ptr::null(); let err = unsafe { AXUIElementCopyAttributeValue(el, key.0, &mut out) }; (err == 0 && !out.is_null()).then(|| Cf(out)) } fn attr_string(el: CFTypeRef, name: &str) -> Option { attr(el, name).and_then(|v| string_of(v.0)) } fn children(el: CFTypeRef) -> (Option, Vec) { let Some(arr) = attr(el, "AXChildren") else { return (None, vec![]) }; unsafe { if CFGetTypeID(arr.0) != CFArrayGetTypeID() { return (Some(arr), vec![]); } let n = CFArrayGetCount(arr.0); let ptr = arr.0; (Some(arr), (0..n).map(|i| CFArrayGetValueAtIndex(ptr, i)).collect()) } } /// (x, y, w, h) in screen points, from AXPosition + AXSize. fn frame(el: CFTypeRef) -> Option<(f64, f64, f64, f64)> { let pos = attr(el, "AXPosition")?; let size = attr(el, "AXSize")?; let mut p = [0f64; 2]; let mut s = [0f64; 2]; unsafe { if !AXValueGetValue(pos.0, AX_VALUE_POINT, p.as_mut_ptr() as *mut c_void) { return None; } if !AXValueGetValue(size.0, 2, s.as_mut_ptr() as *mut c_void) { return None; } } Some((p[0], p[1], s[0], s[1])) } fn pressable(el: CFTypeRef) -> bool { attr_string(el, "AXRole").is_some_and(|r| PRESSABLE_ROLES.contains(&r.as_str())) } /// Best (score, depth, element) pressable whose title/description/value matches `want`. fn find_named(el: CFTypeRef, want: &str, best: &mut (u8, usize, CFTypeRef), seen: &mut usize, depth: usize) { if *seen >= MAX_NODES || depth > MAX_DEPTH { return; } *seen += 1; if pressable(el) { let score = ["AXTitle", "AXDescription", "AXValue"].iter().map(|a| attr_string(el, a).map(|s| name_match(&s, want)).unwrap_or(0)).max().unwrap_or(0); // Deeper wins ties: a button inside a group is the thing, not the group. if score > best.0 || (score == best.0 && score > 0 && depth > best.1) { *best = (score, depth, el); } } let (_keep, kids) = children(el); for k in kids { find_named(k, want, best, seen, depth + 1); } } /// Deepest pressable whose frame holds the point; deepest of any role as a second choice. fn find_at(el: CFTypeRef, x: f64, y: f64, best: &mut (usize, CFTypeRef, bool), seen: &mut usize, depth: usize) { if *seen >= MAX_NODES || depth > MAX_DEPTH { return; } *seen += 1; if let Some((fx, fy, fw, fh)) = frame(el) { if fw > 0.0 && fh > 0.0 && x >= fx && x < fx + fw && y >= fy && y < fy + fh { if pressable(el) && depth >= best.0 { *best = (depth, el, true); } else if !best.2 && depth >= best.0 { *best = (depth, el, false); } } } let (_keep, kids) = children(el); for k in kids { find_at(k, x, y, best, seen, depth + 1); } } fn press(el: CFTypeRef, role: &str) -> Result<(), String> { // Menu items answer AXPick, buttons AXPress; try the fitting one, then the other. let acts = if role == "AXMenuItem" { ["AXPick", "AXPress"] } else { ["AXPress", "AXPick"] }; for a in acts { let key = cfstr(a); if unsafe { AXUIElementPerformAction(el, key.0) } == 0 { return Ok(()); } } Err("這個控件不接受 AXPress".into()) } fn app(pid: i32) -> Option { let el = unsafe { AXUIElementCreateApplication(pid) }; (!el.is_null()).then(|| { unsafe { AXUIElementSetMessagingTimeout(el, TIMEOUT_SECS) }; Cf(el) }) } /// The element's role (kept out of the walk: the caller reports it). fn role_of(el: CFTypeRef) -> String { attr_string(el, "AXRole").unwrap_or_default() } /// Press the control named `want` in the app `pid`. Returns the matched title. pub fn press_named(pid: i32, want: &str) -> Result { let app = app(pid).ok_or("這個 App 沒有輔助使用介面")?; let mut best = (0u8, 0usize, std::ptr::null()); let mut seen = 0; find_named(app.0, want, &mut best, &mut seen, 0); if best.2.is_null() { return Err(format!("輔助使用樹裡沒有可點的「{want}」")); } let label = ["AXTitle", "AXDescription", "AXValue"].iter().filter_map(|a| attr_string(best.2, a)).find(|s| !s.trim().is_empty()).unwrap_or_else(|| want.to_string()); // Keep the element alive through the press: CFRetain, press, then release. unsafe { CFRetain(best.2) }; let r = press(best.2, &role_of(best.2)); unsafe { CFRelease(best.2) }; r.map(|_| label) } /// Press what sits under a screen point, if accessibility can name it. pub fn press_at(pid: i32, x: f64, y: f64) -> Result { let app = app(pid).ok_or("這個 App 沒有輔助使用介面")?; let mut best = (0usize, std::ptr::null(), false); let mut seen = 0; find_at(app.0, x, y, &mut best, &mut seen, 0); if best.1.is_null() { return Err("那個位置不在任何控件上".into()); } if !best.2 { return Err("那個位置的控件不接受點按".into()); } let label = ["AXTitle", "AXDescription", "AXValue"].iter().filter_map(|a| attr_string(best.1, a)).find(|s| !s.trim().is_empty()).map(|s| format!("「{s}」")).unwrap_or_else(|| "控件".into()); unsafe { CFRetain(best.1) }; let r = press(best.1, &role_of(best.1)); unsafe { CFRelease(best.1) }; r.map(|_| label) } /// Set the focused field's text (typing without keystrokes). False = caller types keys. pub fn set_focused_text(pid: i32, text: &str, append: bool) -> bool { let Some(app) = app(pid) else { return false }; let Some(focused) = attr(app.0, "AXFocusedUIElement") else { return false }; if !EDITABLE_ROLES.contains(&role_of(focused.0).as_str()) { return false; } let old = attr_string(focused.0, "AXValue").unwrap_or_default(); let merged = merged_text(&old, text, append); let v = cfstr(&merged); unsafe { AXUIElementSetAttributeValue(focused.0, cfstr("AXValue").0, v.0) == 0 } } } #[cfg(not(target_os = "macos"))] mod mac { /// No accessibility write path off macOS: callers use the pixel path. pub fn press_named(_pid: i32, _want: &str) -> Result { Err("只在 macOS 有輔助使用點按".into()) } pub fn press_at(_pid: i32, _x: f64, _y: f64) -> Result { Err("只在 macOS 有輔助使用點按".into()) } pub fn set_focused_text(_pid: i32, _text: &str, _append: bool) -> bool { false } } pub use mac::{press_at, press_named, set_focused_text}; #[cfg(test)] mod tests { use super::*; #[test] fn exact_names_beat_partials() { assert_eq!(name_match(" 自動更新 ", "自動更新"), 2); assert_eq!(name_match("自動更新(每小時)", "自動更新"), 1); assert_eq!(name_match("OK", "ok"), 2); assert_eq!(name_match("Cancel", "OK"), 0); assert_eq!(name_match("", "OK"), 0); } #[test] fn appending_keeps_what_the_field_held() { assert_eq!(merged_text("Hi", " there", true), "Hi there"); assert_eq!(merged_text("Hi", "there", false), "there"); } }