From d9b6330b524903e857f9084c2ac1cb36e0753120 Mon Sep 17 00:00:00 2001 From: AnyBot Dev Date: Mon, 9 Nov 2026 16:53:12 +0800 Subject: [PATCH] Accessibility write (macOS): press by name/spot and set the focused field's value, pixel path as fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bench's precision failures (quick model clicking the wrong number, Retina scaling, covered windows) all come from acting on pixels. A plain left click now asks the frontmost app's accessibility tree first — by name, or by the numbered spot — and typing writes AXValue into the focused field once, keyboard events only when accessibility can't reach. Any app with a thin tree behaves exactly as before: every AX answer that isn't a clear yes falls through to CGEvent. macOS code type-checked with --target x86_64-apple-darwin; gate: build on the macos-arm64 runner + bench tasks T2/T5 (clicking by name) on a Mac. Linux/Windows: stubs, pixel path unchanged. --- src/axwrite.rs | 307 ++++++++++++++++++++++++++++++++++++++++++++++++ src/computer.rs | 32 ++++- src/main.rs | 1 + 3 files changed, 339 insertions(+), 1 deletion(-) create mode 100644 src/axwrite.rs diff --git a/src/axwrite.rs b/src/axwrite.rs new file mode 100644 index 0000000..aa8e4cc --- /dev/null +++ b/src/axwrite.rs @@ -0,0 +1,307 @@ +// Accessibility WRITE (macOS): press a control the accessibility tree names, set a focused +// text field's value. A control pressed through accessibility does not care about Retina +// scaling, a window covering part of the screen, or the picture's coordinates being one +// pixel off — the failures the bench kept catching when the model clicked by number. +// Everything here answers an error and the caller falls back to the pixel path (CGEvent), +// so an app with a thin accessibility tree behaves exactly as before. +// +// Behaviour spec: press = AXPress on the element (AXPick for menu items); a pressable element +// is a leaf-ish role in PRESSABLE_ROLES whose AXTitle/AXDescription/AXValue matches the name +// (trim, case-insensitive; exact first, then contains); point press = the deepest pressable +// element whose AXPosition/AXSize frame contains the point, else the deepest element under +// the point. Typing = AXValue on the app's AXFocusedUIElement when its role is editable. + +/// Roles a name click may press (anything else under the name is not a click target). +pub const PRESSABLE_ROLES: [&str; 9] = [ + "AXButton", "AXCheckBox", "AXRadioButton", "AXMenuItem", "AXPopUpButton", "AXMenuButton", "AXMenu", "AXTabButton", "AXLink", +]; +/// Roles text may be set on. +pub const EDITABLE_ROLES: [&str; 5] = ["AXTextField", "AXTextArea", "AXComboBox", "AXSearchField", "AXPasswordTextField"]; + +/// Name matching: exact (trimmed, case-insensitive) beats contains. +pub fn name_match(hay: &str, want: &str) -> u8 { + let (h, w) = (hay.trim().to_lowercase(), want.trim().to_lowercase()); + if h.is_empty() || w.is_empty() { + return 0; + } + if h == w { + 2 + } else if h.contains(&w) { + 1 + } else { + 0 + } +} + +/// The text a field ends up holding (accessibility set replaces the whole value). +pub fn merged_text(old: &str, new: &str, append: bool) -> String { + if append { + format!("{old}{new}") + } else { + new.to_string() + } +} + +#[cfg(target_os = "macos")] +mod mac { + use super::{name_match, merged_text, EDITABLE_ROLES, PRESSABLE_ROLES}; + use std::ffi::{c_char, c_void, CString}; + use std::os::raw::c_void as raw_void; + + type CFTypeRef = *const raw_void; + + #[link(name = "ApplicationServices", kind = "framework")] + extern "C" { + fn AXUIElementCreateApplication(pid: i32) -> CFTypeRef; + fn AXUIElementCopyAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: *mut CFTypeRef) -> i32; + fn AXUIElementSetAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: CFTypeRef) -> i32; + fn AXUIElementPerformAction(el: CFTypeRef, action: CFTypeRef) -> i32; + fn AXUIElementSetMessagingTimeout(el: CFTypeRef, secs: f32) -> i32; + fn AXValueGetValue(value: CFTypeRef, kind: u32, out: *mut c_void) -> bool; + } + + #[link(name = "CoreFoundation", kind = "framework")] + extern "C" { + fn CFStringCreateWithCString(alloc: CFTypeRef, s: *const c_char, enc: u32) -> CFTypeRef; + fn CFRelease(x: CFTypeRef); + fn CFRetain(x: CFTypeRef) -> CFTypeRef; + fn CFGetTypeID(x: CFTypeRef) -> usize; + fn CFStringGetTypeID() -> usize; + fn CFArrayGetTypeID() -> usize; + fn CFArrayGetCount(a: CFTypeRef) -> isize; + fn CFArrayGetValueAtIndex(a: CFTypeRef, i: isize) -> CFTypeRef; + fn CFStringGetLength(s: CFTypeRef) -> isize; + fn CFStringGetMaximumSizeForEncoding(len: isize, enc: u32) -> isize; + fn CFStringGetCString(s: CFTypeRef, buf: *mut c_char, size: isize, enc: u32) -> bool; + } + + const UTF8: u32 = 0x0800_0100; + const AX_VALUE_POINT: u32 = 1; + const MAX_NODES: usize = 20_000; + const MAX_DEPTH: usize = 80; + /// An app that answers slowly (Electron building its tree) must not hang the step. + const TIMEOUT_SECS: f32 = 2.0; + + struct Cf(CFTypeRef); + impl Drop for Cf { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { CFRelease(self.0) } + } + } + } + + fn cfstr(s: &str) -> Cf { + let c = CString::new(s).unwrap_or_default(); + Cf(unsafe { CFStringCreateWithCString(std::ptr::null(), c.as_ptr(), UTF8) }) + } + + fn string_of(v: CFTypeRef) -> Option { + unsafe { + if v.is_null() || CFGetTypeID(v) != CFStringGetTypeID() { + return None; + } + let size = CFStringGetMaximumSizeForEncoding(CFStringGetLength(v), UTF8) + 1; + let mut buf = vec![0u8; size.max(1) as usize]; + if !CFStringGetCString(v, buf.as_mut_ptr() as *mut c_char, size, UTF8) { + return None; + } + let end = buf.iter().position(|b| *b == 0).unwrap_or(buf.len()); + Some(String::from_utf8_lossy(&buf[..end]).into_owned()) + } + } + + fn attr(el: CFTypeRef, name: &str) -> Option { + let key = cfstr(name); + let mut out: CFTypeRef = std::ptr::null(); + let err = unsafe { AXUIElementCopyAttributeValue(el, key.0, &mut out) }; + (err == 0 && !out.is_null()).then(|| Cf(out)) + } + + fn attr_string(el: CFTypeRef, name: &str) -> Option { + attr(el, name).and_then(|v| string_of(v.0)) + } + + fn children(el: CFTypeRef) -> (Option, Vec) { + let Some(arr) = attr(el, "AXChildren") else { return (None, vec![]) }; + unsafe { + if CFGetTypeID(arr.0) != CFArrayGetTypeID() { + return (Some(arr), vec![]); + } + let n = CFArrayGetCount(arr.0); + let ptr = arr.0; + (Some(arr), (0..n).map(|i| CFArrayGetValueAtIndex(ptr, i)).collect()) + } + } + + /// (x, y, w, h) in screen points, from AXPosition + AXSize. + fn frame(el: CFTypeRef) -> Option<(f64, f64, f64, f64)> { + let pos = attr(el, "AXPosition")?; + let size = attr(el, "AXSize")?; + let mut p = [0f64; 2]; + let mut s = [0f64; 2]; + unsafe { + if !AXValueGetValue(pos.0, AX_VALUE_POINT, p.as_mut_ptr() as *mut c_void) { + return None; + } + if !AXValueGetValue(size.0, 2, s.as_mut_ptr() as *mut c_void) { + return None; + } + } + Some((p[0], p[1], s[0], s[1])) + } + + fn pressable(el: CFTypeRef) -> bool { + attr_string(el, "AXRole").is_some_and(|r| PRESSABLE_ROLES.contains(&r.as_str())) + } + + /// Best (score, depth, element) pressable whose title/description/value matches `want`. + fn find_named(el: CFTypeRef, want: &str, best: &mut (u8, usize, CFTypeRef), seen: &mut usize, depth: usize) { + if *seen >= MAX_NODES || depth > MAX_DEPTH { + return; + } + *seen += 1; + if pressable(el) { + let score = ["AXTitle", "AXDescription", "AXValue"].iter().map(|a| attr_string(el, a).map(|s| name_match(&s, want)).unwrap_or(0)).max().unwrap_or(0); + // Deeper wins ties: a button inside a group is the thing, not the group. + if score > best.0 || (score == best.0 && score > 0 && depth > best.1) { + *best = (score, depth, el); + } + } + let (_keep, kids) = children(el); + for k in kids { + find_named(k, want, best, seen, depth + 1); + } + } + + /// Deepest pressable whose frame holds the point; deepest of any role as a second choice. + fn find_at(el: CFTypeRef, x: f64, y: f64, best: &mut (usize, CFTypeRef, bool), seen: &mut usize, depth: usize) { + if *seen >= MAX_NODES || depth > MAX_DEPTH { + return; + } + *seen += 1; + if let Some((fx, fy, fw, fh)) = frame(el) { + if fw > 0.0 && fh > 0.0 && x >= fx && x < fx + fw && y >= fy && y < fy + fh { + if pressable(el) && depth >= best.0 { + *best = (depth, el, true); + } else if !best.2 && depth >= best.0 { + *best = (depth, el, false); + } + } + } + let (_keep, kids) = children(el); + for k in kids { + find_at(k, x, y, best, seen, depth + 1); + } + } + + fn press(el: CFTypeRef, role: &str) -> Result<(), String> { + // Menu items answer AXPick, buttons AXPress; try the fitting one, then the other. + let acts = if role == "AXMenuItem" { ["AXPick", "AXPress"] } else { ["AXPress", "AXPick"] }; + for a in acts { + let key = cfstr(a); + if unsafe { AXUIElementPerformAction(el, key.0) } == 0 { + return Ok(()); + } + } + Err("這個控件不接受 AXPress".into()) + } + + fn app(pid: i32) -> Option { + let el = unsafe { AXUIElementCreateApplication(pid) }; + (!el.is_null()).then(|| { + unsafe { AXUIElementSetMessagingTimeout(el, TIMEOUT_SECS) }; + Cf(el) + }) + } + + /// The element's role (kept out of the walk: the caller reports it). + fn role_of(el: CFTypeRef) -> String { + attr_string(el, "AXRole").unwrap_or_default() + } + + /// Press the control named `want` in the app `pid`. Returns the matched title. + pub fn press_named(pid: i32, want: &str) -> Result { + let app = app(pid).ok_or("這個 App 沒有輔助使用介面")?; + let mut best = (0u8, 0usize, std::ptr::null()); + let mut seen = 0; + find_named(app.0, want, &mut best, &mut seen, 0); + if best.2.is_null() { + return Err(format!("輔助使用樹裡沒有可點的「{want}」")); + } + let label = ["AXTitle", "AXDescription", "AXValue"].iter().filter_map(|a| attr_string(best.2, a)).find(|s| !s.trim().is_empty()).unwrap_or_else(|| want.to_string()); + // Keep the element alive through the press: CFRetain, press, then release. + unsafe { CFRetain(best.2) }; + let r = press(best.2, &role_of(best.2)); + unsafe { CFRelease(best.2) }; + r.map(|_| label) + } + + /// Press what sits under a screen point, if accessibility can name it. + pub fn press_at(pid: i32, x: f64, y: f64) -> Result { + let app = app(pid).ok_or("這個 App 沒有輔助使用介面")?; + let mut best = (0usize, std::ptr::null(), false); + let mut seen = 0; + find_at(app.0, x, y, &mut best, &mut seen, 0); + if best.1.is_null() { + return Err("那個位置不在任何控件上".into()); + } + if !best.2 { + return Err("那個位置的控件不接受點按".into()); + } + let label = ["AXTitle", "AXDescription", "AXValue"].iter().filter_map(|a| attr_string(best.1, a)).find(|s| !s.trim().is_empty()).map(|s| format!("「{s}」")).unwrap_or_else(|| "控件".into()); + unsafe { CFRetain(best.1) }; + let r = press(best.1, &role_of(best.1)); + unsafe { CFRelease(best.1) }; + r.map(|_| label) + } + + /// Set the focused field's text (typing without keystrokes). False = caller types keys. + pub fn set_focused_text(pid: i32, text: &str, append: bool) -> bool { + let Some(app) = app(pid) else { return false }; + let Some(focused) = attr(app.0, "AXFocusedUIElement") else { return false }; + if !EDITABLE_ROLES.contains(&role_of(focused.0).as_str()) { + return false; + } + let old = attr_string(focused.0, "AXValue").unwrap_or_default(); + let merged = merged_text(&old, text, append); + let v = cfstr(&merged); + unsafe { AXUIElementSetAttributeValue(focused.0, cfstr("AXValue").0, v.0) == 0 } + } +} + +#[cfg(not(target_os = "macos"))] +mod mac { + /// No accessibility write path off macOS: callers use the pixel path. + pub fn press_named(_pid: i32, _want: &str) -> Result { + Err("只在 macOS 有輔助使用點按".into()) + } + pub fn press_at(_pid: i32, _x: f64, _y: f64) -> Result { + Err("只在 macOS 有輔助使用點按".into()) + } + pub fn set_focused_text(_pid: i32, _text: &str, _append: bool) -> bool { + false + } +} + +pub use mac::{press_at, press_named, set_focused_text}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn exact_names_beat_partials() { + assert_eq!(name_match(" 自動更新 ", "自動更新"), 2); + assert_eq!(name_match("自動更新（每小時）", "自動更新"), 1); + assert_eq!(name_match("OK", "ok"), 2); + assert_eq!(name_match("Cancel", "OK"), 0); + assert_eq!(name_match("", "OK"), 0); + } + + #[test] + fn appending_keeps_what_the_field_held() { + assert_eq!(merged_text("Hi", " there", true), "Hi there"); + assert_eq!(merged_text("Hi", "there", false), "there"); + } +} diff --git a/src/computer.rs b/src/computer.rs index 34bb1fd..4d91485 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -855,11 +855,39 @@ fn keys_go_where_looked(name: &str, v: &Value) -> Result<(), String> { } } +/// Accessibility-first click (macOS): the app's own tree answers name clicks and numbered +/// spots exactly — no Retina scaling, no covered-window misses (bench: the quick model's +/// wrong-number clicks). None = fall through to the pixel path. +fn ax_click(v: &Value, scale: (f64, f64)) -> Option { + if !cfg!(target_os = "macos") { + return None; + } + let (_, pid) = mac::front_app()?; + if let Some(nm) = v["name"].as_str().map(str::trim).filter(|s| !s.is_empty()) { + return crate::axwrite::press_named(pid, nm).ok().map(|w| format!("已點「{w}」（輔助使用，精準點按，不靠座標）。")); + } + let (x, y, _) = place_of(v, scale).ok()??; + crate::axwrite::press_at(pid, x, y).ok().map(|w| format!("已點 {w}（輔助使用，精準點按，不靠座標）。")) +} + +/// Typing through accessibility: one AXValue write on the focused field. False = type keys. +fn ax_type(text: &str, append: bool) -> bool { + match mac::front_app() { + Some((_, pid)) => crate::axwrite::set_focused_text(pid, text, append), + None => false, + } +} + pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result { keys_go_where_looked(name, v)?; let button = v["button"].as_str().unwrap_or("left"); match name { "computer_use_click" => { + if v["count"].as_u64().unwrap_or(1) <= 1 && button == "left" && flags_of(v) == 0 { + if let Some(done) = ax_click(v, scale) { + return Ok(done); + } + } let (x, y, at) = place_of(v, scale)?.ok_or("請給 x、y（截圖上的位置）或 ref（截圖上的編號）")?; let count = v["count"].as_u64().unwrap_or(1).clamp(1, 3) as i64; mac::click(x, y, button, count, flags_of(v)); @@ -922,7 +950,9 @@ pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result { std::thread::sleep(std::time::Duration::from_millis(50)); } } - if !text.is_empty() { + // Straight into the focused field when accessibility can reach it (one write, + // no keystrokes); keystrokes when it can't. + if !text.is_empty() && !ax_type(text, v["append"].as_bool().unwrap_or(false)) { mac::type_text(text); } if submit { diff --git a/src/main.rs b/src/main.rs index 4ada87b..08fae62 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ mod app; mod appctl; mod attach; mod axread; +mod axwrite; mod avatar; mod bidi; mod botbrowser;