Accessibility write (macOS): press by name/spot and set the focused field's value, pixel path as fallback

The bench's precision failures (quick model clicking the wrong number, Retina
scaling, covered windows) all come from acting on pixels. A plain left click
now asks the frontmost app's accessibility tree first — by name, or by the
numbered spot — and typing writes AXValue into the focused field once,
keyboard events only when accessibility can't reach. Any app with a thin tree
behaves exactly as before: every AX answer that isn't a clear yes falls
through to CGEvent.

macOS code type-checked with --target x86_64-apple-darwin; gate: build on the
macos-arm64 runner + bench tasks T2/T5 (clicking by name) on a Mac.
Linux/Windows: stubs, pixel path unchanged.

src/axwrite.rs

@@ -0,0 +1,307 @@
+// Accessibility WRITE (macOS): press a control the accessibility tree names, set a focused
+// text field's value. A control pressed through accessibility does not care about Retina
+// scaling, a window covering part of the screen, or the picture's coordinates being one
+// pixel off — the failures the bench kept catching when the model clicked by number.
+// Everything here answers an error and the caller falls back to the pixel path (CGEvent),
+// so an app with a thin accessibility tree behaves exactly as before.
+//
+// Behaviour spec: press = AXPress on the element (AXPick for menu items); a pressable element
+// is a leaf-ish role in PRESSABLE_ROLES whose AXTitle/AXDescription/AXValue matches the name
+// (trim, case-insensitive; exact first, then contains); point press = the deepest pressable
+// element whose AXPosition/AXSize frame contains the point, else the deepest element under
+// the point. Typing = AXValue on the app's AXFocusedUIElement when its role is editable.
+
+/// Roles a name click may press (anything else under the name is not a click target).
+pub const PRESSABLE_ROLES: [&str; 9] = [
+    "AXButton", "AXCheckBox", "AXRadioButton", "AXMenuItem", "AXPopUpButton", "AXMenuButton", "AXMenu", "AXTabButton", "AXLink",
+];
+/// Roles text may be set on.
+pub const EDITABLE_ROLES: [&str; 5] = ["AXTextField", "AXTextArea", "AXComboBox", "AXSearchField", "AXPasswordTextField"];
+
+/// Name matching: exact (trimmed, case-insensitive) beats contains.
+pub fn name_match(hay: &str, want: &str) -> u8 {
+    let (h, w) = (hay.trim().to_lowercase(), want.trim().to_lowercase());
+    if h.is_empty() || w.is_empty() {
+        return 0;
+    }
+    if h == w {
+        2
+    } else if h.contains(&w) {
+        1
+    } else {
+        0
+    }
+}
+
+/// The text a field ends up holding (accessibility set replaces the whole value).
+pub fn merged_text(old: &str, new: &str, append: bool) -> String {
+    if append {
+        format!("{old}{new}")
+    } else {
+        new.to_string()
+    }
+}
+
+#[cfg(target_os = "macos")]
+mod mac {
+    use super::{name_match, merged_text, EDITABLE_ROLES, PRESSABLE_ROLES};
+    use std::ffi::{c_char, c_void, CString};
+    use std::os::raw::c_void as raw_void;
+
+    type CFTypeRef = *const raw_void;
+
+    #[link(name = "ApplicationServices", kind = "framework")]
+    extern "C" {
+        fn AXUIElementCreateApplication(pid: i32) -> CFTypeRef;
+        fn AXUIElementCopyAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: *mut CFTypeRef) -> i32;
+        fn AXUIElementSetAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: CFTypeRef) -> i32;
+        fn AXUIElementPerformAction(el: CFTypeRef, action: CFTypeRef) -> i32;
+        fn AXUIElementSetMessagingTimeout(el: CFTypeRef, secs: f32) -> i32;
+        fn AXValueGetValue(value: CFTypeRef, kind: u32, out: *mut c_void) -> bool;
+    }
+
+    #[link(name = "CoreFoundation", kind = "framework")]
+    extern "C" {
+        fn CFStringCreateWithCString(alloc: CFTypeRef, s: *const c_char, enc: u32) -> CFTypeRef;
+        fn CFRelease(x: CFTypeRef);
+        fn CFRetain(x: CFTypeRef) -> CFTypeRef;
+        fn CFGetTypeID(x: CFTypeRef) -> usize;
+        fn CFStringGetTypeID() -> usize;
+        fn CFArrayGetTypeID() -> usize;
+        fn CFArrayGetCount(a: CFTypeRef) -> isize;
+        fn CFArrayGetValueAtIndex(a: CFTypeRef, i: isize) -> CFTypeRef;
+        fn CFStringGetLength(s: CFTypeRef) -> isize;
+        fn CFStringGetMaximumSizeForEncoding(len: isize, enc: u32) -> isize;
+        fn CFStringGetCString(s: CFTypeRef, buf: *mut c_char, size: isize, enc: u32) -> bool;
+    }
+
+    const UTF8: u32 = 0x0800_0100;
+    const AX_VALUE_POINT: u32 = 1;
+    const MAX_NODES: usize = 20_000;
+    const MAX_DEPTH: usize = 80;
+    /// An app that answers slowly (Electron building its tree) must not hang the step.
+    const TIMEOUT_SECS: f32 = 2.0;
+
+    struct Cf(CFTypeRef);
+    impl Drop for Cf {
+        fn drop(&mut self) {
+            if !self.0.is_null() {
+                unsafe { CFRelease(self.0) }
+            }
+        }
+    }
+
+    fn cfstr(s: &str) -> Cf {
+        let c = CString::new(s).unwrap_or_default();
+        Cf(unsafe { CFStringCreateWithCString(std::ptr::null(), c.as_ptr(), UTF8) })
+    }
+
+    fn string_of(v: CFTypeRef) -> Option<String> {
+        unsafe {
+            if v.is_null() || CFGetTypeID(v) != CFStringGetTypeID() {
+                return None;
+            }
+            let size = CFStringGetMaximumSizeForEncoding(CFStringGetLength(v), UTF8) + 1;
+            let mut buf = vec![0u8; size.max(1) as usize];
+            if !CFStringGetCString(v, buf.as_mut_ptr() as *mut c_char, size, UTF8) {
+                return None;
+            }
+            let end = buf.iter().position(|b| *b == 0).unwrap_or(buf.len());
+            Some(String::from_utf8_lossy(&buf[..end]).into_owned())
+        }
+    }
+
+    fn attr(el: CFTypeRef, name: &str) -> Option<Cf> {
+        let key = cfstr(name);
+        let mut out: CFTypeRef = std::ptr::null();
+        let err = unsafe { AXUIElementCopyAttributeValue(el, key.0, &mut out) };
+        (err == 0 && !out.is_null()).then(|| Cf(out))
+    }
+
+    fn attr_string(el: CFTypeRef, name: &str) -> Option<String> {
+        attr(el, name).and_then(|v| string_of(v.0))
+    }
+
+    fn children(el: CFTypeRef) -> (Option<Cf>, Vec<CFTypeRef>) {
+        let Some(arr) = attr(el, "AXChildren") else { return (None, vec![]) };
+        unsafe {
+            if CFGetTypeID(arr.0) != CFArrayGetTypeID() {
+                return (Some(arr), vec![]);
+            }
+            let n = CFArrayGetCount(arr.0);
+            let ptr = arr.0;
+            (Some(arr), (0..n).map(|i| CFArrayGetValueAtIndex(ptr, i)).collect())
+        }
+    }
+
+    /// (x, y, w, h) in screen points, from AXPosition + AXSize.
+    fn frame(el: CFTypeRef) -> Option<(f64, f64, f64, f64)> {
+        let pos = attr(el, "AXPosition")?;
+        let size = attr(el, "AXSize")?;
+        let mut p = [0f64; 2];
+        let mut s = [0f64; 2];
+        unsafe {
+            if !AXValueGetValue(pos.0, AX_VALUE_POINT, p.as_mut_ptr() as *mut c_void) {
+                return None;
+            }
+            if !AXValueGetValue(size.0, 2, s.as_mut_ptr() as *mut c_void) {
+                return None;
+            }
+        }
+        Some((p[0], p[1], s[0], s[1]))
+    }
+
+    fn pressable(el: CFTypeRef) -> bool {
+        attr_string(el, "AXRole").is_some_and(|r| PRESSABLE_ROLES.contains(&r.as_str()))
+    }
+
+    /// Best (score, depth, element) pressable whose title/description/value matches `want`.
+    fn find_named(el: CFTypeRef, want: &str, best: &mut (u8, usize, CFTypeRef), seen: &mut usize, depth: usize) {
+        if *seen >= MAX_NODES || depth > MAX_DEPTH {
+            return;
+        }
+        *seen += 1;
+        if pressable(el) {
+            let score = ["AXTitle", "AXDescription", "AXValue"].iter().map(|a| attr_string(el, a).map(|s| name_match(&s, want)).unwrap_or(0)).max().unwrap_or(0);
+            // Deeper wins ties: a button inside a group is the thing, not the group.
+            if score > best.0 || (score == best.0 && score > 0 && depth > best.1) {
+                *best = (score, depth, el);
+            }
+        }
+        let (_keep, kids) = children(el);
+        for k in kids {
+            find_named(k, want, best, seen, depth + 1);
+        }
+    }
+
+    /// Deepest pressable whose frame holds the point; deepest of any role as a second choice.
+    fn find_at(el: CFTypeRef, x: f64, y: f64, best: &mut (usize, CFTypeRef, bool), seen: &mut usize, depth: usize) {
+        if *seen >= MAX_NODES || depth > MAX_DEPTH {
+            return;
+        }
+        *seen += 1;
+        if let Some((fx, fy, fw, fh)) = frame(el) {
+            if fw > 0.0 && fh > 0.0 && x >= fx && x < fx + fw && y >= fy && y < fy + fh {
+                if pressable(el) && depth >= best.0 {
+                    *best = (depth, el, true);
+                } else if !best.2 && depth >= best.0 {
+                    *best = (depth, el, false);
+                }
+            }
+        }
+        let (_keep, kids) = children(el);
+        for k in kids {
+            find_at(k, x, y, best, seen, depth + 1);
+        }
+    }
+
+    fn press(el: CFTypeRef, role: &str) -> Result<(), String> {
+        // Menu items answer AXPick, buttons AXPress; try the fitting one, then the other.
+        let acts = if role == "AXMenuItem" { ["AXPick", "AXPress"] } else { ["AXPress", "AXPick"] };
+        for a in acts {
+            let key = cfstr(a);
+            if unsafe { AXUIElementPerformAction(el, key.0) } == 0 {
+                return Ok(());
+            }
+        }
+        Err("這個控件不接受 AXPress".into())
+    }
+
+    fn app(pid: i32) -> Option<Cf> {
+        let el = unsafe { AXUIElementCreateApplication(pid) };
+        (!el.is_null()).then(|| {
+            unsafe { AXUIElementSetMessagingTimeout(el, TIMEOUT_SECS) };
+            Cf(el)
+        })
+    }
+
+    /// The element's role (kept out of the walk: the caller reports it).
+    fn role_of(el: CFTypeRef) -> String {
+        attr_string(el, "AXRole").unwrap_or_default()
+    }
+
+    /// Press the control named `want` in the app `pid`. Returns the matched title.
+    pub fn press_named(pid: i32, want: &str) -> Result<String, String> {
+        let app = app(pid).ok_or("這個 App 沒有輔助使用介面")?;
+        let mut best = (0u8, 0usize, std::ptr::null());
+        let mut seen = 0;
+        find_named(app.0, want, &mut best, &mut seen, 0);
+        if best.2.is_null() {
+            return Err(format!("輔助使用樹裡沒有可點的「{want}」"));
+        }
+        let label = ["AXTitle", "AXDescription", "AXValue"].iter().filter_map(|a| attr_string(best.2, a)).find(|s| !s.trim().is_empty()).unwrap_or_else(|| want.to_string());
+        // Keep the element alive through the press: CFRetain, press, then release.
+        unsafe { CFRetain(best.2) };
+        let r = press(best.2, &role_of(best.2));
+        unsafe { CFRelease(best.2) };
+        r.map(|_| label)
+    }
+
+    /// Press what sits under a screen point, if accessibility can name it.
+    pub fn press_at(pid: i32, x: f64, y: f64) -> Result<String, String> {
+        let app = app(pid).ok_or("這個 App 沒有輔助使用介面")?;
+        let mut best = (0usize, std::ptr::null(), false);
+        let mut seen = 0;
+        find_at(app.0, x, y, &mut best, &mut seen, 0);
+        if best.1.is_null() {
+            return Err("那個位置不在任何控件上".into());
+        }
+        if !best.2 {
+            return Err("那個位置的控件不接受點按".into());
+        }
+        let label = ["AXTitle", "AXDescription", "AXValue"].iter().filter_map(|a| attr_string(best.1, a)).find(|s| !s.trim().is_empty()).map(|s| format!("「{s}」")).unwrap_or_else(|| "控件".into());
+        unsafe { CFRetain(best.1) };
+        let r = press(best.1, &role_of(best.1));
+        unsafe { CFRelease(best.1) };
+        r.map(|_| label)
+    }
+
+    /// Set the focused field's text (typing without keystrokes). False = caller types keys.
+    pub fn set_focused_text(pid: i32, text: &str, append: bool) -> bool {
+        let Some(app) = app(pid) else { return false };
+        let Some(focused) = attr(app.0, "AXFocusedUIElement") else { return false };
+        if !EDITABLE_ROLES.contains(&role_of(focused.0).as_str()) {
+            return false;
+        }
+        let old = attr_string(focused.0, "AXValue").unwrap_or_default();
+        let merged = merged_text(&old, text, append);
+        let v = cfstr(&merged);
+        unsafe { AXUIElementSetAttributeValue(focused.0, cfstr("AXValue").0, v.0) == 0 }
+    }
+}
+
+#[cfg(not(target_os = "macos"))]
+mod mac {
+    /// No accessibility write path off macOS: callers use the pixel path.
+    pub fn press_named(_pid: i32, _want: &str) -> Result<String, String> {
+        Err("只在 macOS 有輔助使用點按".into())
+    }
+    pub fn press_at(_pid: i32, _x: f64, _y: f64) -> Result<String, String> {
+        Err("只在 macOS 有輔助使用點按".into())
+    }
+    pub fn set_focused_text(_pid: i32, _text: &str, _append: bool) -> bool {
+        false
+    }
+}
+
+pub use mac::{press_at, press_named, set_focused_text};
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn exact_names_beat_partials() {
+        assert_eq!(name_match(" 自動更新 ", "自動更新"), 2);
+        assert_eq!(name_match("自動更新（每小時）", "自動更新"), 1);
+        assert_eq!(name_match("OK", "ok"), 2);
+        assert_eq!(name_match("Cancel", "OK"), 0);
+        assert_eq!(name_match("", "OK"), 0);
+    }
+
+    #[test]
+    fn appending_keeps_what_the_field_held() {
+        assert_eq!(merged_text("Hi", " there", true), "Hi there");
+        assert_eq!(merged_text("Hi", "there", false), "there");
+    }
+}

src/computer.rs

@@ -855,11 +855,39 @@ fn keys_go_where_looked(name: &str, v: &Value) -> Result<(), String> {
     }
 }
 
+/// Accessibility-first click (macOS): the app's own tree answers name clicks and numbered
+/// spots exactly — no Retina scaling, no covered-window misses (bench: the quick model's
+/// wrong-number clicks). None = fall through to the pixel path.
+fn ax_click(v: &Value, scale: (f64, f64)) -> Option<String> {
+    if !cfg!(target_os = "macos") {
+        return None;
+    }
+    let (_, pid) = mac::front_app()?;
+    if let Some(nm) = v["name"].as_str().map(str::trim).filter(|s| !s.is_empty()) {
+        return crate::axwrite::press_named(pid, nm).ok().map(|w| format!("已點「{w}」（輔助使用，精準點按，不靠座標）。"));
+    }
+    let (x, y, _) = place_of(v, scale).ok()??;
+    crate::axwrite::press_at(pid, x, y).ok().map(|w| format!("已點 {w}（輔助使用，精準點按，不靠座標）。"))
+}
+
+/// Typing through accessibility: one AXValue write on the focused field. False = type keys.
+fn ax_type(text: &str, append: bool) -> bool {
+    match mac::front_app() {
+        Some((_, pid)) => crate::axwrite::set_focused_text(pid, text, append),
+        None => false,
+    }
+}
+
 pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result<String, String> {
     keys_go_where_looked(name, v)?;
     let button = v["button"].as_str().unwrap_or("left");
     match name {
         "computer_use_click" => {
+            if v["count"].as_u64().unwrap_or(1) <= 1 && button == "left" && flags_of(v) == 0 {
+                if let Some(done) = ax_click(v, scale) {
+                    return Ok(done);
+                }
+            }
             let (x, y, at) = place_of(v, scale)?.ok_or("請給 x、y（截圖上的位置）或 ref（截圖上的編號）")?;
             let count = v["count"].as_u64().unwrap_or(1).clamp(1, 3) as i64;
             mac::click(x, y, button, count, flags_of(v));
@@ -922,7 +950,9 @@ pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result<String, String> {
                     std::thread::sleep(std::time::Duration::from_millis(50));
                 }
             }
-            if !text.is_empty() {
+            // Straight into the focused field when accessibility can reach it (one write,
+            // no keystrokes); keystrokes when it can't.
+            if !text.is_empty() && !ax_type(text, v["append"].as_bool().unwrap_or(false)) {
                 mac::type_text(text);
             }
             if submit {

src/main.rs

@@ -3,6 +3,7 @@ mod app;
 mod appctl;
 mod attach;
 mod axread;
+mod axwrite;
 mod avatar;
 mod bidi;
 mod botbrowser;