AnyBot/src/axwrite.rs

// Accessibility WRITE (macOS): press a control the accessibility tree names, set a focused
// text field's value. A control pressed through accessibility does not care about Retina
// scaling, a window covering part of the screen, or the picture's coordinates being one
// pixel off — the failures the bench kept catching when the model clicked by number.
// Everything here answers an error and the caller falls back to the pixel path (CGEvent),
// so an app with a thin accessibility tree behaves exactly as before.
//
// Behaviour spec: press = AXPress on the element (AXPick for menu items); a pressable element
// is a leaf-ish role in PRESSABLE_ROLES whose AXTitle/AXDescription/AXValue matches the name
// (trim, case-insensitive; exact first, then contains); point press = the deepest pressable
// element whose AXPosition/AXSize frame contains the point, else the deepest element under
// the point. Typing = AXValue on the app's AXFocusedUIElement when its role is editable.

/// Roles a name click may press (anything else under the name is not a click target).
pub const PRESSABLE_ROLES: [&str; 9] = [
    "AXButton", "AXCheckBox", "AXRadioButton", "AXMenuItem", "AXPopUpButton", "AXMenuButton", "AXMenu", "AXTabButton", "AXLink",
];
/// Roles text may be set on.
pub const EDITABLE_ROLES: [&str; 5] = ["AXTextField", "AXTextArea", "AXComboBox", "AXSearchField", "AXPasswordTextField"];

/// Name matching: exact (trimmed, case-insensitive) beats contains.
pub fn name_match(hay: &str, want: &str) -> u8 {
    let (h, w) = (hay.trim().to_lowercase(), want.trim().to_lowercase());
    if h.is_empty() || w.is_empty() {
        return 0;
    }
    if h == w {
        2
    } else if h.contains(&w) {
        1
    } else {
        0
    }
}

/// The text a field ends up holding (accessibility set replaces the whole value).
pub fn merged_text(old: &str, new: &str, append: bool) -> String {
    if append {
        format!("{old}{new}")
    } else {
        new.to_string()
    }
}

#[cfg(target_os = "macos")]
mod mac {
    use super::{name_match, merged_text, EDITABLE_ROLES, PRESSABLE_ROLES};
    use std::ffi::{c_char, c_void, CString};
    use std::os::raw::c_void as raw_void;

    type CFTypeRef = *const raw_void;

    #[link(name = "ApplicationServices", kind = "framework")]
    extern "C" {
        fn AXUIElementCreateApplication(pid: i32) -> CFTypeRef;
        fn AXUIElementCopyAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: *mut CFTypeRef) -> i32;
        fn AXUIElementSetAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: CFTypeRef) -> i32;
        fn AXUIElementPerformAction(el: CFTypeRef, action: CFTypeRef) -> i32;
        fn AXUIElementSetMessagingTimeout(el: CFTypeRef, secs: f32) -> i32;
        fn AXValueGetValue(value: CFTypeRef, kind: u32, out: *mut c_void) -> bool;
    }

    #[link(name = "CoreFoundation", kind = "framework")]
    extern "C" {
        fn CFStringCreateWithCString(alloc: CFTypeRef, s: *const c_char, enc: u32) -> CFTypeRef;
        fn CFRelease(x: CFTypeRef);
        fn CFRetain(x: CFTypeRef) -> CFTypeRef;
        fn CFGetTypeID(x: CFTypeRef) -> usize;
        fn CFStringGetTypeID() -> usize;
        fn CFArrayGetTypeID() -> usize;
        fn CFArrayGetCount(a: CFTypeRef) -> isize;
        fn CFArrayGetValueAtIndex(a: CFTypeRef, i: isize) -> CFTypeRef;
        fn CFStringGetLength(s: CFTypeRef) -> isize;
        fn CFStringGetMaximumSizeForEncoding(len: isize, enc: u32) -> isize;
        fn CFStringGetCString(s: CFTypeRef, buf: *mut c_char, size: isize, enc: u32) -> bool;
    }

    const UTF8: u32 = 0x0800_0100;
    const AX_VALUE_POINT: u32 = 1;
    const MAX_NODES: usize = 20_000;
    const MAX_DEPTH: usize = 80;
    /// An app that answers slowly (Electron building its tree) must not hang the step.
    const TIMEOUT_SECS: f32 = 2.0;

    struct Cf(CFTypeRef);
    impl Drop for Cf {
        fn drop(&mut self) {
            if !self.0.is_null() {
                unsafe { CFRelease(self.0) }
            }
        }
    }

    fn cfstr(s: &str) -> Cf {
        let c = CString::new(s).unwrap_or_default();
        Cf(unsafe { CFStringCreateWithCString(std::ptr::null(), c.as_ptr(), UTF8) })
    }

    fn string_of(v: CFTypeRef) -> Option<String> {
        unsafe {
            if v.is_null() || CFGetTypeID(v) != CFStringGetTypeID() {
                return None;
            }
            let size = CFStringGetMaximumSizeForEncoding(CFStringGetLength(v), UTF8) + 1;
            let mut buf = vec![0u8; size.max(1) as usize];
            if !CFStringGetCString(v, buf.as_mut_ptr() as *mut c_char, size, UTF8) {
                return None;
            }
            let end = buf.iter().position(|b| *b == 0).unwrap_or(buf.len());
            Some(String::from_utf8_lossy(&buf[..end]).into_owned())
        }
    }

    fn attr(el: CFTypeRef, name: &str) -> Option<Cf> {
        let key = cfstr(name);
        let mut out: CFTypeRef = std::ptr::null();
        let err = unsafe { AXUIElementCopyAttributeValue(el, key.0, &mut out) };
        (err == 0 && !out.is_null()).then(|| Cf(out))
    }

    fn attr_string(el: CFTypeRef, name: &str) -> Option<String> {
        attr(el, name).and_then(|v| string_of(v.0))
    }

    fn children(el: CFTypeRef) -> (Option<Cf>, Vec<CFTypeRef>) {
        let Some(arr) = attr(el, "AXChildren") else { return (None, vec![]) };
        unsafe {
            if CFGetTypeID(arr.0) != CFArrayGetTypeID() {
                return (Some(arr), vec![]);
            }
            let n = CFArrayGetCount(arr.0);
            let ptr = arr.0;
            (Some(arr), (0..n).map(|i| CFArrayGetValueAtIndex(ptr, i)).collect())
        }
    }

    /// (x, y, w, h) in screen points, from AXPosition + AXSize.
    fn frame(el: CFTypeRef) -> Option<(f64, f64, f64, f64)> {
        let pos = attr(el, "AXPosition")?;
        let size = attr(el, "AXSize")?;
        let mut p = [0f64; 2];
        let mut s = [0f64; 2];
        unsafe {
            if !AXValueGetValue(pos.0, AX_VALUE_POINT, p.as_mut_ptr() as *mut c_void) {
                return None;
            }
            if !AXValueGetValue(size.0, 2, s.as_mut_ptr() as *mut c_void) {
                return None;
            }
        }
        Some((p[0], p[1], s[0], s[1]))
    }

    fn pressable(el: CFTypeRef) -> bool {
        attr_string(el, "AXRole").is_some_and(|r| PRESSABLE_ROLES.contains(&r.as_str()))
    }

    /// Best (score, depth, element) pressable whose title/description/value matches `want`.
    fn find_named(el: CFTypeRef, want: &str, best: &mut (u8, usize, CFTypeRef), seen: &mut usize, depth: usize) {
        if *seen >= MAX_NODES || depth > MAX_DEPTH {
            return;
        }
        *seen += 1;
        if pressable(el) {
            let score = ["AXTitle", "AXDescription", "AXValue"].iter().map(|a| attr_string(el, a).map(|s| name_match(&s, want)).unwrap_or(0)).max().unwrap_or(0);
            // Deeper wins ties: a button inside a group is the thing, not the group.
            if score > best.0 || (score == best.0 && score > 0 && depth > best.1) {
                *best = (score, depth, el);
            }
        }
        let (_keep, kids) = children(el);
        for k in kids {
            find_named(k, want, best, seen, depth + 1);
        }
    }

    /// Deepest pressable whose frame holds the point; deepest of any role as a second choice.
    fn find_at(el: CFTypeRef, x: f64, y: f64, best: &mut (usize, CFTypeRef, bool), seen: &mut usize, depth: usize) {
        if *seen >= MAX_NODES || depth > MAX_DEPTH {
            return;
        }
        *seen += 1;
        if let Some((fx, fy, fw, fh)) = frame(el) {
            if fw > 0.0 && fh > 0.0 && x >= fx && x < fx + fw && y >= fy && y < fy + fh {
                if pressable(el) && depth >= best.0 {
                    *best = (depth, el, true);
                } else if !best.2 && depth >= best.0 {
                    *best = (depth, el, false);
                }
            }
        }
        let (_keep, kids) = children(el);
        for k in kids {
            find_at(k, x, y, best, seen, depth + 1);
        }
    }

    fn press(el: CFTypeRef, role: &str) -> Result<(), String> {
        // Menu items answer AXPick, buttons AXPress; try the fitting one, then the other.
        let acts = if role == "AXMenuItem" { ["AXPick", "AXPress"] } else { ["AXPress", "AXPick"] };
        for a in acts {
            let key = cfstr(a);
            if unsafe { AXUIElementPerformAction(el, key.0) } == 0 {
                return Ok(());
            }
        }
        Err("這個控件不接受 AXPress".into())
    }

    fn app(pid: i32) -> Option<Cf> {
        let el = unsafe { AXUIElementCreateApplication(pid) };
        (!el.is_null()).then(|| {
            unsafe { AXUIElementSetMessagingTimeout(el, TIMEOUT_SECS) };
            Cf(el)
        })
    }

    /// The element's role (kept out of the walk: the caller reports it).
    fn role_of(el: CFTypeRef) -> String {
        attr_string(el, "AXRole").unwrap_or_default()
    }

    /// Press the control named `want` in the app `pid`. Returns the matched title.
    pub fn press_named(pid: i32, want: &str) -> Result<String, String> {
        let app = app(pid).ok_or("這個 App 沒有輔助使用介面")?;
        let mut best = (0u8, 0usize, std::ptr::null());
        let mut seen = 0;
        find_named(app.0, want, &mut best, &mut seen, 0);
        if best.2.is_null() {
            return Err(format!("輔助使用樹裡沒有可點的「{want}」"));
        }
        let label = ["AXTitle", "AXDescription", "AXValue"].iter().filter_map(|a| attr_string(best.2, a)).find(|s| !s.trim().is_empty()).unwrap_or_else(|| want.to_string());
        // Keep the element alive through the press: CFRetain, press, then release.
        unsafe { CFRetain(best.2) };
        let r = press(best.2, &role_of(best.2));
        unsafe { CFRelease(best.2) };
        r.map(|_| label)
    }

    /// Press what sits under a screen point, if accessibility can name it.
    pub fn press_at(pid: i32, x: f64, y: f64) -> Result<String, String> {
        let app = app(pid).ok_or("這個 App 沒有輔助使用介面")?;
        let mut best = (0usize, std::ptr::null(), false);
        let mut seen = 0;
        find_at(app.0, x, y, &mut best, &mut seen, 0);
        if best.1.is_null() {
            return Err("那個位置不在任何控件上".into());
        }
        if !best.2 {
            return Err("那個位置的控件不接受點按".into());
        }
        let label = ["AXTitle", "AXDescription", "AXValue"].iter().filter_map(|a| attr_string(best.1, a)).find(|s| !s.trim().is_empty()).map(|s| format!("「{s}」")).unwrap_or_else(|| "控件".into());
        unsafe { CFRetain(best.1) };
        let r = press(best.1, &role_of(best.1));
        unsafe { CFRelease(best.1) };
        r.map(|_| label)
    }

    /// Set the focused field's text (typing without keystrokes). False = caller types keys.
    pub fn set_focused_text(pid: i32, text: &str, append: bool) -> bool {
        let Some(app) = app(pid) else { return false };
        let Some(focused) = attr(app.0, "AXFocusedUIElement") else { return false };
        if !EDITABLE_ROLES.contains(&role_of(focused.0).as_str()) {
            return false;
        }
        let old = attr_string(focused.0, "AXValue").unwrap_or_default();
        let merged = merged_text(&old, text, append);
        let v = cfstr(&merged);
        unsafe { AXUIElementSetAttributeValue(focused.0, cfstr("AXValue").0, v.0) == 0 }
    }
}

#[cfg(not(target_os = "macos"))]
mod mac {
    /// No accessibility write path off macOS: callers use the pixel path.
    pub fn press_named(_pid: i32, _want: &str) -> Result<String, String> {
        Err("只在 macOS 有輔助使用點按".into())
    }
    pub fn press_at(_pid: i32, _x: f64, _y: f64) -> Result<String, String> {
        Err("只在 macOS 有輔助使用點按".into())
    }
    pub fn set_focused_text(_pid: i32, _text: &str, _append: bool) -> bool {
        false
    }
}

pub use mac::{press_at, press_named, set_focused_text};

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn exact_names_beat_partials() {
        assert_eq!(name_match(" 自動更新 ", "自動更新"), 2);
        assert_eq!(name_match("自動更新（每小時）", "自動更新"), 1);
        assert_eq!(name_match("OK", "ok"), 2);
        assert_eq!(name_match("Cancel", "OK"), 0);
        assert_eq!(name_match("", "OK"), 0);
    }

    #[test]
    fn appending_keeps_what_the_field_held() {
        assert_eq!(merged_text("Hi", " there", true), "Hi there");
        assert_eq!(merged_text("Hi", "there", false), "there");
    }
}