Merge c2/ax-write: accessibility-first click/typing on macOS (C2)
This commit is contained in:
daniel wang
commit
649c1fcf50
|
|
@ -0,0 +1,307 @@
|
|||
// Accessibility WRITE (macOS): press a control the accessibility tree names, set a focused
|
||||
// text field's value. A control pressed through accessibility does not care about Retina
|
||||
// scaling, a window covering part of the screen, or the picture's coordinates being one
|
||||
// pixel off — the failures the bench kept catching when the model clicked by number.
|
||||
// Everything here answers an error and the caller falls back to the pixel path (CGEvent),
|
||||
// so an app with a thin accessibility tree behaves exactly as before.
|
||||
//
|
||||
// Behaviour spec: press = AXPress on the element (AXPick for menu items); a pressable element
|
||||
// is a leaf-ish role in PRESSABLE_ROLES whose AXTitle/AXDescription/AXValue matches the name
|
||||
// (trim, case-insensitive; exact first, then contains); point press = the deepest pressable
|
||||
// element whose AXPosition/AXSize frame contains the point, else the deepest element under
|
||||
// the point. Typing = AXValue on the app's AXFocusedUIElement when its role is editable.
|
||||
|
||||
/// Roles a name click may press (anything else under the name is not a click target).
|
||||
pub const PRESSABLE_ROLES: [&str; 9] = [
|
||||
"AXButton", "AXCheckBox", "AXRadioButton", "AXMenuItem", "AXPopUpButton", "AXMenuButton", "AXMenu", "AXTabButton", "AXLink",
|
||||
];
|
||||
/// Roles text may be set on.
|
||||
pub const EDITABLE_ROLES: [&str; 5] = ["AXTextField", "AXTextArea", "AXComboBox", "AXSearchField", "AXPasswordTextField"];
|
||||
|
||||
/// Name matching: exact (trimmed, case-insensitive) beats contains.
|
||||
pub fn name_match(hay: &str, want: &str) -> u8 {
|
||||
let (h, w) = (hay.trim().to_lowercase(), want.trim().to_lowercase());
|
||||
if h.is_empty() || w.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
if h == w {
|
||||
2
|
||||
} else if h.contains(&w) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
/// The text a field ends up holding (accessibility set replaces the whole value).
|
||||
pub fn merged_text(old: &str, new: &str, append: bool) -> String {
|
||||
if append {
|
||||
format!("{old}{new}")
|
||||
} else {
|
||||
new.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod mac {
|
||||
use super::{name_match, merged_text, EDITABLE_ROLES, PRESSABLE_ROLES};
|
||||
use std::ffi::{c_char, c_void, CString};
|
||||
use std::os::raw::c_void as raw_void;
|
||||
|
||||
type CFTypeRef = *const raw_void;
|
||||
|
||||
#[link(name = "ApplicationServices", kind = "framework")]
|
||||
extern "C" {
|
||||
fn AXUIElementCreateApplication(pid: i32) -> CFTypeRef;
|
||||
fn AXUIElementCopyAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: *mut CFTypeRef) -> i32;
|
||||
fn AXUIElementSetAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: CFTypeRef) -> i32;
|
||||
fn AXUIElementPerformAction(el: CFTypeRef, action: CFTypeRef) -> i32;
|
||||
fn AXUIElementSetMessagingTimeout(el: CFTypeRef, secs: f32) -> i32;
|
||||
fn AXValueGetValue(value: CFTypeRef, kind: u32, out: *mut c_void) -> bool;
|
||||
}
|
||||
|
||||
#[link(name = "CoreFoundation", kind = "framework")]
|
||||
extern "C" {
|
||||
fn CFStringCreateWithCString(alloc: CFTypeRef, s: *const c_char, enc: u32) -> CFTypeRef;
|
||||
fn CFRelease(x: CFTypeRef);
|
||||
fn CFRetain(x: CFTypeRef) -> CFTypeRef;
|
||||
fn CFGetTypeID(x: CFTypeRef) -> usize;
|
||||
fn CFStringGetTypeID() -> usize;
|
||||
fn CFArrayGetTypeID() -> usize;
|
||||
fn CFArrayGetCount(a: CFTypeRef) -> isize;
|
||||
fn CFArrayGetValueAtIndex(a: CFTypeRef, i: isize) -> CFTypeRef;
|
||||
fn CFStringGetLength(s: CFTypeRef) -> isize;
|
||||
fn CFStringGetMaximumSizeForEncoding(len: isize, enc: u32) -> isize;
|
||||
fn CFStringGetCString(s: CFTypeRef, buf: *mut c_char, size: isize, enc: u32) -> bool;
|
||||
}
|
||||
|
||||
const UTF8: u32 = 0x0800_0100;
|
||||
const AX_VALUE_POINT: u32 = 1;
|
||||
const MAX_NODES: usize = 20_000;
|
||||
const MAX_DEPTH: usize = 80;
|
||||
/// An app that answers slowly (Electron building its tree) must not hang the step.
|
||||
const TIMEOUT_SECS: f32 = 2.0;
|
||||
|
||||
struct Cf(CFTypeRef);
|
||||
impl Drop for Cf {
|
||||
fn drop(&mut self) {
|
||||
if !self.0.is_null() {
|
||||
unsafe { CFRelease(self.0) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cfstr(s: &str) -> Cf {
|
||||
let c = CString::new(s).unwrap_or_default();
|
||||
Cf(unsafe { CFStringCreateWithCString(std::ptr::null(), c.as_ptr(), UTF8) })
|
||||
}
|
||||
|
||||
fn string_of(v: CFTypeRef) -> Option<String> {
|
||||
unsafe {
|
||||
if v.is_null() || CFGetTypeID(v) != CFStringGetTypeID() {
|
||||
return None;
|
||||
}
|
||||
let size = CFStringGetMaximumSizeForEncoding(CFStringGetLength(v), UTF8) + 1;
|
||||
let mut buf = vec![0u8; size.max(1) as usize];
|
||||
if !CFStringGetCString(v, buf.as_mut_ptr() as *mut c_char, size, UTF8) {
|
||||
return None;
|
||||
}
|
||||
let end = buf.iter().position(|b| *b == 0).unwrap_or(buf.len());
|
||||
Some(String::from_utf8_lossy(&buf[..end]).into_owned())
|
||||
}
|
||||
}
|
||||
|
||||
fn attr(el: CFTypeRef, name: &str) -> Option<Cf> {
|
||||
let key = cfstr(name);
|
||||
let mut out: CFTypeRef = std::ptr::null();
|
||||
let err = unsafe { AXUIElementCopyAttributeValue(el, key.0, &mut out) };
|
||||
(err == 0 && !out.is_null()).then(|| Cf(out))
|
||||
}
|
||||
|
||||
fn attr_string(el: CFTypeRef, name: &str) -> Option<String> {
|
||||
attr(el, name).and_then(|v| string_of(v.0))
|
||||
}
|
||||
|
||||
fn children(el: CFTypeRef) -> (Option<Cf>, Vec<CFTypeRef>) {
|
||||
let Some(arr) = attr(el, "AXChildren") else { return (None, vec![]) };
|
||||
unsafe {
|
||||
if CFGetTypeID(arr.0) != CFArrayGetTypeID() {
|
||||
return (Some(arr), vec![]);
|
||||
}
|
||||
let n = CFArrayGetCount(arr.0);
|
||||
let ptr = arr.0;
|
||||
(Some(arr), (0..n).map(|i| CFArrayGetValueAtIndex(ptr, i)).collect())
|
||||
}
|
||||
}
|
||||
|
||||
/// (x, y, w, h) in screen points, from AXPosition + AXSize.
|
||||
fn frame(el: CFTypeRef) -> Option<(f64, f64, f64, f64)> {
|
||||
let pos = attr(el, "AXPosition")?;
|
||||
let size = attr(el, "AXSize")?;
|
||||
let mut p = [0f64; 2];
|
||||
let mut s = [0f64; 2];
|
||||
unsafe {
|
||||
if !AXValueGetValue(pos.0, AX_VALUE_POINT, p.as_mut_ptr() as *mut c_void) {
|
||||
return None;
|
||||
}
|
||||
if !AXValueGetValue(size.0, 2, s.as_mut_ptr() as *mut c_void) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
Some((p[0], p[1], s[0], s[1]))
|
||||
}
|
||||
|
||||
fn pressable(el: CFTypeRef) -> bool {
|
||||
attr_string(el, "AXRole").is_some_and(|r| PRESSABLE_ROLES.contains(&r.as_str()))
|
||||
}
|
||||
|
||||
/// Best (score, depth, element) pressable whose title/description/value matches `want`.
|
||||
fn find_named(el: CFTypeRef, want: &str, best: &mut (u8, usize, CFTypeRef), seen: &mut usize, depth: usize) {
|
||||
if *seen >= MAX_NODES || depth > MAX_DEPTH {
|
||||
return;
|
||||
}
|
||||
*seen += 1;
|
||||
if pressable(el) {
|
||||
let score = ["AXTitle", "AXDescription", "AXValue"].iter().map(|a| attr_string(el, a).map(|s| name_match(&s, want)).unwrap_or(0)).max().unwrap_or(0);
|
||||
// Deeper wins ties: a button inside a group is the thing, not the group.
|
||||
if score > best.0 || (score == best.0 && score > 0 && depth > best.1) {
|
||||
*best = (score, depth, el);
|
||||
}
|
||||
}
|
||||
let (_keep, kids) = children(el);
|
||||
for k in kids {
|
||||
find_named(k, want, best, seen, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
/// Deepest pressable whose frame holds the point; deepest of any role as a second choice.
|
||||
fn find_at(el: CFTypeRef, x: f64, y: f64, best: &mut (usize, CFTypeRef, bool), seen: &mut usize, depth: usize) {
|
||||
if *seen >= MAX_NODES || depth > MAX_DEPTH {
|
||||
return;
|
||||
}
|
||||
*seen += 1;
|
||||
if let Some((fx, fy, fw, fh)) = frame(el) {
|
||||
if fw > 0.0 && fh > 0.0 && x >= fx && x < fx + fw && y >= fy && y < fy + fh {
|
||||
if pressable(el) && depth >= best.0 {
|
||||
*best = (depth, el, true);
|
||||
} else if !best.2 && depth >= best.0 {
|
||||
*best = (depth, el, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
let (_keep, kids) = children(el);
|
||||
for k in kids {
|
||||
find_at(k, x, y, best, seen, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
fn press(el: CFTypeRef, role: &str) -> Result<(), String> {
|
||||
// Menu items answer AXPick, buttons AXPress; try the fitting one, then the other.
|
||||
let acts = if role == "AXMenuItem" { ["AXPick", "AXPress"] } else { ["AXPress", "AXPick"] };
|
||||
for a in acts {
|
||||
let key = cfstr(a);
|
||||
if unsafe { AXUIElementPerformAction(el, key.0) } == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
Err("這個控件不接受 AXPress".into())
|
||||
}
|
||||
|
||||
fn app(pid: i32) -> Option<Cf> {
|
||||
let el = unsafe { AXUIElementCreateApplication(pid) };
|
||||
(!el.is_null()).then(|| {
|
||||
unsafe { AXUIElementSetMessagingTimeout(el, TIMEOUT_SECS) };
|
||||
Cf(el)
|
||||
})
|
||||
}
|
||||
|
||||
/// The element's role (kept out of the walk: the caller reports it).
|
||||
fn role_of(el: CFTypeRef) -> String {
|
||||
attr_string(el, "AXRole").unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Press the control named `want` in the app `pid`. Returns the matched title.
|
||||
pub fn press_named(pid: i32, want: &str) -> Result<String, String> {
|
||||
let app = app(pid).ok_or("這個 App 沒有輔助使用介面")?;
|
||||
let mut best = (0u8, 0usize, std::ptr::null());
|
||||
let mut seen = 0;
|
||||
find_named(app.0, want, &mut best, &mut seen, 0);
|
||||
if best.2.is_null() {
|
||||
return Err(format!("輔助使用樹裡沒有可點的「{want}」"));
|
||||
}
|
||||
let label = ["AXTitle", "AXDescription", "AXValue"].iter().filter_map(|a| attr_string(best.2, a)).find(|s| !s.trim().is_empty()).unwrap_or_else(|| want.to_string());
|
||||
// Keep the element alive through the press: CFRetain, press, then release.
|
||||
unsafe { CFRetain(best.2) };
|
||||
let r = press(best.2, &role_of(best.2));
|
||||
unsafe { CFRelease(best.2) };
|
||||
r.map(|_| label)
|
||||
}
|
||||
|
||||
/// Press what sits under a screen point, if accessibility can name it.
|
||||
pub fn press_at(pid: i32, x: f64, y: f64) -> Result<String, String> {
|
||||
let app = app(pid).ok_or("這個 App 沒有輔助使用介面")?;
|
||||
let mut best = (0usize, std::ptr::null(), false);
|
||||
let mut seen = 0;
|
||||
find_at(app.0, x, y, &mut best, &mut seen, 0);
|
||||
if best.1.is_null() {
|
||||
return Err("那個位置不在任何控件上".into());
|
||||
}
|
||||
if !best.2 {
|
||||
return Err("那個位置的控件不接受點按".into());
|
||||
}
|
||||
let label = ["AXTitle", "AXDescription", "AXValue"].iter().filter_map(|a| attr_string(best.1, a)).find(|s| !s.trim().is_empty()).map(|s| format!("「{s}」")).unwrap_or_else(|| "控件".into());
|
||||
unsafe { CFRetain(best.1) };
|
||||
let r = press(best.1, &role_of(best.1));
|
||||
unsafe { CFRelease(best.1) };
|
||||
r.map(|_| label)
|
||||
}
|
||||
|
||||
/// Set the focused field's text (typing without keystrokes). False = caller types keys.
|
||||
pub fn set_focused_text(pid: i32, text: &str, append: bool) -> bool {
|
||||
let Some(app) = app(pid) else { return false };
|
||||
let Some(focused) = attr(app.0, "AXFocusedUIElement") else { return false };
|
||||
if !EDITABLE_ROLES.contains(&role_of(focused.0).as_str()) {
|
||||
return false;
|
||||
}
|
||||
let old = attr_string(focused.0, "AXValue").unwrap_or_default();
|
||||
let merged = merged_text(&old, text, append);
|
||||
let v = cfstr(&merged);
|
||||
unsafe { AXUIElementSetAttributeValue(focused.0, cfstr("AXValue").0, v.0) == 0 }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
mod mac {
|
||||
/// No accessibility write path off macOS: callers use the pixel path.
|
||||
pub fn press_named(_pid: i32, _want: &str) -> Result<String, String> {
|
||||
Err("只在 macOS 有輔助使用點按".into())
|
||||
}
|
||||
pub fn press_at(_pid: i32, _x: f64, _y: f64) -> Result<String, String> {
|
||||
Err("只在 macOS 有輔助使用點按".into())
|
||||
}
|
||||
pub fn set_focused_text(_pid: i32, _text: &str, _append: bool) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub use mac::{press_at, press_named, set_focused_text};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn exact_names_beat_partials() {
|
||||
assert_eq!(name_match(" 自動更新 ", "自動更新"), 2);
|
||||
assert_eq!(name_match("自動更新（每小時）", "自動更新"), 1);
|
||||
assert_eq!(name_match("OK", "ok"), 2);
|
||||
assert_eq!(name_match("Cancel", "OK"), 0);
|
||||
assert_eq!(name_match("", "OK"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn appending_keeps_what_the_field_held() {
|
||||
assert_eq!(merged_text("Hi", " there", true), "Hi there");
|
||||
assert_eq!(merged_text("Hi", "there", false), "there");
|
||||
}
|
||||
}
|
||||
|
|
@ -935,11 +935,39 @@ fn keys_go_where_looked(name: &str, v: &Value) -> Result<(), String> {
|
|||
}
|
||||
}
|
||||
|
||||
/// Accessibility-first click (macOS): the app's own tree answers name clicks and numbered
|
||||
/// spots exactly — no Retina scaling, no covered-window misses (bench: the quick model's
|
||||
/// wrong-number clicks). None = fall through to the pixel path.
|
||||
fn ax_click(v: &Value, scale: (f64, f64)) -> Option<String> {
|
||||
if !cfg!(target_os = "macos") {
|
||||
return None;
|
||||
}
|
||||
let (_, pid) = mac::front_app()?;
|
||||
if let Some(nm) = v["name"].as_str().map(str::trim).filter(|s| !s.is_empty()) {
|
||||
return crate::axwrite::press_named(pid, nm).ok().map(|w| format!("已點「{w}」（輔助使用，精準點按，不靠座標）。"));
|
||||
}
|
||||
let (x, y, _) = place_of(v, scale).ok()??;
|
||||
crate::axwrite::press_at(pid, x, y).ok().map(|w| format!("已點 {w}（輔助使用，精準點按，不靠座標）。"))
|
||||
}
|
||||
|
||||
/// Typing through accessibility: one AXValue write on the focused field. False = type keys.
|
||||
fn ax_type(text: &str, append: bool) -> bool {
|
||||
match mac::front_app() {
|
||||
Some((_, pid)) => crate::axwrite::set_focused_text(pid, text, append),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result<String, String> {
|
||||
keys_go_where_looked(name, v)?;
|
||||
let button = v["button"].as_str().unwrap_or("left");
|
||||
match name {
|
||||
"computer_use_click" => {
|
||||
if v["count"].as_u64().unwrap_or(1) <= 1 && button == "left" && flags_of(v) == 0 {
|
||||
if let Some(done) = ax_click(v, scale) {
|
||||
return Ok(done);
|
||||
}
|
||||
}
|
||||
let (x, y, at) = place_of(v, scale)?.ok_or("請給 x、y（截圖上的位置）或 ref（截圖上的編號）")?;
|
||||
let count = v["count"].as_u64().unwrap_or(1).clamp(1, 3) as i64;
|
||||
mac::click(x, y, button, count, flags_of(v));
|
||||
|
|
@ -1002,7 +1030,9 @@ pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result<String, String> {
|
|||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
}
|
||||
}
|
||||
if !text.is_empty() {
|
||||
// Straight into the focused field when accessibility can reach it (one write,
|
||||
// no keystrokes); keystrokes when it can't.
|
||||
if !text.is_empty() && !ax_type(text, v["append"].as_bool().unwrap_or(false)) {
|
||||
mac::type_text(text);
|
||||
}
|
||||
if submit {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ mod app;
|
|||
mod appctl;
|
||||
mod attach;
|
||||
mod axread;
|
||||
mod axwrite;
|
||||
mod avatar;
|
||||
mod bidi;
|
||||
mod botbrowser;
|
||||
|
|
|
|||
Loading…
Reference in New Issue