419 lines
18 KiB
Rust
419 lines
18 KiB
Rust
// Accessibility WRITE (macOS): press a control the accessibility tree names, set a focused
|
||
// text field's value. A control pressed through accessibility does not care about Retina
|
||
// scaling, a window covering part of the screen, or the picture's coordinates being one
|
||
// pixel off — the failures the bench kept catching when the model clicked by number.
|
||
// Everything here answers an error and the caller falls back to the pixel path (CGEvent),
|
||
// so an app with a thin accessibility tree behaves exactly as before.
|
||
//
|
||
// Behaviour spec: press = AXPress on the element (AXPick for menu items); a pressable element
|
||
// is a leaf-ish role in PRESSABLE_ROLES whose AXTitle/AXDescription/AXValue matches the name
|
||
// (trim, case-insensitive; exact first, then contains); point press = the deepest pressable
|
||
// element whose AXPosition/AXSize frame contains the point, else the deepest element under
|
||
// the point. Typing = AXValue on the app's AXFocusedUIElement when its role is editable.
|
||
|
||
/// Roles a name click may press (anything else under the name is not a click target).
|
||
pub const PRESSABLE_ROLES: [&str; 9] = [
|
||
"AXButton", "AXCheckBox", "AXRadioButton", "AXMenuItem", "AXPopUpButton", "AXMenuButton", "AXMenu", "AXTabButton", "AXLink",
|
||
];
|
||
/// Roles text may be set on.
|
||
pub const EDITABLE_ROLES: [&str; 5] = ["AXTextField", "AXTextArea", "AXComboBox", "AXSearchField", "AXPasswordTextField"];
|
||
|
||
/// Name matching: exact (trimmed, case-insensitive) beats contains.
|
||
pub fn name_match(hay: &str, want: &str) -> u8 {
|
||
let (h, w) = (hay.trim().to_lowercase(), want.trim().to_lowercase());
|
||
if h.is_empty() || w.is_empty() {
|
||
return 0;
|
||
}
|
||
if h == w {
|
||
2
|
||
} else if h.contains(&w) {
|
||
1
|
||
} else {
|
||
0
|
||
}
|
||
}
|
||
|
||
/// The text a field ends up holding (accessibility set replaces the whole value).
|
||
pub fn merged_text(old: &str, new: &str, append: bool) -> String {
|
||
if append {
|
||
format!("{old}{new}")
|
||
} else {
|
||
new.to_string()
|
||
}
|
||
}
|
||
|
||
#[cfg(target_os = "macos")]
|
||
mod mac {
|
||
use super::{name_match, merged_text, EDITABLE_ROLES, PRESSABLE_ROLES};
|
||
use std::ffi::{c_char, c_void, CString};
|
||
use std::os::raw::c_void as raw_void;
|
||
|
||
type CFTypeRef = *const raw_void;
|
||
|
||
#[link(name = "ApplicationServices", kind = "framework")]
|
||
extern "C" {
|
||
fn AXUIElementCreateApplication(pid: i32) -> CFTypeRef;
|
||
fn AXUIElementCopyAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: *mut CFTypeRef) -> i32;
|
||
fn AXUIElementSetAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: CFTypeRef) -> i32;
|
||
fn AXUIElementPerformAction(el: CFTypeRef, action: CFTypeRef) -> i32;
|
||
fn AXUIElementSetMessagingTimeout(el: CFTypeRef, secs: f32) -> i32;
|
||
fn AXValueGetValue(value: CFTypeRef, kind: u32, out: *mut c_void) -> bool;
|
||
}
|
||
|
||
#[link(name = "CoreFoundation", kind = "framework")]
|
||
extern "C" {
|
||
fn CFStringCreateWithCString(alloc: CFTypeRef, s: *const c_char, enc: u32) -> CFTypeRef;
|
||
fn CFRelease(x: CFTypeRef);
|
||
fn CFRetain(x: CFTypeRef) -> CFTypeRef;
|
||
fn CFGetTypeID(x: CFTypeRef) -> usize;
|
||
fn CFStringGetTypeID() -> usize;
|
||
fn CFArrayGetTypeID() -> usize;
|
||
fn CFArrayGetCount(a: CFTypeRef) -> isize;
|
||
fn CFArrayGetValueAtIndex(a: CFTypeRef, i: isize) -> CFTypeRef;
|
||
fn CFStringGetLength(s: CFTypeRef) -> isize;
|
||
fn CFStringGetMaximumSizeForEncoding(len: isize, enc: u32) -> isize;
|
||
fn CFStringGetCString(s: CFTypeRef, buf: *mut c_char, size: isize, enc: u32) -> bool;
|
||
}
|
||
|
||
const UTF8: u32 = 0x0800_0100;
|
||
const AX_VALUE_POINT: u32 = 1;
|
||
const MAX_NODES: usize = 20_000;
|
||
const MAX_DEPTH: usize = 80;
|
||
/// An app that answers slowly (Electron building its tree) must not hang the step.
|
||
const TIMEOUT_SECS: f32 = 2.0;
|
||
|
||
struct Cf(CFTypeRef);
|
||
impl Drop for Cf {
|
||
fn drop(&mut self) {
|
||
if !self.0.is_null() {
|
||
unsafe { CFRelease(self.0) }
|
||
}
|
||
}
|
||
}
|
||
|
||
fn cfstr(s: &str) -> Cf {
|
||
let c = CString::new(s).unwrap_or_default();
|
||
Cf(unsafe { CFStringCreateWithCString(std::ptr::null(), c.as_ptr(), UTF8) })
|
||
}
|
||
|
||
fn string_of(v: CFTypeRef) -> Option<String> {
|
||
unsafe {
|
||
if v.is_null() || CFGetTypeID(v) != CFStringGetTypeID() {
|
||
return None;
|
||
}
|
||
let size = CFStringGetMaximumSizeForEncoding(CFStringGetLength(v), UTF8) + 1;
|
||
let mut buf = vec![0u8; size.max(1) as usize];
|
||
if !CFStringGetCString(v, buf.as_mut_ptr() as *mut c_char, size, UTF8) {
|
||
return None;
|
||
}
|
||
let end = buf.iter().position(|b| *b == 0).unwrap_or(buf.len());
|
||
Some(String::from_utf8_lossy(&buf[..end]).into_owned())
|
||
}
|
||
}
|
||
|
||
fn attr(el: CFTypeRef, name: &str) -> Option<Cf> {
|
||
let key = cfstr(name);
|
||
let mut out: CFTypeRef = std::ptr::null();
|
||
let err = unsafe { AXUIElementCopyAttributeValue(el, key.0, &mut out) };
|
||
(err == 0 && !out.is_null()).then(|| Cf(out))
|
||
}
|
||
|
||
fn attr_string(el: CFTypeRef, name: &str) -> Option<String> {
|
||
attr(el, name).and_then(|v| string_of(v.0))
|
||
}
|
||
|
||
fn children(el: CFTypeRef) -> (Option<Cf>, Vec<CFTypeRef>) {
|
||
let Some(arr) = attr(el, "AXChildren") else { return (None, vec![]) };
|
||
unsafe {
|
||
if CFGetTypeID(arr.0) != CFArrayGetTypeID() {
|
||
return (Some(arr), vec![]);
|
||
}
|
||
let n = CFArrayGetCount(arr.0);
|
||
let ptr = arr.0;
|
||
(Some(arr), (0..n).map(|i| CFArrayGetValueAtIndex(ptr, i)).collect())
|
||
}
|
||
}
|
||
|
||
/// (x, y, w, h) in screen points, from AXPosition + AXSize.
|
||
fn frame(el: CFTypeRef) -> Option<(f64, f64, f64, f64)> {
|
||
let pos = attr(el, "AXPosition")?;
|
||
let size = attr(el, "AXSize")?;
|
||
let mut p = [0f64; 2];
|
||
let mut s = [0f64; 2];
|
||
unsafe {
|
||
if !AXValueGetValue(pos.0, AX_VALUE_POINT, p.as_mut_ptr() as *mut c_void) {
|
||
return None;
|
||
}
|
||
if !AXValueGetValue(size.0, 2, s.as_mut_ptr() as *mut c_void) {
|
||
return None;
|
||
}
|
||
}
|
||
Some((p[0], p[1], s[0], s[1]))
|
||
}
|
||
|
||
fn pressable(el: CFTypeRef) -> bool {
|
||
attr_string(el, "AXRole").is_some_and(|r| PRESSABLE_ROLES.contains(&r.as_str()))
|
||
}
|
||
|
||
/// Best (score, depth, element) pressable whose title/description/value matches `want`.
|
||
fn find_named(el: CFTypeRef, want: &str, best: &mut (u8, usize, CFTypeRef), seen: &mut usize, depth: usize) {
|
||
if *seen >= MAX_NODES || depth > MAX_DEPTH {
|
||
return;
|
||
}
|
||
*seen += 1;
|
||
if pressable(el) {
|
||
let score = ["AXTitle", "AXDescription", "AXValue"].iter().map(|a| attr_string(el, a).map(|s| name_match(&s, want)).unwrap_or(0)).max().unwrap_or(0);
|
||
// Deeper wins ties: a button inside a group is the thing, not the group.
|
||
if score > best.0 || (score == best.0 && score > 0 && depth > best.1) {
|
||
*best = (score, depth, el);
|
||
}
|
||
}
|
||
let (_keep, kids) = children(el);
|
||
for k in kids {
|
||
find_named(k, want, best, seen, depth + 1);
|
||
}
|
||
}
|
||
|
||
/// Deepest pressable whose frame holds the point; deepest of any role as a second choice.
|
||
fn find_at(el: CFTypeRef, x: f64, y: f64, best: &mut (usize, CFTypeRef, bool), seen: &mut usize, depth: usize) {
|
||
if *seen >= MAX_NODES || depth > MAX_DEPTH {
|
||
return;
|
||
}
|
||
*seen += 1;
|
||
if let Some((fx, fy, fw, fh)) = frame(el) {
|
||
if fw > 0.0 && fh > 0.0 && x >= fx && x < fx + fw && y >= fy && y < fy + fh {
|
||
if pressable(el) && depth >= best.0 {
|
||
*best = (depth, el, true);
|
||
} else if !best.2 && depth >= best.0 {
|
||
*best = (depth, el, false);
|
||
}
|
||
}
|
||
}
|
||
let (_keep, kids) = children(el);
|
||
for k in kids {
|
||
find_at(k, x, y, best, seen, depth + 1);
|
||
}
|
||
}
|
||
|
||
fn press(el: CFTypeRef, role: &str) -> Result<(), String> {
|
||
// Menu items answer AXPick, buttons AXPress; try the fitting one, then the other.
|
||
let acts = if role == "AXMenuItem" { ["AXPick", "AXPress"] } else { ["AXPress", "AXPick"] };
|
||
for a in acts {
|
||
let key = cfstr(a);
|
||
if unsafe { AXUIElementPerformAction(el, key.0) } == 0 {
|
||
return Ok(());
|
||
}
|
||
}
|
||
Err("這個控件不接受 AXPress".into())
|
||
}
|
||
|
||
fn app(pid: i32) -> Option<Cf> {
|
||
let el = unsafe { AXUIElementCreateApplication(pid) };
|
||
(!el.is_null()).then(|| {
|
||
unsafe { AXUIElementSetMessagingTimeout(el, TIMEOUT_SECS) };
|
||
Cf(el)
|
||
})
|
||
}
|
||
|
||
/// The element's role (kept out of the walk: the caller reports it).
|
||
fn role_of(el: CFTypeRef) -> String {
|
||
attr_string(el, "AXRole").unwrap_or_default()
|
||
}
|
||
|
||
/// Press the control named `want` in the app `pid`. Returns the matched title.
|
||
pub fn press_named(pid: i32, want: &str) -> Result<String, String> {
|
||
let app = app(pid).ok_or("這個 App 沒有輔助使用介面")?;
|
||
let mut best = (0u8, 0usize, std::ptr::null());
|
||
let mut seen = 0;
|
||
find_named(app.0, want, &mut best, &mut seen, 0);
|
||
if best.2.is_null() {
|
||
return Err(format!("輔助使用樹裡沒有可點的「{want}」"));
|
||
}
|
||
let label = ["AXTitle", "AXDescription", "AXValue"].iter().filter_map(|a| attr_string(best.2, a)).find(|s| !s.trim().is_empty()).unwrap_or_else(|| want.to_string());
|
||
// Keep the element alive through the press: CFRetain, press, then release.
|
||
unsafe { CFRetain(best.2) };
|
||
let r = press(best.2, &role_of(best.2));
|
||
unsafe { CFRelease(best.2) };
|
||
r.map(|_| label)
|
||
}
|
||
|
||
/// Press what sits under a screen point, if accessibility can name it.
|
||
pub fn press_at(pid: i32, x: f64, y: f64) -> Result<String, String> {
|
||
let app = app(pid).ok_or("這個 App 沒有輔助使用介面")?;
|
||
let mut best = (0usize, std::ptr::null(), false);
|
||
let mut seen = 0;
|
||
find_at(app.0, x, y, &mut best, &mut seen, 0);
|
||
if best.1.is_null() {
|
||
return Err("那個位置不在任何控件上".into());
|
||
}
|
||
if !best.2 {
|
||
return Err("那個位置的控件不接受點按".into());
|
||
}
|
||
let label = ["AXTitle", "AXDescription", "AXValue"].iter().filter_map(|a| attr_string(best.1, a)).find(|s| !s.trim().is_empty()).map(|s| format!("「{s}」")).unwrap_or_else(|| "控件".into());
|
||
unsafe { CFRetain(best.1) };
|
||
let r = press(best.1, &role_of(best.1));
|
||
unsafe { CFRelease(best.1) };
|
||
r.map(|_| label)
|
||
}
|
||
|
||
/// Set the focused field's text (typing without keystrokes). False = caller types keys.
|
||
pub fn set_focused_text(pid: i32, text: &str, append: bool) -> bool {
|
||
let Some(app) = app(pid) else { return false };
|
||
let Some(focused) = attr(app.0, "AXFocusedUIElement") else { return false };
|
||
if !EDITABLE_ROLES.contains(&role_of(focused.0).as_str()) {
|
||
return false;
|
||
}
|
||
let old = attr_string(focused.0, "AXValue").unwrap_or_default();
|
||
let merged = merged_text(&old, text, append);
|
||
let v = cfstr(&merged);
|
||
unsafe { AXUIElementSetAttributeValue(focused.0, cfstr("AXValue").0, v.0) == 0 }
|
||
}
|
||
/// Deepest AXMenuItem (in any open menu) whose words match `want` best (exact first).
|
||
fn find_menu_item(el: CFTypeRef, want: &str, best: &mut (u8, usize, CFTypeRef), seen: &mut usize, depth: usize) {
|
||
if *seen >= MAX_NODES || depth > MAX_DEPTH {
|
||
return;
|
||
}
|
||
*seen += 1;
|
||
if attr_string(el, "AXRole").is_some_and(|r| r == "AXMenuItem" || r == "AXMenuBarItem") {
|
||
let score = ["AXTitle", "AXDescription"].iter().map(|a| attr_string(el, a).map(|s| name_match(&s, want)).unwrap_or(0)).max().unwrap_or(0);
|
||
if score > best.0 {
|
||
*best = (score, depth, el);
|
||
}
|
||
}
|
||
let (_keep, kids) = children(el);
|
||
for k in kids {
|
||
find_menu_item(k, want, best, seen, depth + 1);
|
||
}
|
||
}
|
||
|
||
/// Walk the app's real menu bar: press 檔案, then 新視窗… (what took the bot three
|
||
/// screenshot rounds per level, menus being invisible to the screenshot numbering).
|
||
pub fn menu_click(pid: i32, path: &[String]) -> Result<String, String> {
|
||
let app = app(pid).ok_or("這個 App 沒有輔助使用介面")?;
|
||
let bar = attr(app.0, "AXMenuBar").ok_or("這個 App 沒有選單列")?;
|
||
let (_keep, items) = children(bar.0);
|
||
let want = path[0].trim();
|
||
let item = items
|
||
.iter()
|
||
.filter(|k| attr_string(**k, "AXRole").is_some_and(|r| r == "AXMenuBarItem"))
|
||
.max_by_key(|k| ["AXTitle", "AXDescription"].iter().map(|a| attr_string(**k, a).map(|s| name_match(&s, want)).unwrap_or(0)).max().unwrap_or(0))
|
||
.ok_or("選單列是空的")?;
|
||
let score = ["AXTitle", "AXDescription"].iter().map(|a| attr_string(*item, a).map(|s| name_match(&s, want)).unwrap_or(0)).max().unwrap_or(0);
|
||
if score == 0 {
|
||
let names = items.iter().filter_map(|k| attr_string(*k, "AXTitle")).collect::<Vec<_>>().join("、");
|
||
return Err(format!("選單列裡沒有「{want}」（有：{names}）"));
|
||
}
|
||
unsafe { CFRetain(*item) };
|
||
let r = press(*item, "AXMenuBarItem");
|
||
unsafe { CFRelease(*item) };
|
||
r?;
|
||
std::thread::sleep(std::time::Duration::from_millis(350));
|
||
for seg in &path[1..] {
|
||
let mut best = (0u8, 0usize, std::ptr::null());
|
||
let mut seen = 0;
|
||
find_menu_item(app.0, seg.trim(), &mut best, &mut seen, 0);
|
||
if best.2.is_null() {
|
||
return Err(format!("打開的選單裡沒有「{}」（前面已點：{}）", seg.trim(), path[..path.len() - 1].join("→")));
|
||
}
|
||
unsafe { CFRetain(best.2) };
|
||
let r = press(best.2, "AXMenuItem");
|
||
unsafe { CFRelease(best.2) };
|
||
r?;
|
||
std::thread::sleep(std::time::Duration::from_millis(250));
|
||
}
|
||
Ok(path.join(" → "))
|
||
}
|
||
|
||
/// Fill named fields with one AXValue write each (no clicks, no keystrokes). Names match
|
||
/// title / description / placeholder, as the numbered list shows them.
|
||
pub fn fill_form(pid: i32, fields: &[(String, String)]) -> Vec<(String, Result<(), String>)> {
|
||
let app = match app(pid) {
|
||
Some(a) => a,
|
||
None => return fields.iter().map(|(n, _)| (n.clone(), Err("這個 App 沒有輔助使用介面".into()))).collect(),
|
||
};
|
||
let mut editables: Vec<CFTypeRef> = Vec::new();
|
||
collect_editables(app.0, &mut editables, &mut 0, 0);
|
||
let mut out = Vec::new();
|
||
for (name, value) in fields {
|
||
let best = editables
|
||
.iter()
|
||
.filter_map(|e| {
|
||
let s = name_match(&name_of(*e), name);
|
||
(s > 0 && is_editable(*e)).then_some((s, *e))
|
||
})
|
||
.max_by_key(|(s, _)| *s);
|
||
out.push(match best {
|
||
Some((_, e)) => {
|
||
let v = cfstr(value);
|
||
let ok = unsafe { AXUIElementSetAttributeValue(e, cfstr("AXValue").0, v.0) == 0 };
|
||
(name.clone(), if ok { Ok(()) } else { Err("這個欄位不接受直接寫入".into()) })
|
||
}
|
||
None => (name.clone(), Err("畫面裡沒有這個名字的欄位（先截圖看清單）".into())),
|
||
});
|
||
}
|
||
out
|
||
}
|
||
|
||
fn is_editable(el: CFTypeRef) -> bool {
|
||
attr_string(el, "AXRole").is_some_and(|r| EDITABLE_ROLES.contains(&r.as_str()))
|
||
}
|
||
|
||
fn name_of(el: CFTypeRef) -> String {
|
||
["AXTitle", "AXDescription", "AXPlaceholderValue", "AXHelp"].iter().filter_map(|a| attr_string(el, a)).find(|s| !s.trim().is_empty()).unwrap_or_default()
|
||
}
|
||
|
||
fn collect_editables(el: CFTypeRef, out: &mut Vec<CFTypeRef>, seen: &mut usize, depth: usize) {
|
||
if *seen >= MAX_NODES || depth > MAX_DEPTH {
|
||
return;
|
||
}
|
||
*seen += 1;
|
||
out.push(el);
|
||
let (_keep, kids) = children(el);
|
||
for k in kids {
|
||
collect_editables(k, out, seen, depth + 1);
|
||
}
|
||
}
|
||
}
|
||
|
||
#[cfg(not(target_os = "macos"))]
|
||
mod mac {
|
||
/// No accessibility write path off macOS: callers use the pixel path.
|
||
pub fn press_named(_pid: i32, _want: &str) -> Result<String, String> {
|
||
Err("只在 macOS 有輔助使用點按".into())
|
||
}
|
||
pub fn press_at(_pid: i32, _x: f64, _y: f64) -> Result<String, String> {
|
||
Err("只在 macOS 有輔助使用點按".into())
|
||
}
|
||
pub fn set_focused_text(_pid: i32, _text: &str, _append: bool) -> bool {
|
||
false
|
||
}
|
||
pub fn menu_click(_pid: i32, _path: &[String]) -> Result<String, String> {
|
||
Err("選單直走只在 macOS 有（其他系統照編號點選單）".into())
|
||
}
|
||
pub fn fill_form(_pid: i32, fields: &[(String, String)]) -> Vec<(String, Result<(), String>)> {
|
||
fields.iter().map(|(n, _)| (n.clone(), Err("選欄直填只在 macOS 有（其他系統請逐欄 click＋typing）".into()))).collect()
|
||
}
|
||
}
|
||
|
||
pub use mac::{fill_form, menu_click, press_at, press_named, set_focused_text};
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn exact_names_beat_partials() {
|
||
assert_eq!(name_match(" 自動更新 ", "自動更新"), 2);
|
||
assert_eq!(name_match("自動更新（每小時）", "自動更新"), 1);
|
||
assert_eq!(name_match("OK", "ok"), 2);
|
||
assert_eq!(name_match("Cancel", "OK"), 0);
|
||
assert_eq!(name_match("", "OK"), 0);
|
||
}
|
||
|
||
#[test]
|
||
fn appending_keeps_what_the_field_held() {
|
||
assert_eq!(merged_text("Hi", " there", true), "Hi there");
|
||
assert_eq!(merged_text("Hi", "there", false), "there");
|
||
}
|
||
}
|