644 lines
28 KiB
Rust
644 lines
28 KiB
Rust
// Read the person's own browser window through macOS accessibility: no browser setting
|
||
// ("允許來自 Apple 事件的 JavaScript") and no remote control of Chrome, so it works in whichever
|
||
// Chrome profile the person has open. macOS asks once to let AnyBot use 輔助使用 (Accessibility).
|
||
//
|
||
// Behaviour spec: an app counts as a Chromium/Electron "renderer" when its bundle id is
|
||
// `com.github.Electron` or `com.google.Chrome`, starts with `com.google.Chrome.`, its executable
|
||
// path contains `/Electron.app/Contents/MacOS/Electron`, or `<bundle>/Contents/Frameworks/`
|
||
// holds one of: Electron Framework, Chromium Embedded Framework, Google Chrome Framework.
|
||
// On such apps, set `AXManualAccessibility` first, then `AXEnhancedUserInterface`, on the app's
|
||
// accessibility element, so their web content shows up in the accessibility tree.
|
||
|
||
/// Bundle ids that are renderers outright (isRenderer).
|
||
pub const RENDERER_BUNDLE_IDS: [&str; 2] = ["com.github.Electron", "com.google.Chrome"];
|
||
/// Bundle id prefix for Chrome's channels and helpers (isRenderer).
|
||
pub const RENDERER_BUNDLE_PREFIX: &str = "com.google.Chrome.";
|
||
/// Executable path piece of a bare Electron app (isRenderer).
|
||
pub const ELECTRON_EXE_PIECE: &str = "/Electron.app/Contents/MacOS/Electron";
|
||
/// Frameworks that mark a Chromium-based bundle (isRendererBundle).
|
||
pub const RENDERER_FRAMEWORKS: [&str; 3] = ["Electron Framework.framework", "Chromium Embedded Framework.framework", "Google Chrome Framework.framework"];
|
||
/// The two flags written on renderer apps, in order (writeFlags).
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
pub const RENDERER_AX_FLAGS: [&str; 2] = ["AXManualAccessibility", "AXEnhancedUserInterface"];
|
||
/// Most accessibility elements walked for one read (estimated).
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
pub const MAX_AX_NODES: usize = 20_000;
|
||
/// Deepest accessibility nesting followed (estimated).
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
const MAX_AX_DEPTH: usize = 80;
|
||
/// Tries while Chrome builds its accessibility tree after the flags are set (estimated).
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
const READ_TRIES: usize = 5;
|
||
/// Pause between those tries (estimated).
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
const RETRY_PAUSE_MS: u64 = 700;
|
||
/// Longest wait for one accessibility answer (estimated).
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
const AX_TIMEOUT_SECS: f32 = 2.0;
|
||
|
||
/// isRenderer(bundleIdentifier:executablePath:) + isRendererBundle(atPath:), with the framework
|
||
/// check passed in so it can be tested.
|
||
pub fn is_renderer(bundle_id: Option<&str>, exe_path: Option<&str>, has_framework: impl Fn(&str) -> bool) -> bool {
|
||
if let Some(id) = bundle_id {
|
||
if RENDERER_BUNDLE_IDS.contains(&id) || id.starts_with(RENDERER_BUNDLE_PREFIX) {
|
||
return true;
|
||
}
|
||
}
|
||
if let Some(p) = exe_path {
|
||
if p.contains(ELECTRON_EXE_PIECE) {
|
||
return true;
|
||
}
|
||
}
|
||
RENDERER_FRAMEWORKS.iter().any(|f| has_framework(f))
|
||
}
|
||
|
||
/// The `.app` folder an executable lives in.
|
||
pub fn bundle_of(exe: &str) -> Option<&str> {
|
||
exe.find(".app/").map(|i| &exe[..i + 4])
|
||
}
|
||
|
||
/// The person's own browser process (not AnyBot's own window, which runs from AnyBot's folder).
|
||
pub fn pick_person_pid(ps: &str, exe: &str) -> Option<i32> {
|
||
ps.lines().find_map(|l| {
|
||
let l = l.trim_start();
|
||
let (pid, args) = l.split_once(' ')?;
|
||
let args = args.trim_start();
|
||
let own = args.strip_prefix(exe)?;
|
||
(own.is_empty() || own.starts_with(' ')).then_some(())?;
|
||
(!args.contains("/AnyBot/browser") && !args.contains(" --type=")).then_some(())?;
|
||
pid.parse().ok()
|
||
})
|
||
}
|
||
|
||
pub struct Page {
|
||
pub title: String,
|
||
pub url: String,
|
||
pub text: String,
|
||
}
|
||
|
||
#[cfg(target_os = "macos")]
|
||
mod mac {
|
||
use super::*;
|
||
use std::ffi::{c_char, c_void, CString};
|
||
|
||
type CFTypeRef = *const c_void;
|
||
|
||
#[link(name = "ApplicationServices", kind = "framework")]
|
||
extern "C" {
|
||
fn AXIsProcessTrustedWithOptions(options: CFTypeRef) -> bool;
|
||
fn AXUIElementCreateApplication(pid: i32) -> CFTypeRef;
|
||
fn AXUIElementSetAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: CFTypeRef) -> i32;
|
||
fn AXUIElementCopyAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: *mut CFTypeRef) -> i32;
|
||
fn AXUIElementSetMessagingTimeout(el: CFTypeRef, secs: f32) -> i32;
|
||
fn AXValueGetValue(value: CFTypeRef, kind: u32, out: *mut c_void) -> bool;
|
||
static kAXTrustedCheckOptionPrompt: CFTypeRef;
|
||
}
|
||
|
||
#[link(name = "CoreFoundation", kind = "framework")]
|
||
extern "C" {
|
||
fn CFStringCreateWithCString(alloc: CFTypeRef, s: *const c_char, enc: u32) -> CFTypeRef;
|
||
fn CFRelease(x: CFTypeRef);
|
||
fn CFRetain(x: CFTypeRef) -> CFTypeRef;
|
||
fn CFGetTypeID(x: CFTypeRef) -> usize;
|
||
fn CFStringGetTypeID() -> usize;
|
||
fn CFArrayGetTypeID() -> usize;
|
||
fn CFURLGetTypeID() -> usize;
|
||
fn CFArrayGetCount(a: CFTypeRef) -> isize;
|
||
fn CFArrayGetValueAtIndex(a: CFTypeRef, i: isize) -> CFTypeRef;
|
||
fn CFStringGetLength(s: CFTypeRef) -> isize;
|
||
fn CFStringGetMaximumSizeForEncoding(len: isize, enc: u32) -> isize;
|
||
fn CFStringGetCString(s: CFTypeRef, buf: *mut c_char, size: isize, enc: u32) -> bool;
|
||
fn CFURLGetString(u: CFTypeRef) -> CFTypeRef;
|
||
fn CFNumberGetTypeID() -> usize;
|
||
fn CFNumberGetValue(n: CFTypeRef, kind: isize, out: *mut c_void) -> bool;
|
||
fn CFDictionaryCreate(alloc: CFTypeRef, keys: *const CFTypeRef, values: *const CFTypeRef, n: isize, kcb: *const c_void, vcb: *const c_void) -> CFTypeRef;
|
||
static kCFBooleanTrue: CFTypeRef;
|
||
static kCFTypeDictionaryKeyCallBacks: [usize; 6];
|
||
static kCFTypeDictionaryValueCallBacks: [usize; 5];
|
||
}
|
||
|
||
const UTF8: u32 = 0x0800_0100;
|
||
|
||
/// An owned Core Foundation value, released when dropped.
|
||
struct Cf(CFTypeRef);
|
||
impl Drop for Cf {
|
||
fn drop(&mut self) {
|
||
if !self.0.is_null() {
|
||
unsafe { CFRelease(self.0) }
|
||
}
|
||
}
|
||
}
|
||
|
||
fn cfstr(s: &str) -> Cf {
|
||
let c = CString::new(s).unwrap_or_default();
|
||
Cf(unsafe { CFStringCreateWithCString(std::ptr::null(), c.as_ptr(), UTF8) })
|
||
}
|
||
|
||
fn string_of(v: CFTypeRef) -> Option<String> {
|
||
unsafe {
|
||
if v.is_null() || CFGetTypeID(v) != CFStringGetTypeID() {
|
||
return None;
|
||
}
|
||
let size = CFStringGetMaximumSizeForEncoding(CFStringGetLength(v), UTF8) + 1;
|
||
let mut buf = vec![0u8; size.max(1) as usize];
|
||
if !CFStringGetCString(v, buf.as_mut_ptr() as *mut c_char, size, UTF8) {
|
||
return None;
|
||
}
|
||
let end = buf.iter().position(|b| *b == 0).unwrap_or(buf.len());
|
||
Some(String::from_utf8_lossy(&buf[..end]).into_owned())
|
||
}
|
||
}
|
||
|
||
fn attr(el: CFTypeRef, name: &str) -> Option<Cf> {
|
||
let key = cfstr(name);
|
||
let mut out: CFTypeRef = std::ptr::null();
|
||
let err = unsafe { AXUIElementCopyAttributeValue(el, key.0, &mut out) };
|
||
(err == 0 && !out.is_null()).then(|| Cf(out))
|
||
}
|
||
|
||
fn attr_string(el: CFTypeRef, name: &str) -> Option<String> {
|
||
attr(el, name).and_then(|v| string_of(v.0))
|
||
}
|
||
|
||
fn children(el: CFTypeRef) -> (Option<Cf>, Vec<CFTypeRef>) {
|
||
let Some(arr) = attr(el, "AXChildren") else { return (None, vec![]) };
|
||
unsafe {
|
||
if CFGetTypeID(arr.0) != CFArrayGetTypeID() {
|
||
return (Some(arr), vec![]);
|
||
}
|
||
let n = CFArrayGetCount(arr.0);
|
||
let items = (0..n).map(|i| CFArrayGetValueAtIndex(arr.0, i)).collect();
|
||
(Some(arr), items)
|
||
}
|
||
}
|
||
|
||
/// Is AnyBot allowed to use accessibility? `prompt` makes macOS show its own question.
|
||
pub fn trusted(prompt: bool) -> bool {
|
||
unsafe {
|
||
if !prompt {
|
||
return AXIsProcessTrustedWithOptions(std::ptr::null());
|
||
}
|
||
let keys = [kAXTrustedCheckOptionPrompt];
|
||
let values = [kCFBooleanTrue];
|
||
let dict = Cf(CFDictionaryCreate(std::ptr::null(), keys.as_ptr(), values.as_ptr(), 1, kCFTypeDictionaryKeyCallBacks.as_ptr() as *const c_void, kCFTypeDictionaryValueCallBacks.as_ptr() as *const c_void));
|
||
AXIsProcessTrustedWithOptions(dict.0)
|
||
}
|
||
}
|
||
|
||
///: both flags on the app element, in the same order.
|
||
fn write_flags(app: CFTypeRef) -> bool {
|
||
RENDERER_AX_FLAGS.iter().fold(true, |ok, f| {
|
||
let key = cfstr(f);
|
||
let err = unsafe { AXUIElementSetAttributeValue(app, key.0, kCFBooleanTrue) };
|
||
ok && err == 0
|
||
})
|
||
}
|
||
|
||
fn find_role(el: CFTypeRef, role: &str, seen: &mut usize, depth: usize) -> Option<Cf> {
|
||
if *seen >= MAX_AX_NODES || depth > MAX_AX_DEPTH {
|
||
return None;
|
||
}
|
||
*seen += 1;
|
||
let (_keep, kids) = children(el);
|
||
for k in kids {
|
||
if attr_string(k, "AXRole").as_deref() == Some(role) {
|
||
unsafe { CFRetain(k) };
|
||
return Some(Cf(k));
|
||
}
|
||
if let Some(found) = find_role(k, role, seen, depth + 1) {
|
||
return Some(found);
|
||
}
|
||
}
|
||
None
|
||
}
|
||
|
||
/// Visible text under the page: runs of static text become one line.
|
||
fn collect(el: CFTypeRef, out: &mut Vec<String>, line: &mut String, seen: &mut usize, depth: usize, max_chars: usize, total: &mut usize) {
|
||
if *seen >= MAX_AX_NODES || depth > MAX_AX_DEPTH || *total > max_chars {
|
||
return;
|
||
}
|
||
*seen += 1;
|
||
let (_keep, kids) = children(el);
|
||
for k in kids {
|
||
let role = attr_string(k, "AXRole").unwrap_or_default();
|
||
if role == "AXStaticText" {
|
||
if let Some(t) = attr_string(k, "AXValue") {
|
||
*total += t.chars().count();
|
||
line.push_str(&t);
|
||
}
|
||
} else {
|
||
flush(out, line);
|
||
collect(k, out, line, seen, depth + 1, max_chars, total);
|
||
flush(out, line);
|
||
}
|
||
}
|
||
}
|
||
|
||
fn flush(out: &mut Vec<String>, line: &mut String) {
|
||
let t = line.trim();
|
||
if !t.is_empty() {
|
||
out.push(t.to_string());
|
||
}
|
||
line.clear();
|
||
}
|
||
|
||
fn url_of(web: CFTypeRef) -> String {
|
||
let Some(u) = attr(web, "AXURL") else { return String::new() };
|
||
unsafe {
|
||
if CFGetTypeID(u.0) == CFURLGetTypeID() {
|
||
string_of(CFURLGetString(u.0)).unwrap_or_default()
|
||
} else {
|
||
string_of(u.0).unwrap_or_default()
|
||
}
|
||
}
|
||
}
|
||
|
||
pub fn read(pid: i32, renderer: bool, max_chars: usize) -> Result<Page, String> {
|
||
let app = Cf(unsafe { AXUIElementCreateApplication(pid) });
|
||
if app.0.is_null() {
|
||
return Err("連不上你的瀏覽器".into());
|
||
}
|
||
unsafe { AXUIElementSetMessagingTimeout(app.0, AX_TIMEOUT_SECS) };
|
||
if renderer {
|
||
write_flags(app.0);
|
||
}
|
||
let mut last_title = String::new();
|
||
for i in 0..READ_TRIES {
|
||
if i > 0 {
|
||
std::thread::sleep(std::time::Duration::from_millis(RETRY_PAUSE_MS));
|
||
}
|
||
let Some(win) = attr(app.0, "AXFocusedWindow").or_else(|| attr(app.0, "AXMainWindow")) else { continue };
|
||
last_title = attr_string(win.0, "AXTitle").unwrap_or_default();
|
||
let mut seen = 0;
|
||
let Some(web) = find_role(win.0, "AXWebArea", &mut seen, 0) else { continue };
|
||
let title = attr_string(web.0, "AXTitle").filter(|t| !t.is_empty()).unwrap_or_else(|| last_title.clone());
|
||
let (mut lines, mut line, mut seen, mut total) = (vec![], String::new(), 0, 0);
|
||
collect(web.0, &mut lines, &mut line, &mut seen, 0, max_chars, &mut total);
|
||
flush(&mut lines, &mut line);
|
||
if lines.is_empty() && i + 1 < READ_TRIES {
|
||
continue;
|
||
}
|
||
return Ok(Page { title, url: url_of(web.0), text: lines.join("\n") });
|
||
}
|
||
if last_title.is_empty() {
|
||
Err("你的瀏覽器現在沒有打開的視窗".into())
|
||
} else {
|
||
Err(format!("讀不到「{last_title}」的頁面內容"))
|
||
}
|
||
}
|
||
|
||
/// One status item in the menu bar (an app's icon at the top right): its app, its words,
|
||
/// and where it is in screen points.
|
||
pub struct StatusItem {
|
||
pub app: String,
|
||
pub label: String,
|
||
pub rect: [f64; 4],
|
||
}
|
||
|
||
/// The status items of these apps (pid, name), read through each app's AXExtrasMenuBar. An
|
||
/// app that doesn't answer within a quarter second is skipped.
|
||
pub fn status_items(apps: &[(i32, String)]) -> Vec<StatusItem> {
|
||
// Several apps at once: one that is slow to answer holds up only its share.
|
||
let chunk = apps.len().div_ceil(8).max(1);
|
||
std::thread::scope(|sc| {
|
||
let parts: Vec<_> = apps.chunks(chunk).map(|part| sc.spawn(move || status_items_of(part))).collect();
|
||
parts.into_iter().flat_map(|h| h.join().unwrap_or_default()).collect()
|
||
})
|
||
}
|
||
|
||
/// An icon's words: its own title or description, else the first found among its children
|
||
/// (the system's own icons are a group hosting the real button), else its identifier.
|
||
fn words_within(el: CFTypeRef, depth: usize) -> String {
|
||
let own = ["AXTitle", "AXDescription", "AXValue", "AXHelp"].iter().filter_map(|k| attr_string(el, k)).map(|t| t.trim().to_string()).find(|t| !t.is_empty());
|
||
if let Some(w) = own {
|
||
return w;
|
||
}
|
||
if depth > 0 {
|
||
let (_keep, kids) = children(el);
|
||
for k in kids {
|
||
let w = words_within(k, depth - 1);
|
||
if !w.is_empty() {
|
||
return w;
|
||
}
|
||
}
|
||
}
|
||
attr_string(el, "AXIdentifier").map(|id| super::menu_extra_name(&id)).unwrap_or_default()
|
||
}
|
||
|
||
/// One entry of an app's menus: a title in its menu bar (「檔案」), or an item of a menu that is
|
||
/// open now, with its keyboard shortcut when it has one.
|
||
pub struct MenuEntry {
|
||
pub label: String,
|
||
pub rect: [f64; 4],
|
||
/// 0: in the menu bar; 1: in the open menu; 2 and more: in a submenu opened from it.
|
||
pub depth: u8,
|
||
pub enabled: bool,
|
||
pub shortcut: String,
|
||
/// The item macOS shows in this place while option is held (「複製」 hides 「儲存為⋯」).
|
||
pub alternate: Option<(String, String)>,
|
||
}
|
||
|
||
fn rect_of(el: CFTypeRef) -> Option<[f64; 4]> {
|
||
#[repr(C)]
|
||
#[derive(Default, Clone, Copy)]
|
||
struct Xy {
|
||
a: f64,
|
||
b: f64,
|
||
}
|
||
let get = |k: &str, kind: u32| -> Option<Xy> {
|
||
let v = attr(el, k)?;
|
||
let mut xy = Xy::default();
|
||
unsafe { AXValueGetValue(v.0, kind, &mut xy as *mut Xy as *mut c_void) }.then_some(xy)
|
||
};
|
||
// kAXValueCGPointType = 1, kAXValueCGSizeType = 2
|
||
let (p, z) = (get("AXPosition", 1)?, get("AXSize", 2)?);
|
||
(z.a >= 1.0 && z.b >= 1.0).then_some([p.a, p.b, z.a, z.b])
|
||
}
|
||
|
||
fn attr_bool(el: CFTypeRef, name: &str) -> Option<bool> {
|
||
attr(el, name).map(|v| v.0 == unsafe { kCFBooleanTrue })
|
||
}
|
||
|
||
fn attr_int(el: CFTypeRef, name: &str) -> Option<i64> {
|
||
let v = attr(el, name)?;
|
||
let mut n: i64 = 0;
|
||
// kCFNumberSInt64Type = 4
|
||
unsafe { (CFGetTypeID(v.0) == CFNumberGetTypeID() && CFNumberGetValue(v.0, 4, &mut n as *mut i64 as *mut c_void)).then_some(n) }
|
||
}
|
||
|
||
/// The menus of the app `pid`: its menu bar's titles, and the items of whichever menu is open
|
||
/// (one pulled down from the bar, a submenu opened from it, or a right-click menu). Nothing
|
||
/// when the app doesn't answer within a quarter second.
|
||
pub fn app_menus(pid: i32) -> Vec<MenuEntry> {
|
||
let app = Cf(unsafe { AXUIElementCreateApplication(pid) });
|
||
if app.0.is_null() {
|
||
return vec![];
|
||
}
|
||
unsafe { AXUIElementSetMessagingTimeout(app.0, 0.25) };
|
||
let mut out = vec![];
|
||
if let Some(bar) = attr(app.0, "AXMenuBar") {
|
||
let (_keep, items) = children(bar.0);
|
||
for it in items {
|
||
let Some(rect) = rect_of(it) else { continue };
|
||
let label = attr_string(it, "AXTitle").unwrap_or_default().trim().to_string();
|
||
if label.is_empty() || rect[1] > 60.0 {
|
||
continue;
|
||
}
|
||
out.push(MenuEntry { label, rect, depth: 0, enabled: attr_bool(it, "AXEnabled").unwrap_or(true), shortcut: String::new(), alternate: None });
|
||
if attr_bool(it, "AXSelected") == Some(true) {
|
||
let (_k, menus) = children(it);
|
||
for m in menus {
|
||
menu_items(m, 1, &mut out);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// A right-click menu hangs off the app itself.
|
||
let (_keep, kids) = children(app.0);
|
||
for k in kids {
|
||
if attr_string(k, "AXRole").as_deref() == Some("AXMenu") {
|
||
menu_items(k, 1, &mut out);
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
fn menu_items(menu: CFTypeRef, depth: u8, out: &mut Vec<MenuEntry>) {
|
||
if depth > 3 || out.len() > 300 {
|
||
return;
|
||
}
|
||
let (_keep, items) = children(menu);
|
||
for it in items {
|
||
let label = attr_string(it, "AXTitle").unwrap_or_default().trim().to_string();
|
||
// Separators have no title; a closed menu's items have no place on the screen.
|
||
let Some(rect) = rect_of(it) else { continue };
|
||
if label.is_empty() {
|
||
continue;
|
||
}
|
||
let shortcut = attr_string(it, "AXMenuItemCmdChar").filter(|c| !c.is_empty()).map(|c| shortcut_words(&c, attr_int(it, "AXMenuItemCmdModifiers").unwrap_or(0))).unwrap_or_default();
|
||
let entry = MenuEntry { label, rect, depth, enabled: attr_bool(it, "AXEnabled").unwrap_or(true), shortcut, alternate: None };
|
||
// Two items in one place: one of them is the option-key alternate of the other, and
|
||
// only one shows. AX says nothing about which (AXMenuItemPrimaryUIElement is empty),
|
||
// so the one that needs option is taken for the hidden one.
|
||
if let Some(last) = out.last_mut().filter(|l| l.depth == depth && l.rect == entry.rect && l.alternate.is_none()) {
|
||
let needs_option = |e: &MenuEntry| e.shortcut.contains("option");
|
||
if needs_option(last) && !needs_option(&entry) {
|
||
let hidden = std::mem::replace(last, entry);
|
||
last.alternate = Some((hidden.label, hidden.shortcut));
|
||
} else {
|
||
last.alternate = Some((entry.label, entry.shortcut));
|
||
}
|
||
continue;
|
||
}
|
||
let entry_selected = attr_bool(it, "AXSelected") == Some(true);
|
||
out.push(entry);
|
||
if entry_selected {
|
||
let (_k, subs) = children(it);
|
||
for m in subs {
|
||
menu_items(m, depth + 1, out);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
fn status_items_of(apps: &[(i32, String)]) -> Vec<StatusItem> {
|
||
#[repr(C)]
|
||
#[derive(Default, Clone, Copy)]
|
||
struct Xy {
|
||
a: f64,
|
||
b: f64,
|
||
}
|
||
let mut out = vec![];
|
||
for (pid, name) in apps {
|
||
let app = Cf(unsafe { AXUIElementCreateApplication(*pid) });
|
||
if app.0.is_null() {
|
||
continue;
|
||
}
|
||
unsafe { AXUIElementSetMessagingTimeout(app.0, 0.25) };
|
||
let Some(bar) = attr(app.0, "AXExtrasMenuBar") else { continue };
|
||
let (_keep, items) = children(bar.0);
|
||
for it in items {
|
||
let get = |k: &str, kind: u32| -> Option<Xy> {
|
||
let v = attr(it, k)?;
|
||
let mut xy = Xy::default();
|
||
unsafe { AXValueGetValue(v.0, kind, &mut xy as *mut Xy as *mut c_void) }.then_some(xy)
|
||
};
|
||
// kAXValueCGPointType = 1, kAXValueCGSizeType = 2
|
||
let (Some(p), Some(z)) = (get("AXPosition", 1), get("AXSize", 2)) else { continue };
|
||
if z.a < 1.0 || z.b < 1.0 || p.b > 60.0 {
|
||
continue;
|
||
}
|
||
let label = words_within(it, 3);
|
||
out.push(StatusItem { app: name.clone(), label, rect: [p.a, p.b, z.a, z.b] });
|
||
}
|
||
}
|
||
out
|
||
}
|
||
}
|
||
|
||
#[cfg(target_os = "macos")]
|
||
pub use mac::app_menus;
|
||
#[cfg(target_os = "macos")]
|
||
pub use mac::trusted;
|
||
#[cfg(target_os = "macos")]
|
||
pub use mac::status_items;
|
||
|
||
/// A menu item's shortcut as the bot presses it (computer_use_press_key): AXMenuItemCmdModifiers
|
||
/// is 0 for ⌘ alone, plus 1 shift, 2 option, 4 control, 8 without ⌘.
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
pub fn shortcut_words(key: &str, mods: i64) -> String {
|
||
let mut parts = vec![];
|
||
if mods & 4 != 0 {
|
||
parts.push("ctrl");
|
||
}
|
||
if mods & 2 != 0 {
|
||
parts.push("option");
|
||
}
|
||
if mods & 1 != 0 {
|
||
parts.push("shift");
|
||
}
|
||
if mods & 8 == 0 {
|
||
parts.push("cmd");
|
||
}
|
||
// macOS writes arrows and function keys as private-use characters (NSUpArrowFunctionKey…).
|
||
let key = match key {
|
||
"\u{8}" | "\u{7f}" => "delete".to_string(),
|
||
"\r" | "\u{3}" => "return".to_string(),
|
||
"\u{1b}" => "escape".to_string(),
|
||
"\t" => "tab".to_string(),
|
||
" " => "space".to_string(),
|
||
"\u{f700}" => "up".to_string(),
|
||
"\u{f701}" => "down".to_string(),
|
||
"\u{f702}" => "left".to_string(),
|
||
"\u{f703}" => "right".to_string(),
|
||
k if k.chars().count() == 1 && ('\u{f704}'..='\u{f726}').contains(&k.chars().next().unwrap_or(' ')) => format!("f{}", k.chars().next().unwrap_or(' ') as u32 - 0xf703),
|
||
k => k.trim().to_lowercase(),
|
||
};
|
||
parts.push(&key);
|
||
parts.join("+")
|
||
}
|
||
|
||
/// A system menu-bar icon's words from its identifier (com.apple.menuextra.wifi → Wi-Fi): the
|
||
/// system's own icons carry no title.
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
pub fn menu_extra_name(id: &str) -> String {
|
||
let tail = id.rsplit(['.', '/']).next().unwrap_or(id).to_lowercase();
|
||
let known = [
|
||
("wifi", "Wi-Fi"), ("airport", "Wi-Fi"), ("battery", "電池"), ("bluetooth", "藍牙"), ("clock", "時鐘"),
|
||
("controlcenter", "控制中心"), ("sound", "聲音"), ("volume", "聲音"), ("focusmodes", "專注模式"), ("focus", "專注模式"),
|
||
("display", "顯示器"), ("nowplaying", "正在播放"), ("screenmirroring", "螢幕鏡像輸出"), ("vpn", "VPN"),
|
||
("textinput", "輸入法"), ("spotlight", "Spotlight"), ("siri", "Siri"), ("user", "使用者"), ("timemachine", "時光機"),
|
||
("keyboardbrightness", "鍵盤亮度"), ("accessibility", "輔助使用"), ("stagemanager", "幕前調度"), ("airdrop", "AirDrop"),
|
||
];
|
||
known.iter().find(|(k, _)| tail.contains(k)).map(|(_, v)| v.to_string()).unwrap_or_else(|| id.to_string())
|
||
}
|
||
|
||
#[cfg(not(target_os = "macos"))]
|
||
pub fn trusted(_prompt: bool) -> bool {
|
||
false
|
||
}
|
||
|
||
/// Read the front window of the person's browser at `exe` (e.g. Chrome's executable).
|
||
pub fn read_person_browser(exe: &str, max_chars: usize) -> Result<Page, String> {
|
||
if !cfg!(target_os = "macos") {
|
||
return Err("只支援 macOS".into());
|
||
}
|
||
if !trusted(false) {
|
||
crate::computer::ask_permission("Privacy_Accessibility");
|
||
return Err("macOS 還沒讓 AnyBot 使用「輔助使用」，所以看不到你的瀏覽器。我已經打開「系統設定 → 隱私權與安全性 → 輔助使用」：把 AnyBot（或「終端機」）打開，然後再叫我試一次。只要開一次。".into());
|
||
}
|
||
let ps = std::process::Command::new("ps").args(["-axo", "pid=,args="]).output().map(|o| String::from_utf8_lossy(&o.stdout).into_owned()).unwrap_or_default();
|
||
let Some(pid) = pick_person_pid(&ps, exe) else { return Err("你的瀏覽器沒有開著。請先打開它（或叫我用 browser_open 打開網址）。".into()) };
|
||
let bundle = bundle_of(exe).unwrap_or("");
|
||
let bundle_id = std::process::Command::new("plutil")
|
||
.args(["-extract", "CFBundleIdentifier", "raw", "-o", "-", &format!("{bundle}/Contents/Info.plist")])
|
||
.output()
|
||
.ok()
|
||
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
|
||
.filter(|s| !s.is_empty());
|
||
let renderer = is_renderer(bundle_id.as_deref(), Some(exe), |f| std::path::Path::new(bundle).join("Contents/Frameworks").join(f).exists());
|
||
#[cfg(target_os = "macos")]
|
||
{
|
||
mac::read(pid, renderer, max_chars)
|
||
}
|
||
#[cfg(not(target_os = "macos"))]
|
||
{
|
||
let _ = (pid, renderer, max_chars);
|
||
Err("只支援 macOS".into())
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn renderer_apps_follow_the_original_rule() {
|
||
let none = |_: &str| false;
|
||
assert!(is_renderer(Some("com.google.Chrome"), None, none));
|
||
assert!(is_renderer(Some("com.google.Chrome.canary"), None, none));
|
||
assert!(is_renderer(Some("com.github.Electron"), None, none));
|
||
assert!(is_renderer(None, Some("/tmp/x/Electron.app/Contents/MacOS/Electron"), none));
|
||
assert!(!is_renderer(Some("com.google.Chromeish"), None, none));
|
||
assert!(!is_renderer(Some("com.apple.Safari"), Some("/Applications/Safari.app/Contents/MacOS/Safari"), none));
|
||
// Brave / Edge carry their own framework names, so only the framework check could catch them.
|
||
assert!(is_renderer(Some("com.example.app"), None, |f| f == "Electron Framework.framework"));
|
||
}
|
||
|
||
#[test]
|
||
fn the_persons_browser_is_not_anybots_window() {
|
||
let exe = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
|
||
let ps = format!(
|
||
" 101 {exe} --user-data-dir=/Users/me/Library/Application Support/AnyBot/browser --remote-debugging-port=9333\n 102 {exe} --type=utility\n 205 {exe}\n 300 {exe} --type=renderer"
|
||
);
|
||
assert_eq!(pick_person_pid(&ps, exe), Some(205));
|
||
assert_eq!(pick_person_pid(" 1 /Applications/Safari.app/Contents/MacOS/Safari", exe), None);
|
||
}
|
||
|
||
#[test]
|
||
fn a_menu_items_shortcut_reads_as_keys_to_press() {
|
||
assert_eq!(shortcut_words("S", 0), "cmd+s");
|
||
assert_eq!(shortcut_words("S", 3), "option+shift+cmd+s");
|
||
assert_eq!(shortcut_words("F", 4), "ctrl+cmd+f");
|
||
assert_eq!(shortcut_words("\u{8}", 0), "cmd+delete");
|
||
assert_eq!(shortcut_words("\u{f701}", 2), "option+cmd+down");
|
||
assert_eq!(shortcut_words("\u{f704}", 8), "f1");
|
||
}
|
||
|
||
#[test]
|
||
fn bundle_is_found_from_the_executable() {
|
||
assert_eq!(bundle_of("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"), Some("/Applications/Google Chrome.app"));
|
||
assert_eq!(bundle_of("/usr/bin/true"), None);
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod live {
|
||
/// By hand: `ANYBOT_MENU_PID=<pid> cargo test live_app_menus -- --ignored --nocapture`.
|
||
#[cfg(target_os = "macos")]
|
||
#[test]
|
||
#[ignore]
|
||
fn live_app_menus() {
|
||
let pid: i32 = std::env::var("ANYBOT_MENU_PID").ok().and_then(|p| p.parse().ok()).unwrap_or(0);
|
||
let t = std::time::Instant::now();
|
||
let m = super::app_menus(pid);
|
||
println!("{} entries in {} ms", m.len(), t.elapsed().as_millis());
|
||
for e in m.iter().take(40) {
|
||
println!("{} {} [{}] {:?} {} {:?}", e.depth, e.label, e.shortcut, e.rect, if e.enabled { "" } else { "停用" }, e.alternate);
|
||
}
|
||
}
|
||
|
||
/// By hand: `cargo test live_person_chrome -- --ignored --nocapture`. Prints counts only.
|
||
#[test]
|
||
#[ignore]
|
||
fn live_person_chrome() {
|
||
println!("trusted={}", super::trusted(false));
|
||
if super::trusted(false) {
|
||
match super::read_person_browser("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", 5000) {
|
||
Ok(p) => println!("lines={} title_chars={} url_ok={}", p.text.lines().count(), p.title.chars().count(), p.url.starts_with("http")),
|
||
Err(e) => println!("err={e}"),
|
||
}
|
||
}
|
||
}
|
||
}
|