// Read the person's own browser window through macOS accessibility, the way Grok Bot's Computer // Use helper (CUGrokBotService) sees their screen: no browser setting ("允許來自 Apple 事件的 // JavaScript") and no remote control of Chrome, so it works in whichever Chrome profile the person // has open. macOS asks once to let AnyBot use 輔助使用 (Accessibility), as it does for that helper. // // Decompiled rules (CUGrokBotService, CUCore): // - `CURendererApp.isRenderer(bundleIdentifier:executablePath:)` @0x1000a1b6c: an app is a // Chromium/Electron "renderer" if its bundle id is `com.github.Electron` or `com.google.Chrome`, // starts with `com.google.Chrome.`, its executable path contains // `/Electron.app/Contents/MacOS/Electron`, or `isRendererBundle(atPath:)` says so. // - `CURendererApp.isRendererBundle(atPath:)` @0x1000a19ec: `/Contents/Frameworks/` holds // one of 3 frameworks: Electron Framework, Chromium Embedded Framework, Google Chrome Framework. // - `CURendererAccessibility.writeFlags(on:)` @0x1000a1554: on such apps sets // `AXManualAccessibility` then `AXEnhancedUserInterface` on the app's accessibility element, so // their web content shows up in the accessibility tree. /// Bundle ids that are renderers outright (isRenderer @0x1000a1b6c). pub const RENDERER_BUNDLE_IDS: [&str; 2] = ["com.github.Electron", "com.google.Chrome"]; /// Bundle id prefix for Chrome's channels and helpers (isRenderer @0x1000a1b6c). pub const RENDERER_BUNDLE_PREFIX: &str = "com.google.Chrome."; /// Executable path piece of a bare Electron app (isRenderer @0x1000a1b6c). pub const ELECTRON_EXE_PIECE: &str = "/Electron.app/Contents/MacOS/Electron"; /// Frameworks that mark a Chromium-based bundle (isRendererBundle @0x1000a19ec). pub const RENDERER_FRAMEWORKS: [&str; 3] = ["Electron Framework.framework", "Chromium Embedded Framework.framework", "Google Chrome Framework.framework"]; /// The two flags written on renderer apps, in order (writeFlags @0x1000a1554). #[cfg_attr(not(target_os = "macos"), allow(dead_code))] pub const RENDERER_AX_FLAGS: [&str; 2] = ["AXManualAccessibility", "AXEnhancedUserInterface"]; /// Most accessibility elements walked for one read (estimated). #[cfg_attr(not(target_os = "macos"), allow(dead_code))] pub const MAX_AX_NODES: usize = 20_000; /// Deepest accessibility nesting followed (estimated). #[cfg_attr(not(target_os = "macos"), allow(dead_code))] const MAX_AX_DEPTH: usize = 80; /// Tries while Chrome builds its accessibility tree after the flags are set (estimated). #[cfg_attr(not(target_os = "macos"), allow(dead_code))] const READ_TRIES: usize = 5; /// Pause between those tries (estimated). #[cfg_attr(not(target_os = "macos"), allow(dead_code))] const RETRY_PAUSE_MS: u64 = 700; /// Longest wait for one accessibility answer (estimated). #[cfg_attr(not(target_os = "macos"), allow(dead_code))] const AX_TIMEOUT_SECS: f32 = 2.0; /// isRenderer(bundleIdentifier:executablePath:) + isRendererBundle(atPath:), with the framework /// check passed in so it can be tested. pub fn is_renderer(bundle_id: Option<&str>, exe_path: Option<&str>, has_framework: impl Fn(&str) -> bool) -> bool { if let Some(id) = bundle_id { if RENDERER_BUNDLE_IDS.contains(&id) || id.starts_with(RENDERER_BUNDLE_PREFIX) { return true; } } if let Some(p) = exe_path { if p.contains(ELECTRON_EXE_PIECE) { return true; } } RENDERER_FRAMEWORKS.iter().any(|f| has_framework(f)) } /// The `.app` folder an executable lives in. pub fn bundle_of(exe: &str) -> Option<&str> { exe.find(".app/").map(|i| &exe[..i + 4]) } /// The person's own browser process (not AnyBot's own window, which runs from AnyBot's folder). pub fn pick_person_pid(ps: &str, exe: &str) -> Option { ps.lines().find_map(|l| { let l = l.trim_start(); let (pid, args) = l.split_once(' ')?; let args = args.trim_start(); let own = args.strip_prefix(exe)?; (own.is_empty() || own.starts_with(' ')).then_some(())?; (!args.contains("/AnyBot/browser") && !args.contains(" --type=")).then_some(())?; pid.parse().ok() }) } pub struct Page { pub title: String, pub url: String, pub text: String, } #[cfg(target_os = "macos")] mod mac { use super::*; use std::ffi::{c_char, c_void, CString}; type CFTypeRef = *const c_void; #[link(name = "ApplicationServices", kind = "framework")] extern "C" { fn AXIsProcessTrustedWithOptions(options: CFTypeRef) -> bool; fn AXUIElementCreateApplication(pid: i32) -> CFTypeRef; fn AXUIElementSetAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: CFTypeRef) -> i32; fn AXUIElementCopyAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: *mut CFTypeRef) -> i32; fn AXUIElementSetMessagingTimeout(el: CFTypeRef, secs: f32) -> i32; static kAXTrustedCheckOptionPrompt: CFTypeRef; } #[link(name = "CoreFoundation", kind = "framework")] extern "C" { fn CFStringCreateWithCString(alloc: CFTypeRef, s: *const c_char, enc: u32) -> CFTypeRef; fn CFRelease(x: CFTypeRef); fn CFRetain(x: CFTypeRef) -> CFTypeRef; fn CFGetTypeID(x: CFTypeRef) -> usize; fn CFStringGetTypeID() -> usize; fn CFArrayGetTypeID() -> usize; fn CFURLGetTypeID() -> usize; fn CFArrayGetCount(a: CFTypeRef) -> isize; fn CFArrayGetValueAtIndex(a: CFTypeRef, i: isize) -> CFTypeRef; fn CFStringGetLength(s: CFTypeRef) -> isize; fn CFStringGetMaximumSizeForEncoding(len: isize, enc: u32) -> isize; fn CFStringGetCString(s: CFTypeRef, buf: *mut c_char, size: isize, enc: u32) -> bool; fn CFURLGetString(u: CFTypeRef) -> CFTypeRef; fn CFDictionaryCreate(alloc: CFTypeRef, keys: *const CFTypeRef, values: *const CFTypeRef, n: isize, kcb: *const c_void, vcb: *const c_void) -> CFTypeRef; static kCFBooleanTrue: CFTypeRef; static kCFTypeDictionaryKeyCallBacks: [usize; 6]; static kCFTypeDictionaryValueCallBacks: [usize; 5]; } const UTF8: u32 = 0x0800_0100; /// An owned Core Foundation value, released when dropped. struct Cf(CFTypeRef); impl Drop for Cf { fn drop(&mut self) { if !self.0.is_null() { unsafe { CFRelease(self.0) } } } } fn cfstr(s: &str) -> Cf { let c = CString::new(s).unwrap_or_default(); Cf(unsafe { CFStringCreateWithCString(std::ptr::null(), c.as_ptr(), UTF8) }) } fn string_of(v: CFTypeRef) -> Option { unsafe { if v.is_null() || CFGetTypeID(v) != CFStringGetTypeID() { return None; } let size = CFStringGetMaximumSizeForEncoding(CFStringGetLength(v), UTF8) + 1; let mut buf = vec![0u8; size.max(1) as usize]; if !CFStringGetCString(v, buf.as_mut_ptr() as *mut c_char, size, UTF8) { return None; } let end = buf.iter().position(|b| *b == 0).unwrap_or(buf.len()); Some(String::from_utf8_lossy(&buf[..end]).into_owned()) } } fn attr(el: CFTypeRef, name: &str) -> Option { let key = cfstr(name); let mut out: CFTypeRef = std::ptr::null(); let err = unsafe { AXUIElementCopyAttributeValue(el, key.0, &mut out) }; (err == 0 && !out.is_null()).then(|| Cf(out)) } fn attr_string(el: CFTypeRef, name: &str) -> Option { attr(el, name).and_then(|v| string_of(v.0)) } fn children(el: CFTypeRef) -> (Option, Vec) { let Some(arr) = attr(el, "AXChildren") else { return (None, vec![]) }; unsafe { if CFGetTypeID(arr.0) != CFArrayGetTypeID() { return (Some(arr), vec![]); } let n = CFArrayGetCount(arr.0); let items = (0..n).map(|i| CFArrayGetValueAtIndex(arr.0, i)).collect(); (Some(arr), items) } } /// Is AnyBot allowed to use accessibility? `prompt` makes macOS show its own question. pub fn trusted(prompt: bool) -> bool { unsafe { if !prompt { return AXIsProcessTrustedWithOptions(std::ptr::null()); } let keys = [kAXTrustedCheckOptionPrompt]; let values = [kCFBooleanTrue]; let dict = Cf(CFDictionaryCreate(std::ptr::null(), keys.as_ptr(), values.as_ptr(), 1, kCFTypeDictionaryKeyCallBacks.as_ptr() as *const c_void, kCFTypeDictionaryValueCallBacks.as_ptr() as *const c_void)); AXIsProcessTrustedWithOptions(dict.0) } } /// writeFlags(on:) @0x1000a1554: both flags on the app element, in the same order. fn write_flags(app: CFTypeRef) -> bool { RENDERER_AX_FLAGS.iter().fold(true, |ok, f| { let key = cfstr(f); let err = unsafe { AXUIElementSetAttributeValue(app, key.0, kCFBooleanTrue) }; ok && err == 0 }) } fn find_role(el: CFTypeRef, role: &str, seen: &mut usize, depth: usize) -> Option { if *seen >= MAX_AX_NODES || depth > MAX_AX_DEPTH { return None; } *seen += 1; let (_keep, kids) = children(el); for k in kids { if attr_string(k, "AXRole").as_deref() == Some(role) { unsafe { CFRetain(k) }; return Some(Cf(k)); } if let Some(found) = find_role(k, role, seen, depth + 1) { return Some(found); } } None } /// Visible text under the page: runs of static text become one line. fn collect(el: CFTypeRef, out: &mut Vec, line: &mut String, seen: &mut usize, depth: usize, max_chars: usize, total: &mut usize) { if *seen >= MAX_AX_NODES || depth > MAX_AX_DEPTH || *total > max_chars { return; } *seen += 1; let (_keep, kids) = children(el); for k in kids { let role = attr_string(k, "AXRole").unwrap_or_default(); if role == "AXStaticText" { if let Some(t) = attr_string(k, "AXValue") { *total += t.chars().count(); line.push_str(&t); } } else { flush(out, line); collect(k, out, line, seen, depth + 1, max_chars, total); flush(out, line); } } } fn flush(out: &mut Vec, line: &mut String) { let t = line.trim(); if !t.is_empty() { out.push(t.to_string()); } line.clear(); } fn url_of(web: CFTypeRef) -> String { let Some(u) = attr(web, "AXURL") else { return String::new() }; unsafe { if CFGetTypeID(u.0) == CFURLGetTypeID() { string_of(CFURLGetString(u.0)).unwrap_or_default() } else { string_of(u.0).unwrap_or_default() } } } pub fn read(pid: i32, renderer: bool, max_chars: usize) -> Result { let app = Cf(unsafe { AXUIElementCreateApplication(pid) }); if app.0.is_null() { return Err("連不上你的瀏覽器".into()); } unsafe { AXUIElementSetMessagingTimeout(app.0, AX_TIMEOUT_SECS) }; if renderer { write_flags(app.0); } let mut last_title = String::new(); for i in 0..READ_TRIES { if i > 0 { std::thread::sleep(std::time::Duration::from_millis(RETRY_PAUSE_MS)); } let Some(win) = attr(app.0, "AXFocusedWindow").or_else(|| attr(app.0, "AXMainWindow")) else { continue }; last_title = attr_string(win.0, "AXTitle").unwrap_or_default(); let mut seen = 0; let Some(web) = find_role(win.0, "AXWebArea", &mut seen, 0) else { continue }; let title = attr_string(web.0, "AXTitle").filter(|t| !t.is_empty()).unwrap_or_else(|| last_title.clone()); let (mut lines, mut line, mut seen, mut total) = (vec![], String::new(), 0, 0); collect(web.0, &mut lines, &mut line, &mut seen, 0, max_chars, &mut total); flush(&mut lines, &mut line); if lines.is_empty() && i + 1 < READ_TRIES { continue; } return Ok(Page { title, url: url_of(web.0), text: lines.join("\n") }); } if last_title.is_empty() { Err("你的瀏覽器現在沒有打開的視窗".into()) } else { Err(format!("讀不到「{last_title}」的頁面內容")) } } } #[cfg(target_os = "macos")] pub use mac::trusted; #[cfg(not(target_os = "macos"))] pub fn trusted(_prompt: bool) -> bool { false } /// Read the front window of the person's browser at `exe` (e.g. Chrome's executable). pub fn read_person_browser(exe: &str, max_chars: usize) -> Result { if !cfg!(target_os = "macos") { return Err("只支援 macOS".into()); } if !trusted(false) { crate::computer::ask_permission("Privacy_Accessibility"); return Err("macOS 還沒讓 AnyBot 使用「輔助使用」,所以看不到你的瀏覽器。我已經打開「系統設定 → 隱私權與安全性 → 輔助使用」:把 AnyBot(或「終端機」)打開,然後再叫我試一次。只要開一次。".into()); } let ps = std::process::Command::new("ps").args(["-axo", "pid=,args="]).output().map(|o| String::from_utf8_lossy(&o.stdout).into_owned()).unwrap_or_default(); let Some(pid) = pick_person_pid(&ps, exe) else { return Err("你的瀏覽器沒有開著。請先打開它(或叫我用 browser_open 打開網址)。".into()) }; let bundle = bundle_of(exe).unwrap_or(""); let bundle_id = std::process::Command::new("plutil") .args(["-extract", "CFBundleIdentifier", "raw", "-o", "-", &format!("{bundle}/Contents/Info.plist")]) .output() .ok() .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) .filter(|s| !s.is_empty()); let renderer = is_renderer(bundle_id.as_deref(), Some(exe), |f| std::path::Path::new(bundle).join("Contents/Frameworks").join(f).exists()); #[cfg(target_os = "macos")] { mac::read(pid, renderer, max_chars) } #[cfg(not(target_os = "macos"))] { let _ = (pid, renderer, max_chars); Err("只支援 macOS".into()) } } #[cfg(test)] mod tests { use super::*; #[test] fn renderer_apps_follow_the_original_rule() { let none = |_: &str| false; assert!(is_renderer(Some("com.google.Chrome"), None, none)); assert!(is_renderer(Some("com.google.Chrome.canary"), None, none)); assert!(is_renderer(Some("com.github.Electron"), None, none)); assert!(is_renderer(None, Some("/tmp/x/Electron.app/Contents/MacOS/Electron"), none)); assert!(!is_renderer(Some("com.google.Chromeish"), None, none)); assert!(!is_renderer(Some("com.apple.Safari"), Some("/Applications/Safari.app/Contents/MacOS/Safari"), none)); // Brave / Edge carry their own framework names, so only the framework check could catch them. assert!(is_renderer(Some("com.example.app"), None, |f| f == "Electron Framework.framework")); } #[test] fn the_persons_browser_is_not_anybots_window() { let exe = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"; let ps = format!( " 101 {exe} --user-data-dir=/Users/me/Library/Application Support/AnyBot/browser --remote-debugging-port=9333\n 102 {exe} --type=utility\n 205 {exe}\n 300 {exe} --type=renderer" ); assert_eq!(pick_person_pid(&ps, exe), Some(205)); assert_eq!(pick_person_pid(" 1 /Applications/Safari.app/Contents/MacOS/Safari", exe), None); } #[test] fn bundle_is_found_from_the_executable() { assert_eq!(bundle_of("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"), Some("/Applications/Google Chrome.app")); assert_eq!(bundle_of("/usr/bin/true"), None); } } #[cfg(test)] mod live { /// By hand: `cargo test live_person_chrome -- --ignored --nocapture`. Prints counts only. #[test] #[ignore] fn live_person_chrome() { println!("trusted={}", super::trusted(false)); if super::trusted(false) { match super::read_person_browser("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", 5000) { Ok(p) => println!("lines={} title_chars={} url_ok={}", p.text.lines().count(), p.title.chars().count(), p.url.starts_with("http")), Err(e) => println!("err={e}"), } } } }