AnyBot/src/axread.rs

// Read the person's own browser window through macOS accessibility, the way Grok Bot's Computer
// Use helper (CUGrokBotService) sees their screen: no browser setting ("允許來自 Apple 事件的
// JavaScript") and no remote control of Chrome, so it works in whichever Chrome profile the person
// has open. macOS asks once to let AnyBot use 輔助使用 (Accessibility), as it does for that helper.
//
// Decompiled rules (CUGrokBotService, CUCore):
// - `CURendererApp.isRenderer(bundleIdentifier:executablePath:)` @0x1000a1b6c: an app is a
//   Chromium/Electron "renderer" if its bundle id is `com.github.Electron` or `com.google.Chrome`,
//   starts with `com.google.Chrome.`, its executable path contains
//   `/Electron.app/Contents/MacOS/Electron`, or `isRendererBundle(atPath:)` says so.
// - `CURendererApp.isRendererBundle(atPath:)` @0x1000a19ec: `<bundle>/Contents/Frameworks/` holds
//   one of 3 frameworks: Electron Framework, Chromium Embedded Framework, Google Chrome Framework.
// - `CURendererAccessibility.writeFlags(on:)` @0x1000a1554: on such apps sets
//   `AXManualAccessibility` then `AXEnhancedUserInterface` on the app's accessibility element, so
//   their web content shows up in the accessibility tree.

/// Bundle ids that are renderers outright (isRenderer @0x1000a1b6c).
pub const RENDERER_BUNDLE_IDS: [&str; 2] = ["com.github.Electron", "com.google.Chrome"];
/// Bundle id prefix for Chrome's channels and helpers (isRenderer @0x1000a1b6c).
pub const RENDERER_BUNDLE_PREFIX: &str = "com.google.Chrome.";
/// Executable path piece of a bare Electron app (isRenderer @0x1000a1b6c).
pub const ELECTRON_EXE_PIECE: &str = "/Electron.app/Contents/MacOS/Electron";
/// Frameworks that mark a Chromium-based bundle (isRendererBundle @0x1000a19ec).
pub const RENDERER_FRAMEWORKS: [&str; 3] = ["Electron Framework.framework", "Chromium Embedded Framework.framework", "Google Chrome Framework.framework"];
/// The two flags written on renderer apps, in order (writeFlags @0x1000a1554).
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
pub const RENDERER_AX_FLAGS: [&str; 2] = ["AXManualAccessibility", "AXEnhancedUserInterface"];
/// Most accessibility elements walked for one read (estimated).
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
pub const MAX_AX_NODES: usize = 20_000;
/// Deepest accessibility nesting followed (estimated).
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
const MAX_AX_DEPTH: usize = 80;
/// Tries while Chrome builds its accessibility tree after the flags are set (estimated).
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
const READ_TRIES: usize = 5;
/// Pause between those tries (estimated).
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
const RETRY_PAUSE_MS: u64 = 700;
/// Longest wait for one accessibility answer (estimated).
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
const AX_TIMEOUT_SECS: f32 = 2.0;

/// isRenderer(bundleIdentifier:executablePath:) + isRendererBundle(atPath:), with the framework
/// check passed in so it can be tested.
pub fn is_renderer(bundle_id: Option<&str>, exe_path: Option<&str>, has_framework: impl Fn(&str) -> bool) -> bool {
    if let Some(id) = bundle_id {
        if RENDERER_BUNDLE_IDS.contains(&id) || id.starts_with(RENDERER_BUNDLE_PREFIX) {
            return true;
        }
    }
    if let Some(p) = exe_path {
        if p.contains(ELECTRON_EXE_PIECE) {
            return true;
        }
    }
    RENDERER_FRAMEWORKS.iter().any(|f| has_framework(f))
}

/// The `.app` folder an executable lives in.
pub fn bundle_of(exe: &str) -> Option<&str> {
    exe.find(".app/").map(|i| &exe[..i + 4])
}

/// The person's own browser process (not AnyBot's own window, which runs from AnyBot's folder).
pub fn pick_person_pid(ps: &str, exe: &str) -> Option<i32> {
    ps.lines().find_map(|l| {
        let l = l.trim_start();
        let (pid, args) = l.split_once(' ')?;
        let args = args.trim_start();
        let own = args.strip_prefix(exe)?;
        (own.is_empty() || own.starts_with(' ')).then_some(())?;
        (!args.contains("/AnyBot/browser") && !args.contains(" --type=")).then_some(())?;
        pid.parse().ok()
    })
}

pub struct Page {
    pub title: String,
    pub url: String,
    pub text: String,
}

#[cfg(target_os = "macos")]
mod mac {
    use super::*;
    use std::ffi::{c_char, c_void, CString};

    type CFTypeRef = *const c_void;

    #[link(name = "ApplicationServices", kind = "framework")]
    extern "C" {
        fn AXIsProcessTrustedWithOptions(options: CFTypeRef) -> bool;
        fn AXUIElementCreateApplication(pid: i32) -> CFTypeRef;
        fn AXUIElementSetAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: CFTypeRef) -> i32;
        fn AXUIElementCopyAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: *mut CFTypeRef) -> i32;
        fn AXUIElementSetMessagingTimeout(el: CFTypeRef, secs: f32) -> i32;
        static kAXTrustedCheckOptionPrompt: CFTypeRef;
    }

    #[link(name = "CoreFoundation", kind = "framework")]
    extern "C" {
        fn CFStringCreateWithCString(alloc: CFTypeRef, s: *const c_char, enc: u32) -> CFTypeRef;
        fn CFRelease(x: CFTypeRef);
        fn CFRetain(x: CFTypeRef) -> CFTypeRef;
        fn CFGetTypeID(x: CFTypeRef) -> usize;
        fn CFStringGetTypeID() -> usize;
        fn CFArrayGetTypeID() -> usize;
        fn CFURLGetTypeID() -> usize;
        fn CFArrayGetCount(a: CFTypeRef) -> isize;
        fn CFArrayGetValueAtIndex(a: CFTypeRef, i: isize) -> CFTypeRef;
        fn CFStringGetLength(s: CFTypeRef) -> isize;
        fn CFStringGetMaximumSizeForEncoding(len: isize, enc: u32) -> isize;
        fn CFStringGetCString(s: CFTypeRef, buf: *mut c_char, size: isize, enc: u32) -> bool;
        fn CFURLGetString(u: CFTypeRef) -> CFTypeRef;
        fn CFDictionaryCreate(alloc: CFTypeRef, keys: *const CFTypeRef, values: *const CFTypeRef, n: isize, kcb: *const c_void, vcb: *const c_void) -> CFTypeRef;
        static kCFBooleanTrue: CFTypeRef;
        static kCFTypeDictionaryKeyCallBacks: [usize; 6];
        static kCFTypeDictionaryValueCallBacks: [usize; 5];
    }

    const UTF8: u32 = 0x0800_0100;

    /// An owned Core Foundation value, released when dropped.
    struct Cf(CFTypeRef);
    impl Drop for Cf {
        fn drop(&mut self) {
            if !self.0.is_null() {
                unsafe { CFRelease(self.0) }
            }
        }
    }

    fn cfstr(s: &str) -> Cf {
        let c = CString::new(s).unwrap_or_default();
        Cf(unsafe { CFStringCreateWithCString(std::ptr::null(), c.as_ptr(), UTF8) })
    }

    fn string_of(v: CFTypeRef) -> Option<String> {
        unsafe {
            if v.is_null() || CFGetTypeID(v) != CFStringGetTypeID() {
                return None;
            }
            let size = CFStringGetMaximumSizeForEncoding(CFStringGetLength(v), UTF8) + 1;
            let mut buf = vec![0u8; size.max(1) as usize];
            if !CFStringGetCString(v, buf.as_mut_ptr() as *mut c_char, size, UTF8) {
                return None;
            }
            let end = buf.iter().position(|b| *b == 0).unwrap_or(buf.len());
            Some(String::from_utf8_lossy(&buf[..end]).into_owned())
        }
    }

    fn attr(el: CFTypeRef, name: &str) -> Option<Cf> {
        let key = cfstr(name);
        let mut out: CFTypeRef = std::ptr::null();
        let err = unsafe { AXUIElementCopyAttributeValue(el, key.0, &mut out) };
        (err == 0 && !out.is_null()).then(|| Cf(out))
    }

    fn attr_string(el: CFTypeRef, name: &str) -> Option<String> {
        attr(el, name).and_then(|v| string_of(v.0))
    }

    fn children(el: CFTypeRef) -> (Option<Cf>, Vec<CFTypeRef>) {
        let Some(arr) = attr(el, "AXChildren") else { return (None, vec![]) };
        unsafe {
            if CFGetTypeID(arr.0) != CFArrayGetTypeID() {
                return (Some(arr), vec![]);
            }
            let n = CFArrayGetCount(arr.0);
            let items = (0..n).map(|i| CFArrayGetValueAtIndex(arr.0, i)).collect();
            (Some(arr), items)
        }
    }

    /// Is AnyBot allowed to use accessibility? `prompt` makes macOS show its own question.
    pub fn trusted(prompt: bool) -> bool {
        unsafe {
            if !prompt {
                return AXIsProcessTrustedWithOptions(std::ptr::null());
            }
            let keys = [kAXTrustedCheckOptionPrompt];
            let values = [kCFBooleanTrue];
            let dict = Cf(CFDictionaryCreate(std::ptr::null(), keys.as_ptr(), values.as_ptr(), 1, kCFTypeDictionaryKeyCallBacks.as_ptr() as *const c_void, kCFTypeDictionaryValueCallBacks.as_ptr() as *const c_void));
            AXIsProcessTrustedWithOptions(dict.0)
        }
    }

    /// writeFlags(on:) @0x1000a1554: both flags on the app element, in the same order.
    fn write_flags(app: CFTypeRef) -> bool {
        RENDERER_AX_FLAGS.iter().fold(true, |ok, f| {
            let key = cfstr(f);
            let err = unsafe { AXUIElementSetAttributeValue(app, key.0, kCFBooleanTrue) };
            ok && err == 0
        })
    }

    fn find_role(el: CFTypeRef, role: &str, seen: &mut usize, depth: usize) -> Option<Cf> {
        if *seen >= MAX_AX_NODES || depth > MAX_AX_DEPTH {
            return None;
        }
        *seen += 1;
        let (_keep, kids) = children(el);
        for k in kids {
            if attr_string(k, "AXRole").as_deref() == Some(role) {
                unsafe { CFRetain(k) };
                return Some(Cf(k));
            }
            if let Some(found) = find_role(k, role, seen, depth + 1) {
                return Some(found);
            }
        }
        None
    }

    /// Visible text under the page: runs of static text become one line.
    fn collect(el: CFTypeRef, out: &mut Vec<String>, line: &mut String, seen: &mut usize, depth: usize, max_chars: usize, total: &mut usize) {
        if *seen >= MAX_AX_NODES || depth > MAX_AX_DEPTH || *total > max_chars {
            return;
        }
        *seen += 1;
        let (_keep, kids) = children(el);
        for k in kids {
            let role = attr_string(k, "AXRole").unwrap_or_default();
            if role == "AXStaticText" {
                if let Some(t) = attr_string(k, "AXValue") {
                    *total += t.chars().count();
                    line.push_str(&t);
                }
            } else {
                flush(out, line);
                collect(k, out, line, seen, depth + 1, max_chars, total);
                flush(out, line);
            }
        }
    }

    fn flush(out: &mut Vec<String>, line: &mut String) {
        let t = line.trim();
        if !t.is_empty() {
            out.push(t.to_string());
        }
        line.clear();
    }

    fn url_of(web: CFTypeRef) -> String {
        let Some(u) = attr(web, "AXURL") else { return String::new() };
        unsafe {
            if CFGetTypeID(u.0) == CFURLGetTypeID() {
                string_of(CFURLGetString(u.0)).unwrap_or_default()
            } else {
                string_of(u.0).unwrap_or_default()
            }
        }
    }

    pub fn read(pid: i32, renderer: bool, max_chars: usize) -> Result<Page, String> {
        let app = Cf(unsafe { AXUIElementCreateApplication(pid) });
        if app.0.is_null() {
            return Err("連不上你的瀏覽器".into());
        }
        unsafe { AXUIElementSetMessagingTimeout(app.0, AX_TIMEOUT_SECS) };
        if renderer {
            write_flags(app.0);
        }
        let mut last_title = String::new();
        for i in 0..READ_TRIES {
            if i > 0 {
                std::thread::sleep(std::time::Duration::from_millis(RETRY_PAUSE_MS));
            }
            let Some(win) = attr(app.0, "AXFocusedWindow").or_else(|| attr(app.0, "AXMainWindow")) else { continue };
            last_title = attr_string(win.0, "AXTitle").unwrap_or_default();
            let mut seen = 0;
            let Some(web) = find_role(win.0, "AXWebArea", &mut seen, 0) else { continue };
            let title = attr_string(web.0, "AXTitle").filter(|t| !t.is_empty()).unwrap_or_else(|| last_title.clone());
            let (mut lines, mut line, mut seen, mut total) = (vec![], String::new(), 0, 0);
            collect(web.0, &mut lines, &mut line, &mut seen, 0, max_chars, &mut total);
            flush(&mut lines, &mut line);
            if lines.is_empty() && i + 1 < READ_TRIES {
                continue;
            }
            return Ok(Page { title, url: url_of(web.0), text: lines.join("\n") });
        }
        if last_title.is_empty() {
            Err("你的瀏覽器現在沒有打開的視窗".into())
        } else {
            Err(format!("讀不到「{last_title}」的頁面內容"))
        }
    }
}

#[cfg(target_os = "macos")]
pub use mac::trusted;

#[cfg(not(target_os = "macos"))]
pub fn trusted(_prompt: bool) -> bool {
    false
}

/// Read the front window of the person's browser at `exe` (e.g. Chrome's executable).
pub fn read_person_browser(exe: &str, max_chars: usize) -> Result<Page, String> {
    if !cfg!(target_os = "macos") {
        return Err("只支援 macOS".into());
    }
    if !trusted(true) {
        let _ = crate::computer::spawn_reaped(&mut std::process::Command::new("open").arg("x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility"));
        return Err("macOS 還沒讓 AnyBot 使用「輔助使用」，所以看不到你的瀏覽器。我已經打開「系統設定 → 隱私權與安全性 → 輔助使用」：把 AnyBot（或「終端機」）打開，然後再叫我試一次。只要開一次。".into());
    }
    let ps = std::process::Command::new("ps").args(["-axo", "pid=,args="]).output().map(|o| String::from_utf8_lossy(&o.stdout).into_owned()).unwrap_or_default();
    let Some(pid) = pick_person_pid(&ps, exe) else { return Err("你的瀏覽器沒有開著。請先打開它（或叫我用 browser_open 打開網址）。".into()) };
    let bundle = bundle_of(exe).unwrap_or("");
    let bundle_id = std::process::Command::new("plutil")
        .args(["-extract", "CFBundleIdentifier", "raw", "-o", "-", &format!("{bundle}/Contents/Info.plist")])
        .output()
        .ok()
        .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
        .filter(|s| !s.is_empty());
    let renderer = is_renderer(bundle_id.as_deref(), Some(exe), |f| std::path::Path::new(bundle).join("Contents/Frameworks").join(f).exists());
    #[cfg(target_os = "macos")]
    {
        mac::read(pid, renderer, max_chars)
    }
    #[cfg(not(target_os = "macos"))]
    {
        let _ = (pid, renderer, max_chars);
        Err("只支援 macOS".into())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn renderer_apps_follow_the_original_rule() {
        let none = |_: &str| false;
        assert!(is_renderer(Some("com.google.Chrome"), None, none));
        assert!(is_renderer(Some("com.google.Chrome.canary"), None, none));
        assert!(is_renderer(Some("com.github.Electron"), None, none));
        assert!(is_renderer(None, Some("/tmp/x/Electron.app/Contents/MacOS/Electron"), none));
        assert!(!is_renderer(Some("com.google.Chromeish"), None, none));
        assert!(!is_renderer(Some("com.apple.Safari"), Some("/Applications/Safari.app/Contents/MacOS/Safari"), none));
        // Brave / Edge carry their own framework names, so only the framework check could catch them.
        assert!(is_renderer(Some("com.example.app"), None, |f| f == "Electron Framework.framework"));
    }

    #[test]
    fn the_persons_browser_is_not_anybots_window() {
        let exe = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
        let ps = format!(
            "  101 {exe} --user-data-dir=/Users/me/Library/Application Support/AnyBot/browser --remote-debugging-port=9333\n  102 {exe} --type=utility\n  205 {exe}\n  300 {exe} --type=renderer"
        );
        assert_eq!(pick_person_pid(&ps, exe), Some(205));
        assert_eq!(pick_person_pid("  1 /Applications/Safari.app/Contents/MacOS/Safari", exe), None);
    }

    #[test]
    fn bundle_is_found_from_the_executable() {
        assert_eq!(bundle_of("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"), Some("/Applications/Google Chrome.app"));
        assert_eq!(bundle_of("/usr/bin/true"), None);
    }
}

#[cfg(test)]
mod live {
    /// By hand: `cargo test live_person_chrome -- --ignored --nocapture`. Prints counts only.
    #[test]
    #[ignore]
    fn live_person_chrome() {
        println!("trusted={}", super::trusted(false));
        if super::trusted(false) {
            match super::read_person_browser("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", 5000) {
                Ok(p) => println!("lines={} title_chars={} url_ok={}", p.text.lines().count(), p.title.chars().count(), p.url.starts_with("http")),
                Err(e) => println!("err={e}"),
            }
        }
    }
}