AnyBot/src/desktop.rs

// Reading other apps through the OS accessibility APIs, with vercel-labs/agent-desktop compiled
// in (AX on macOS, AT-SPI on Linux, UI Automation on Windows): an app's controls and text as
// numbered lines (e1, e2…), each number remembered with where it is on the screen. That is all
// it does. Acting on a number is a real click or keystroke at that spot (computer.rs), the same
// way as acting on screenshot coordinates: one way to act on a screen, whatever the bot read it
// with. (agent-desktop's own "semantic" actions were a second way with its own key names and
// failures: typing into a terminal, "/" and "`" from a background thread, focus rules.)

use agent_desktop_core as ad;
use serde_json::Value;

fn adapter() -> Box<dyn ad::PlatformAdapter> {
    #[cfg(target_os = "macos")]
    {
        Box::new(agent_desktop_macos::MacOSAdapter::new())
    }
    #[cfg(target_os = "linux")]
    {
        Box::new(agent_desktop_linux::LinuxAdapter::new())
    }
    #[cfg(target_os = "windows")]
    {
        Box::new(agent_desktop_windows::WindowsAdapter::new())
    }
}

fn err(e: ad::AppError) -> String {
    let mut s = format!("{e}（{}）", e.code());
    if let Some(hint) = e.suggestion() {
        s.push_str(&format!("：{hint}"));
    }
    s
}

fn st(v: &Value, k: &str) -> String {
    v.get(k).map(|x| x.as_str().map(String::from).unwrap_or_else(|| if x.is_null() { String::new() } else { x.to_string() })).unwrap_or_default().trim().to_string()
}

fn opt(v: &Value, k: &str) -> Option<String> {
    Some(st(v, k)).filter(|s| !s.is_empty())
}

/// One agent-desktop command (the CLI's names): apps, windows, snapshot, launch, permissions.
pub fn run(cmd: &str, a: &Value) -> Result<Value, String> {
    use ad::commands as c;
    let ad = adapter();
    let ad = ad.as_ref();
    let ctx = ad::CommandContext::default();
    let out = match cmd {
        "apps" => c::list_apps::execute(c::list_apps::ListAppsArgs { app: opt(a, "app"), timeout_ms: None }, ad),
        "windows" => c::list_windows::execute(c::list_windows::ListWindowsArgs { app: opt(a, "app"), timeout_ms: None }, ad),
        "snapshot" => c::snapshot::execute(
            c::snapshot::SnapshotArgs {
                app: opt(a, "app"),
                window_id: opt(a, "window_id"),
                max_depth: a["max_depth"].as_u64().unwrap_or(30).min(60) as u8,
                include_bounds: a["bounds"].as_bool().unwrap_or(true),
                interactive_only: a["interactive_only"].as_bool().unwrap_or(true),
                compact: true,
                surface: a["surface"].as_str().and_then(|n| serde_json::from_value(Value::String(n.to_string())).ok()).unwrap_or_default(),
                skeleton: false,
                root_ref: opt(a, "root"),
                timeout_ms: None,
                force_electron_a11y: true,
            },
            ad,
            &ctx,
        ),
        "launch" => c::launch::execute(c::launch::LaunchArgs { app: st(a, "app"), options: ad::system::launch_options::LaunchOptions { activate: true, ..Default::default() } }, ad),
        "permissions" => {
            let d = ad::Deadline::standard().map_err(|e| err(e.into()))?;
            return ad.permission_report(d).map(|r| Value::String(format!("{r:?}"))).map_err(|e| err(e.into()));
        }
        other => return Err(format!("agent-desktop 沒有「{other}」這個動作")),
    };
    out.map_err(err)
}

// ---------- what the bot uses ----------

fn role_name(role: &str) -> &str {
    match role {
        "button" => "按鈕",
        "link" => "連結",
        "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield" => "輸入框",
        "checkbox" | "switch" | "toggle" => "勾選框",
        "radiobutton" => "選項",
        "popupbutton" | "menubutton" => "選單按鈕",
        "disclosure" => "展開按鈕",
        "colorwell" => "顏色",
        "datefield" | "timefield" => "日期",
        "menuitem" => "選單項目",
        "tab" => "分頁",
        "row" | "cell" | "listitem" | "treeitem" | "outlinerow" => "項目",
        "slider" => "滑桿",
        "statictext" | "text" => "文字",
        "image" => "圖片",
        "heading" => "標題",
        other => other,
    }
}

/// Roles a person clicks or types into (the rest is text to read).
fn active(role: &str) -> bool {
    matches!(
        role,
        "button" | "link" | "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield" | "checkbox" | "switch" | "toggle" | "radiobutton" | "popupbutton" | "menubutton" | "menuitem" | "tab" | "row" | "cell" | "listitem" | "treeitem" | "outlinerow" | "slider" | "incrementor" | "disclosure" | "colorwell" | "datefield" | "timefield"
    )
}

/// Longest value shown whole; longer ones (a terminal's scrollback, a document) keep their
/// start and their end, where a terminal's prompt and newest output are.
const VALUE_HEAD: usize = 160;
const VALUE_TAIL: usize = 400;

fn shorten(v: &str) -> String {
    let n = v.chars().count();
    if n <= VALUE_HEAD + VALUE_TAIL + 20 {
        return v.to_string();
    }
    let head: String = v.chars().take(VALUE_HEAD).collect();
    let tail: String = v.chars().skip(n - VALUE_TAIL).collect();
    format!("{head} …（中間省略 {} 字）… {tail}", n - VALUE_HEAD - VALUE_TAIL)
}

/// The first words inside a node (a row's label).
fn words_inside(n: &Value) -> String {
    n["children"].as_array().into_iter().flatten().find_map(|c| {
        let w = c["name"].as_str().or_else(|| c["value"].as_str()).unwrap_or("").trim().to_string();
        if w.is_empty() { Some(words_inside(c)).filter(|w| !w.is_empty()) } else { Some(w) }
    }).unwrap_or_default()
}

/// Something inside this node is a control with a name.
fn named_inside(n: &Value) -> bool {
    n["children"].as_array().into_iter().flatten().any(|c| (c["ref_id"].is_string() && active(c["role"].as_str().unwrap_or("")) && !c["name"].as_str().unwrap_or("").trim().is_empty()) || named_inside(c))
}

/// The controls and text of a snapshot, with where each is on the screen (nodes the app gave
/// no place are left out: they can't be shown or clicked).
pub fn elements_of(tree: &Value, app: &str) -> Vec<crate::marks::Element> {
    fn walk(n: &Value, out: &mut Vec<crate::marks::Element>, app: &str) {
        if out.len() >= 600 {
            return;
        }
        let role = n["role"].as_str().unwrap_or("");
        let mut name = n["name"].as_str().unwrap_or("").trim().to_string();
        // An icon-only control often has its words only as a description (the save panel's
        // triangle: 「顯示較多選項」).
        if name.is_empty() && n["ref_id"].is_string() && active(role) {
            name = n["description"].as_str().unwrap_or("").trim().to_string();
        }
        // A nameless row, cell or list item goes by the words inside it (an icon-only button
        // keeps its number without a name: it can still be clicked).
        let row_like = matches!(role, "row" | "cell" | "treeitem" | "outlinerow" | "listitem");
        // …unless a named control inside it is what gets clicked (System Settings' sidebar).
        let wraps_named = name.is_empty() && named_inside(n);
        if name.is_empty() && row_like && !wraps_named {
            name = words_inside(n);
        }
        let value = n["value"].as_str().map(str::to_string).or_else(|| n["value"].as_f64().map(|f| f.to_string())).unwrap_or_default();
        let b = &n["bounds"];
        if let (Some(x), Some(y), Some(w), Some(h)) = (b["x"].as_f64(), b["y"].as_f64(), b["width"].as_f64(), b["height"].as_f64()) {
            let is_active = n["ref_id"].is_string() && active(role);
            let has = |k: &str| n["states"].as_array().map(|a| a.iter().any(|x| x == k)).unwrap_or(false) || n[k] == true;
            // Hidden or scrolled out of view: nothing a click could reach. A nameless row whose
            // own button has the name (System Settings' sidebar) needs no number of its own: a
            // list of 「項目」 told the bot nothing.
            let empty = name.is_empty() && value.trim().is_empty();
            let unseen = has("hidden") || has("offscreen") || (is_active && wraps_named) || (row_like && empty);
            if !unseen && w >= 1.0 && h >= 1.0 && x + w > 0.0 && y + h > 0.0 && (is_active || !name.is_empty() || !value.trim().is_empty()) && role != "window" && role != "group" {
                let mut flags = vec![];
                for (k, word) in [("checked", "已勾"), ("selected", "已選"), ("disabled", "停用"), ("focused", "游標在這")] {
                    if has(k) && (k != "focused" || role.contains("text") || role.contains("field")) {
                        flags.push(word);
                    }
                }
                out.push(crate::marks::Element { kind: role_name(role).to_string(), name, value: shorten(value.trim()), x, y, w, h, active: is_active, flags, window: app.to_string() });
            }
        }
        for c in n["children"].as_array().into_iter().flatten() {
            walk(c, out, app);
        }
    }
    let mut out = vec![];
    walk(tree, &mut out, app);
    out
}

/// The app in front (or `app`): its name, its window's number (macOS: the CGWindowID) and its
/// controls and text, in screen coordinates.
pub fn elements(app: &str) -> Result<(String, Option<u32>, Vec<crate::marks::Element>), String> {
    let mut args = serde_json::json!({"interactive_only": false, "max_depth": 40, "bounds": true});
    if !app.trim().is_empty() {
        args["app"] = Value::String(app.trim().to_string());
    }
    let v = run("snapshot", &args)?;
    let name = v["app"].as_str().unwrap_or(app).to_string();
    let els = elements_of(&v["tree"], &name);
    let window = v["window"]["id"].as_str().and_then(|w| w.strip_prefix("w-")).and_then(|n| n.parse().ok());
    Ok((name, window, els))
}

/// Open an app, or bring it to the front when it runs (macOS uses LaunchServices instead).
#[cfg(not(target_os = "macos"))]
pub fn launch(app: &str) -> Result<(), String> {
    run("launch", &serde_json::json!({"app": app})).map(|_| ())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn hidden_and_offscreen_controls_get_no_number() {
        let tree = serde_json::json!({"role": "window", "children": [
            {"role": "button", "name": "Close", "ref_id": "a", "bounds": {"x": 8.0, "y": 38.0, "width": 16.0, "height": 16.0}},
            {"role": "scrollbar", "name": "bar", "ref_id": "b", "states": ["disabled", "hidden"], "bounds": {"x": 1902.0, "y": 61.0, "width": 19.0, "height": 1000.0}},
            {"role": "link", "name": "Below the fold", "ref_id": "c", "states": ["offscreen"], "bounds": {"x": 20.0, "y": 2400.0, "width": 80.0, "height": 20.0}}
        ]});
        let names: Vec<String> = elements_of(&tree, "App").into_iter().map(|e| e.name).collect();
        assert_eq!(names, vec!["Close".to_string()]);
        // A nameless row around a named button: only the button is numbered.
        let rows = serde_json::json!({"role": "window", "children": [
            {"role": "row", "name": "", "ref_id": "r", "bounds": {"x": 0.0, "y": 100.0, "width": 200.0, "height": 30.0}, "children": [
                {"role": "button", "name": "Wi-Fi", "ref_id": "w", "bounds": {"x": 10.0, "y": 105.0, "width": 80.0, "height": 20.0}}]},
            {"role": "row", "name": "", "ref_id": "r2", "bounds": {"x": 0.0, "y": 140.0, "width": 200.0, "height": 30.0}}
        ]});
        let kept: Vec<(String, String)> = elements_of(&rows, "App").into_iter().map(|e| (e.kind, e.name)).collect();
        assert_eq!(kept[0].1, "Wi-Fi");
        assert_eq!(kept.len(), 1, "an empty spacer row gets no number: {kept:?}");
    }

    #[test]
    fn a_save_panels_expand_triangle_gets_a_number() {
        // TextEdit's save sheet: the bot zoomed and clicked beside a triangle it had no number for.
        let sheet = serde_json::json!({"role": "sheet", "children": [
            {"role": "disclosure", "description": "顯示較多選項", "ref_id": "d", "bounds": {"x": 340.0, "y": 180.0, "width": 21.0, "height": 21.0}}
        ]});
        let els = elements_of(&sheet, "文字編輯");
        assert_eq!(els.len(), 1);
        assert!(els[0].active && els[0].kind == "展開按鈕" && els[0].name == "顯示較多選項", "{:?} {:?}", els[0].kind, els[0].name);
    }

    #[test]
    fn controls_and_text_come_with_their_place() {
        let tree = serde_json::json!({"role": "window", "name": "計算機", "bounds": {"x": 0.0, "y": 0.0, "width": 300.0, "height": 400.0}, "children": [
            {"role": "statictext", "value": "54", "bounds": {"x": 10.0, "y": 10.0, "width": 100.0, "height": 30.0}},
            {"ref_id": "@s1:e7", "role": "button", "name": "7", "bounds": {"x": 100.0, "y": 200.0, "width": 40.0, "height": 30.0}},
            {"ref_id": "@s1:e8", "role": "button", "name": "8", "bounds": {"x": 0.0, "y": 0.0, "width": 0.0, "height": 0.0}},
            {"ref_id": "@s1:e9", "role": "textarea", "value": format!("{}END", "x".repeat(2000)), "bounds": {"x": 0.0, "y": 300.0, "width": 300.0, "height": 90.0}}
        ]});
        let els = elements_of(&tree, "Calculator");
        // The window itself and a control with no place are left out.
        assert_eq!(els.len(), 3);
        assert_eq!((els[0].kind.as_str(), els[0].value.as_str(), els[0].active), ("文字", "54", false));
        assert_eq!((els[1].kind.as_str(), els[1].name.as_str(), els[1].active, els[1].centre()), ("按鈕", "7", true, (120.0, 215.0)));
        // A terminal's whole scrollback: the end (prompt, newest output) stays.
        assert!(els[2].value.ends_with("END") && els[2].value.contains("中間省略") && els[2].active);
    }
}