AnyBot/src/desktop.rs

// Reading other apps through the OS accessibility APIs, with vercel-labs/agent-desktop compiled
// in (AX on macOS, AT-SPI on Linux, UI Automation on Windows): an app's controls and text as
// numbered lines (e1, e2…), each number remembered with where it is on the screen. That is all
// it does. Acting on a number is a real click or keystroke at that spot (computer.rs), the same
// way as acting on screenshot coordinates: one way to act on a screen, whatever the bot read it
// with. (agent-desktop's own "semantic" actions were a second way with its own key names and
// failures: typing into a terminal, "/" and "`" from a background thread, focus rules.)

use agent_desktop_core as ad;
use serde_json::Value;

fn adapter() -> Box<dyn ad::PlatformAdapter> {
    #[cfg(target_os = "macos")]
    {
        Box::new(agent_desktop_macos::MacOSAdapter::new())
    }
    #[cfg(target_os = "linux")]
    {
        Box::new(agent_desktop_linux::LinuxAdapter::new())
    }
    #[cfg(target_os = "windows")]
    {
        Box::new(agent_desktop_windows::WindowsAdapter::new())
    }
}

fn err(e: ad::AppError) -> String {
    let mut s = format!("{e}（{}）", e.code());
    if let Some(hint) = e.suggestion() {
        s.push_str(&format!("：{hint}"));
    }
    s
}

fn st(v: &Value, k: &str) -> String {
    v.get(k).map(|x| x.as_str().map(String::from).unwrap_or_else(|| if x.is_null() { String::new() } else { x.to_string() })).unwrap_or_default().trim().to_string()
}

fn opt(v: &Value, k: &str) -> Option<String> {
    Some(st(v, k)).filter(|s| !s.is_empty())
}

/// One agent-desktop command (the CLI's names): apps, windows, snapshot, launch, permissions.
pub fn run(cmd: &str, a: &Value) -> Result<Value, String> {
    use ad::commands as c;
    let ad = adapter();
    let ad = ad.as_ref();
    let ctx = ad::CommandContext::default();
    let out = match cmd {
        "apps" => c::list_apps::execute(c::list_apps::ListAppsArgs { app: opt(a, "app"), timeout_ms: None }, ad),
        "windows" => c::list_windows::execute(c::list_windows::ListWindowsArgs { app: opt(a, "app"), timeout_ms: None }, ad),
        "snapshot" => c::snapshot::execute(
            c::snapshot::SnapshotArgs {
                app: opt(a, "app"),
                window_id: opt(a, "window_id"),
                max_depth: a["max_depth"].as_u64().unwrap_or(30).min(60) as u8,
                include_bounds: a["bounds"].as_bool().unwrap_or(true),
                interactive_only: a["interactive_only"].as_bool().unwrap_or(true),
                compact: true,
                surface: ad::SnapshotSurface::default(),
                skeleton: false,
                root_ref: opt(a, "root"),
                timeout_ms: None,
                force_electron_a11y: true,
            },
            ad,
            &ctx,
        ),
        "launch" => c::launch::execute(c::launch::LaunchArgs { app: st(a, "app"), options: ad::system::launch_options::LaunchOptions { activate: true, ..Default::default() } }, ad),
        "permissions" => {
            let d = ad::Deadline::standard().map_err(|e| err(e.into()))?;
            return ad.permission_report(d).map(|r| Value::String(format!("{r:?}"))).map_err(|e| err(e.into()));
        }
        other => return Err(format!("agent-desktop 沒有「{other}」這個動作")),
    };
    out.map_err(err)
}

// ---------- what the bot uses ----------

/// Where a numbered control is: its centre in screen coordinates (the space the mouse uses),
/// and what it is (「按鈕「7」」), for the step's description.
#[derive(Clone, Debug, PartialEq)]
pub struct Spot {
    pub app: String,
    pub x: f64,
    pub y: f64,
    pub what: String,
}

/// Each number of a reading and its spot (None when the app didn't say where).
pub type Spots = std::collections::HashMap<String, Option<Spot>>;

/// The latest reading: the app and its numbers.
static LAST: std::sync::Mutex<Option<(String, Spots)>> = std::sync::Mutex::new(None);

/// "e6", "[e6]", "6" or a full "@snap:e6" → "e6".
pub fn short_ref(r: &str) -> Result<String, String> {
    let r = r.trim().trim_matches(['[', ']']);
    let r = r.rsplit(':').next().unwrap_or(r);
    let n = r.trim_start_matches('e');
    if n.is_empty() || !n.chars().all(|c| c.is_ascii_digit()) {
        return Err(format!("「{r}」不是元素編號，請用 computer_use_read_app 列出的 e 編號"));
    }
    Ok(format!("e{n}"))
}

/// Where to click for a number from the latest reading.
pub fn spot(r: &str) -> Result<Spot, String> {
    let r = short_ref(r)?;
    let last = LAST.lock().unwrap_or_else(|e| e.into_inner());
    let (_, spots) = last.as_ref().ok_or("請先用 computer_use_read_app 讀一下那個 App 的畫面")?;
    match spots.get(&r) {
        Some(Some(s)) => Ok(s.clone()),
        // Wayland doesn't tell apps where other windows' controls are: none has a place.
        Some(None) if spots.values().all(Option::is_none) => Err(format!("這台電腦讀得到 {r}，但系統不告訴 App 它在螢幕上的位置：請用 computer_use_screenshot 看，用截圖上的 x、y 點。")),
        Some(None) => Err(format!("{r} 現在不在畫面上（被捲走或藏起來了）。先捲動或改用截圖上的座標。")),
        None => Err(format!("最近一次讀到的畫面裡沒有 {r}：先用 computer_use_read_app 再讀一次。")),
    }
}

/// The app the latest reading was of (for the per-app permission).
pub fn last_app() -> Option<String> {
    LAST.lock().unwrap_or_else(|e| e.into_inner()).as_ref().map(|l| l.0.clone())
}

fn role_name(role: &str) -> &str {
    match role {
        "button" => "按鈕",
        "link" => "連結",
        "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield" => "輸入框",
        "checkbox" | "switch" | "toggle" => "勾選框",
        "radiobutton" => "選項",
        "popupbutton" | "menubutton" => "選單按鈕",
        "menuitem" => "選單項目",
        "tab" => "分頁",
        "row" | "cell" | "listitem" | "treeitem" | "outlinerow" => "項目",
        "slider" => "滑桿",
        "statictext" | "text" => "文字",
        "image" => "圖片",
        "scrollarea" => "捲動區",
        "group" => "群組",
        "outline" | "list" | "table" | "tree" | "grid" => "清單",
        "toolbar" => "工具列",
        "menubar" | "menu" => "選單",
        "heading" => "標題",
        "window" => "視窗",
        other => other,
    }
}

/// Longest value shown whole; longer ones (a terminal's scrollback, a document) keep their
/// start and their end, where a terminal's prompt and newest output are.
const VALUE_HEAD: usize = 160;
const VALUE_TAIL: usize = 400;

fn shorten(v: &str) -> String {
    let n = v.chars().count();
    if n <= VALUE_HEAD + VALUE_TAIL + 20 {
        return v.to_string();
    }
    let head: String = v.chars().take(VALUE_HEAD).collect();
    let tail: String = v.chars().skip(n - VALUE_TAIL).collect();
    format!("{head} …（中間省略 {} 字）… {tail}", n - VALUE_HEAD - VALUE_TAIL)
}

/// The centre of a node's box, when it has a real one on the screen.
fn centre(n: &Value) -> Option<(f64, f64)> {
    let b = n.get("bounds")?;
    let (x, y, w, h) = (b["x"].as_f64()?, b["y"].as_f64()?, b["width"].as_f64()?, b["height"].as_f64()?);
    if w < 1.0 || h < 1.0 || x + w <= 0.0 || y + h <= 0.0 {
        return None;
    }
    Some((x + w / 2.0, y + h / 2.0))
}

/// A snapshot as short lines, and each number's spot.
pub fn lines_of(tree: &Value, app: &str) -> (Vec<String>, Spots) {
    let mut lines = vec![];
    let mut spots = std::collections::HashMap::new();
    fn walk(n: &Value, depth: usize, lines: &mut Vec<String>, spots: &mut Spots, parent: &str, app: &str) {
        if lines.len() >= 400 {
            return;
        }
        let name = n["name"].as_str().unwrap_or("").trim();
        let value = n["value"].as_str().map(str::to_string).or_else(|| n["value"].as_f64().map(|f| f.to_string())).unwrap_or_default();
        let value = shorten(value.trim());
        let role = n["role"].as_str().unwrap_or("");
        let text = if !name.is_empty() && !value.is_empty() && value != name { format!("「{name}」：{value}") } else if !name.is_empty() { format!("「{name}」") } else if !value.is_empty() { format!("：{value}") } else { String::new() };
        let mut flags = vec![];
        let is_field = matches!(role, "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield");
        for (k, word) in [("checked", "已勾"), ("selected", "已選"), ("disabled", "停用"), ("focused", "游標在這")] {
            // Only text fields report focus usefully (Finder marks every sidebar cell focused).
            if k == "focused" && !is_field {
                continue;
            }
            if n["states"].as_array().map(|a| a.iter().any(|x| x == k)).unwrap_or(false) || n[k] == true {
                flags.push(word);
            }
        }
        let flags = if flags.is_empty() { String::new() } else { format!("（{}）", flags.join("、")) };
        let indent = "  ".repeat(depth.min(6));
        // A cell or label that only repeats its parent's name adds nothing.
        let repeats = !name.is_empty() && name == parent;
        if repeats && n["ref_id"].is_null() {
            // plain text under a control with the same name
        } else if repeats && lines.last().map(|l| l.contains(&format!("「{name}」"))).unwrap_or(false) {
            // e.g. Finder's row → cell with the same name: keep the row only
        } else if let Some(r) = n["ref_id"].as_str() {
            let short = r.rsplit(':').next().unwrap_or(r).to_string();
            if !text.is_empty() || matches!(role, "button" | "textfield" | "textarea" | "checkbox" | "searchfield" | "popupbutton") {
                let what = format!("{}{}", role_name(role), if name.is_empty() { String::new() } else { format!("「{}」", name.chars().take(30).collect::<String>()) });
                spots.insert(short.clone(), centre(n).map(|(x, y)| Spot { app: app.to_string(), x, y, what }));
                lines.push(format!("{indent}{short} {}{text}{flags}", role_name(role)));
            }
        } else if !text.is_empty() && matches!(role, "statictext" | "text" | "heading" | "cell" | "value" | "") {
            lines.push(format!("{indent}{}{text}", role_name(role)));
        }
        let next_parent = if name.is_empty() { parent } else { name };
        for c in n["children"].as_array().into_iter().flatten() {
            walk(c, depth + 1, lines, spots, next_parent, app);
        }
    }
    walk(tree, 0, &mut lines, &mut spots, "", app);
    let pad = lines.iter().map(|l| l.len() - l.trim_start().len()).min().unwrap_or(0);
    (lines.into_iter().map(|l| l[pad..].to_string()).collect(), spots)
}

/// An app's controls (numbered) and visible text, as short lines. `app` empty = the app in front.
/// The numbers and where they are on the screen are remembered for the next click or typing.
pub fn read_app(app: &str) -> Result<(String, Vec<String>), String> {
    let mut args = serde_json::json!({"interactive_only": false, "max_depth": 40, "bounds": true});
    if !app.trim().is_empty() {
        args["app"] = Value::String(app.trim().to_string());
    }
    let v = match run("snapshot", &args) {
        // Running without a window (closed, or minimised to the Dock): bring it forward once.
        Err(e) if e.contains("WINDOW_NOT_FOUND") && !app.trim().is_empty() => {
            let _ = run("launch", &serde_json::json!({"app": app.trim()}));
            std::thread::sleep(std::time::Duration::from_millis(900));
            match run("snapshot", &args) {
                Ok(v) => v,
                Err(e) => {
                    // macOS: "open" sends the reopen event, like clicking the app in the Dock.
                    if !cfg!(target_os = "macos") {
                        return Err(e);
                    }
                    let _ = std::process::Command::new("open").args(["-a", app.trim()]).status();
                    std::thread::sleep(std::time::Duration::from_millis(1200));
                    run("snapshot", &args)?
                }
            }
        }
        r => r?,
    };
    let app_name = v["app"].as_str().unwrap_or(app).to_string();
    let (lines, spots) = lines_of(&v["tree"], &app_name);
    *LAST.lock().unwrap_or_else(|e| e.into_inner()) = Some((app_name.clone(), spots));
    Ok((app_name, lines))
}

/// Open an app, or bring it to the front when it runs (macOS uses LaunchServices instead).
#[cfg(not(target_os = "macos"))]
pub fn launch(app: &str) -> Result<(), String> {
    run("launch", &serde_json::json!({"app": app})).map(|_| ())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn numbers_keep_where_they_are() {
        let tree = serde_json::json!({"role": "window", "name": "計算機", "children": [
            {"role": "statictext", "value": "54"},
            {"ref_id": "@s1:e7", "role": "button", "name": "7", "bounds": {"x": 100.0, "y": 200.0, "width": 40.0, "height": 30.0}},
            {"ref_id": "@s1:e8", "role": "button", "name": "8", "bounds": {"x": 0.0, "y": 0.0, "width": 0.0, "height": 0.0}},
            {"ref_id": "@s1:e9", "role": "textarea", "value": format!("{}END", "x".repeat(2000))}
        ]});
        let (lines, spots) = lines_of(&tree, "Calculator");
        assert!(lines.iter().any(|l| l == "e7 按鈕「7」"));
        assert_eq!(spots["e7"], Some(Spot { app: "Calculator".into(), x: 120.0, y: 215.0, what: "按鈕「7」".into() }));
        // No real box (hidden, scrolled away): known, but not clickable.
        assert_eq!(spots["e8"], None);
        // A terminal's whole scrollback: the end (prompt, newest output) stays.
        let e9 = lines.iter().find(|l| l.starts_with("e9")).unwrap();
        assert!(e9.ends_with("END") && e9.contains("中間省略") && e9.chars().count() < 700);
        assert_eq!(short_ref("[e7]").unwrap(), "e7");
        assert_eq!(short_ref("@s1:e7").unwrap(), "e7");
        assert_eq!(short_ref("12").unwrap(), "e12");
        assert!(short_ref("ok").is_err());
    }
}