// Seeing and using other apps through the OS accessibility APIs, with vercel-labs/agent-desktop // compiled in (AX on macOS, AT-SPI on Linux, UI Automation on Windows). A snapshot gives every // control of an app a ref; actions take a ref. No screenshots needed, and by default no pointer // movement or focus change (the person can keep working). use agent_desktop_core as ad; use serde_json::Value; fn adapter() -> Box { #[cfg(target_os = "macos")] { Box::new(agent_desktop_macos::MacOSAdapter::new()) } #[cfg(target_os = "linux")] { Box::new(agent_desktop_linux::LinuxAdapter::new()) } #[cfg(target_os = "windows")] { Box::new(agent_desktop_windows::WindowsAdapter::new()) } } fn err(e: ad::AppError) -> String { let mut s = format!("{e}({})", e.code()); if let Some(hint) = e.suggestion() { s.push_str(&format!(":{hint}")); } s } fn st(v: &Value, k: &str) -> String { v.get(k).map(|x| x.as_str().map(String::from).unwrap_or_else(|| if x.is_null() { String::new() } else { x.to_string() })).unwrap_or_default().trim().to_string() } fn opt(v: &Value, k: &str) -> Option { Some(st(v, k)).filter(|s| !s.is_empty()) } /// One agent-desktop command (the CLI's names): apps, windows, snapshot, click, double_click, /// right_click, type, set_value, select, press, scroll, focus_window, launch, screenshot. pub fn run(cmd: &str, a: &Value) -> Result { use ad::commands as c; let ad = adapter(); let ad = ad.as_ref(); let ctx = ad::CommandContext::default(); let r = |rf: &Value| c::helpers::RefArgs { ref_id: st(rf, "ref"), timeout_ms: None }; let out = match cmd { "apps" => c::list_apps::execute(c::list_apps::ListAppsArgs { app: opt(a, "app"), timeout_ms: None }, ad), "windows" => c::list_windows::execute(c::list_windows::ListWindowsArgs { app: opt(a, "app"), timeout_ms: None }, ad), "snapshot" => c::snapshot::execute( c::snapshot::SnapshotArgs { app: opt(a, "app"), window_id: opt(a, "window_id"), max_depth: a["max_depth"].as_u64().unwrap_or(30).min(60) as u8, include_bounds: false, interactive_only: a["interactive_only"].as_bool().unwrap_or(true), compact: true, surface: ad::SnapshotSurface::default(), skeleton: false, root_ref: opt(a, "root"), timeout_ms: None, force_electron_a11y: true, }, ad, &ctx, ), "click" => c::click::execute(r(a), ad, &ctx), "double_click" => c::double_click::execute(r(a), ad, &ctx), "right_click" => c::right_click::execute(r(a), ad, &ctx), "type" => c::type_text::execute(c::type_text::TypeArgs { ref_id: st(a, "ref"), text: st(a, "text"), timeout_ms: None, interval_ms: None }, ad, &ctx), "set_value" => c::set_value::execute(c::set_value::SetValueArgs { ref_id: st(a, "ref"), value: st(a, "value"), timeout_ms: None }, ad, &ctx), "select" => c::select::execute(c::select::SelectArgs { ref_id: st(a, "ref"), value: st(a, "value"), timeout_ms: None }, ad, &ctx), "press" => c::press::execute(c::press::PressArgs { combo: st(a, "combo"), app: opt(a, "app"), window_id: opt(a, "window_id"), force: false }, ad, &ctx), "scroll" => { let direction = match st(a, "direction").as_str() { "up" => ad::Direction::Up, "left" => ad::Direction::Left, "right" => ad::Direction::Right, _ => ad::Direction::Down, }; c::scroll::execute(c::scroll::ScrollArgs { ref_id: st(a, "ref"), direction, amount: a["amount"].as_u64().unwrap_or(3) as u32, timeout_ms: None }, ad, &ctx) } "focus_window" => c::focus_window::execute(c::focus_window::FocusWindowArgs { window_id: opt(a, "window_id"), app: opt(a, "app"), title: opt(a, "title") }, ad), "launch" => c::launch::execute(c::launch::LaunchArgs { app: st(a, "app"), options: ad::system::launch_options::LaunchOptions { activate: true, ..Default::default() } }, ad), "permissions" => { let d = ad::Deadline::standard().map_err(|e| err(e.into()))?; return ad.permission_report(d).map(|r| Value::String(format!("{r:?}"))).map_err(|e| err(e.into())); } other => return Err(format!("agent-desktop 沒有「{other}」這個動作")), }; out.map_err(err) } // ---------- what the bot uses ---------- /// The latest snapshot (id, app) so the bot can say "e6" instead of the full ref. static LAST: std::sync::Mutex> = std::sync::Mutex::new(None); /// "e6", "[e6]", "6" or a full "@snap:e6" → the full ref of the latest snapshot. pub fn full_ref(r: &str) -> Result { let r = r.trim().trim_matches(['[', ']']); if r.starts_with('@') { return Ok(r.to_string()); } let n = r.trim_start_matches('e'); if n.is_empty() || !n.chars().all(|c| c.is_ascii_digit()) { return Err(format!("「{r}」不是元素編號,請用 computer_use_read_app 列出的 e 編號")); } let last = LAST.lock().unwrap().clone().ok_or("請先用 computer_use_read_app 看一下 App 的畫面")?; Ok(format!("@{}:e{n}", last.0)) } /// The app the latest snapshot was of (for the per-app permission). pub fn last_app() -> Option { LAST.lock().unwrap().as_ref().map(|l| l.1.clone()) } fn role_name(role: &str) -> &str { match role { "button" => "按鈕", "link" => "連結", "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield" => "輸入框", "checkbox" | "switch" | "toggle" => "勾選框", "radiobutton" => "選項", "popupbutton" | "menubutton" => "選單按鈕", "menuitem" => "選單項目", "tab" => "分頁", "row" | "cell" | "listitem" | "treeitem" | "outlinerow" => "項目", "slider" => "滑桿", "statictext" | "text" => "文字", "image" => "圖片", "scrollarea" => "捲動區", "group" => "群組", "outline" | "list" | "table" | "tree" | "grid" => "清單", "toolbar" => "工具列", "menubar" | "menu" => "選單", "heading" => "標題", "window" => "視窗", other => other, } } /// An app's controls (numbered) and visible text, as short lines. `app` empty = the app in front. pub fn read_app(app: &str) -> Result<(String, Vec), String> { read_app_as(app, true) } fn read_app_as(app: &str, remember: bool) -> Result<(String, Vec), String> { let mut args = serde_json::json!({"interactive_only": false, "max_depth": 40}); if !app.trim().is_empty() { args["app"] = Value::String(app.trim().to_string()); } let v = match run("snapshot", &args) { // Running without a window (closed, or minimised to the Dock): bring it forward once. Err(e) if e.contains("WINDOW_NOT_FOUND") && !app.trim().is_empty() => { let _ = run("launch", &serde_json::json!({"app": app.trim()})); std::thread::sleep(std::time::Duration::from_millis(900)); match run("snapshot", &args) { Ok(v) => v, Err(e) => { // macOS: "open" sends the reopen event, like clicking the app in the Dock. if !cfg!(target_os = "macos") { return Err(e); } let _ = std::process::Command::new("open").args(["-a", app.trim()]).status(); std::thread::sleep(std::time::Duration::from_millis(1200)); run("snapshot", &args)? } } } r => r?, }; let snap = v["snapshot_id"].as_str().unwrap_or("").to_string(); let app_name = v["app"].as_str().unwrap_or(app).to_string(); if remember { *LAST.lock().unwrap() = Some((snap.clone(), app_name.clone())); } let mut lines = vec![]; fn walk(n: &Value, depth: usize, lines: &mut Vec, parent: &str) { if lines.len() >= 400 { return; } let name = n["name"].as_str().unwrap_or("").trim(); let value = n["value"].as_str().map(str::to_string).or_else(|| n["value"].as_f64().map(|f| f.to_string())).unwrap_or_default(); let role = n["role"].as_str().unwrap_or(""); let text = if !name.is_empty() && !value.is_empty() && value != name { format!("「{name}」:{value}") } else if !name.is_empty() { format!("「{name}」") } else if !value.is_empty() { format!(":{value}") } else { String::new() }; let mut flags = vec![]; let is_field = matches!(role, "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield"); for (k, word) in [("checked", "已勾"), ("selected", "已選"), ("disabled", "停用"), ("focused", "游標在這")] { // Only text fields report focus usefully (Finder marks every sidebar cell focused). if k == "focused" && !is_field { continue; } if n["states"].as_array().map(|a| a.iter().any(|x| x == k)).unwrap_or(false) || n[k] == true { flags.push(word); } } let flags = if flags.is_empty() { String::new() } else { format!("({})", flags.join("、")) }; let indent = " ".repeat(depth.min(6)); // A cell or label that only repeats its parent's name adds nothing. let repeats = !name.is_empty() && name == parent; if repeats && n["ref_id"].is_null() { // plain text under a control with the same name } else if repeats && lines.last().map(|l| l.contains(&format!("「{name}」"))).unwrap_or(false) { // e.g. Finder's row → cell with the same name: keep the row only } else if let Some(r) = n["ref_id"].as_str() { let short = r.rsplit(':').next().unwrap_or(r); if !text.is_empty() || matches!(role, "button" | "textfield" | "textarea" | "checkbox" | "searchfield" | "popupbutton") { lines.push(format!("{indent}{short} {}{text}{flags}", role_name(role))); } } else if !text.is_empty() && matches!(role, "statictext" | "text" | "heading" | "cell" | "value" | "") { lines.push(format!("{indent}{}{text}", role_name(role))); } let next_parent = if name.is_empty() { parent } else { name }; for c in n["children"].as_array().into_iter().flatten() { walk(c, depth + 1, lines, next_parent); } } walk(&v["tree"], 0, &mut lines, ""); let pad = lines.iter().map(|l| l.len() - l.trim_start().len()).min().unwrap_or(0); let lines = lines.into_iter().map(|l| l[pad..].to_string()).collect(); Ok((app_name, lines)) } /// Act on an element by its number: click, double_click, right_click, type, set_value, select, /// scroll. Returns what the app shows now (new text since the action). pub fn act(cmd: &str, r: &str, extra: &Value) -> Result { let full = full_ref(r)?; let app = last_app().unwrap_or_default(); let before = read_app_as(&app, false).map(|x| x.1).unwrap_or_default(); let mut a = extra.clone(); if !a.is_object() { a = serde_json::json!({}); } a["ref"] = Value::String(full.clone()); let res = run(cmd, &a); // Typing into some fields fails as keystrokes; setting the value works there. let res = match (cmd, res) { ("type", Err(_)) => run("set_value", &serde_json::json!({"ref": full, "value": a["text"]})), (_, r) => r, }; res?; std::thread::sleep(std::time::Duration::from_millis(400)); let (_, after) = read_app_as(&app, false).unwrap_or_default(); let old: std::collections::HashSet = before.iter().map(|l| l.split_whitespace().skip(1).collect::>().join(" ")).collect(); let new: Vec<&String> = after.iter().filter(|l| !old.contains(&l.split_whitespace().skip(1).collect::>().join(" "))).take(30).collect(); // Show what changed without numbers (they belong to a different reading); if controls came // or went, the bot's numbers may now point elsewhere. let strip = |l: &str| -> String { let t = l.trim(); match t.split_once(' ') { Some((first, rest)) if first.starts_with('e') && first[1..].chars().all(|c| c.is_ascii_digit()) => rest.to_string(), _ => t.to_string(), } }; let controls_changed = new.iter().any(|l| l.trim().starts_with('e')) || before.iter().filter(|l| l.trim().starts_with('e')).count() != after.iter().filter(|l| l.trim().starts_with('e')).count(); let mut out = format!("好了(在「{app}」)。"); if !new.is_empty() { out.push_str(&format!("\n現在畫面上新出現:\n{}", new.iter().map(|l| strip(l)).collect::>().join("\n"))); } if controls_changed { out.push_str("\n(按鈕或欄位有變動,編號可能已經不同:下一步前先用 computer_use_read_app 看新的清單。)"); } Ok(out) }