// Reading other apps through the OS accessibility APIs, with vercel-labs/agent-desktop compiled // in (AX on macOS, AT-SPI on Linux, UI Automation on Windows): an app's controls and text as // numbered lines (e1, e2…), each number remembered with where it is on the screen. That is all // it does. Acting on a number is a real click or keystroke at that spot (computer.rs), the same // way as acting on screenshot coordinates: one way to act on a screen, whatever the bot read it // with. (agent-desktop's own "semantic" actions were a second way with its own key names and // failures: typing into a terminal, "/" and "`" from a background thread, focus rules.) use agent_desktop_core as ad; use serde_json::Value; fn adapter() -> Box { #[cfg(target_os = "macos")] { Box::new(agent_desktop_macos::MacOSAdapter::new()) } #[cfg(target_os = "linux")] { Box::new(agent_desktop_linux::LinuxAdapter::new()) } #[cfg(target_os = "windows")] { Box::new(agent_desktop_windows::WindowsAdapter::new()) } } fn err(e: ad::AppError) -> String { let mut s = format!("{e}({})", e.code()); if let Some(hint) = e.suggestion() { s.push_str(&format!(":{hint}")); } s } fn st(v: &Value, k: &str) -> String { v.get(k).map(|x| x.as_str().map(String::from).unwrap_or_else(|| if x.is_null() { String::new() } else { x.to_string() })).unwrap_or_default().trim().to_string() } fn opt(v: &Value, k: &str) -> Option { Some(st(v, k)).filter(|s| !s.is_empty()) } /// One agent-desktop command (the CLI's names): apps, windows, snapshot, launch, permissions. pub fn run(cmd: &str, a: &Value) -> Result { use ad::commands as c; let ad = adapter(); let ad = ad.as_ref(); let ctx = ad::CommandContext::default(); let out = match cmd { "apps" => c::list_apps::execute(c::list_apps::ListAppsArgs { app: opt(a, "app"), timeout_ms: None }, ad), "windows" => c::list_windows::execute(c::list_windows::ListWindowsArgs { app: opt(a, "app"), timeout_ms: None }, ad), "snapshot" => c::snapshot::execute( c::snapshot::SnapshotArgs { app: opt(a, "app"), window_id: opt(a, "window_id"), max_depth: a["max_depth"].as_u64().unwrap_or(30).min(60) as u8, include_bounds: a["bounds"].as_bool().unwrap_or(true), interactive_only: a["interactive_only"].as_bool().unwrap_or(true), compact: true, surface: ad::SnapshotSurface::default(), skeleton: false, root_ref: opt(a, "root"), timeout_ms: None, force_electron_a11y: true, }, ad, &ctx, ), "launch" => c::launch::execute(c::launch::LaunchArgs { app: st(a, "app"), options: ad::system::launch_options::LaunchOptions { activate: true, ..Default::default() } }, ad), "permissions" => { let d = ad::Deadline::standard().map_err(|e| err(e.into()))?; return ad.permission_report(d).map(|r| Value::String(format!("{r:?}"))).map_err(|e| err(e.into())); } other => return Err(format!("agent-desktop 沒有「{other}」這個動作")), }; out.map_err(err) } // ---------- what the bot uses ---------- /// Where a numbered control is: its centre in screen coordinates (the space the mouse uses), /// and what it is (「按鈕「7」」), for the step's description. #[derive(Clone, Debug, PartialEq)] pub struct Spot { pub app: String, pub x: f64, pub y: f64, pub what: String, } /// Each number of a reading and its spot (None when the app didn't say where). pub type Spots = std::collections::HashMap>; /// The latest reading: the app and its numbers. static LAST: std::sync::Mutex> = std::sync::Mutex::new(None); /// "e6", "[e6]", "6" or a full "@snap:e6" → "e6". pub fn short_ref(r: &str) -> Result { let r = r.trim().trim_matches(['[', ']']); let r = r.rsplit(':').next().unwrap_or(r); let n = r.trim_start_matches('e'); if n.is_empty() || !n.chars().all(|c| c.is_ascii_digit()) { return Err(format!("「{r}」不是元素編號,請用 computer_use_read_app 列出的 e 編號")); } Ok(format!("e{n}")) } /// Where to click for a number from the latest reading. pub fn spot(r: &str) -> Result { let r = short_ref(r)?; let last = LAST.lock().unwrap_or_else(|e| e.into_inner()); let (_, spots) = last.as_ref().ok_or("請先用 computer_use_read_app 讀一下那個 App 的畫面")?; match spots.get(&r) { Some(Some(s)) => Ok(s.clone()), // Wayland doesn't tell apps where other windows' controls are: none has a place. Some(None) if spots.values().all(Option::is_none) => Err(format!("這台電腦讀得到 {r},但系統不告訴 App 它在螢幕上的位置:請用 computer_use_screenshot 看,用截圖上的 x、y 點。")), Some(None) => Err(format!("{r} 現在不在畫面上(被捲走或藏起來了)。先捲動或改用截圖上的座標。")), None => Err(format!("最近一次讀到的畫面裡沒有 {r}:先用 computer_use_read_app 再讀一次。")), } } /// The app the latest reading was of (for the per-app permission). pub fn last_app() -> Option { LAST.lock().unwrap_or_else(|e| e.into_inner()).as_ref().map(|l| l.0.clone()) } fn role_name(role: &str) -> &str { match role { "button" => "按鈕", "link" => "連結", "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield" => "輸入框", "checkbox" | "switch" | "toggle" => "勾選框", "radiobutton" => "選項", "popupbutton" | "menubutton" => "選單按鈕", "menuitem" => "選單項目", "tab" => "分頁", "row" | "cell" | "listitem" | "treeitem" | "outlinerow" => "項目", "slider" => "滑桿", "statictext" | "text" => "文字", "image" => "圖片", "scrollarea" => "捲動區", "group" => "群組", "outline" | "list" | "table" | "tree" | "grid" => "清單", "toolbar" => "工具列", "menubar" | "menu" => "選單", "heading" => "標題", "window" => "視窗", other => other, } } /// Longest value shown whole; longer ones (a terminal's scrollback, a document) keep their /// start and their end, where a terminal's prompt and newest output are. const VALUE_HEAD: usize = 160; const VALUE_TAIL: usize = 400; fn shorten(v: &str) -> String { let n = v.chars().count(); if n <= VALUE_HEAD + VALUE_TAIL + 20 { return v.to_string(); } let head: String = v.chars().take(VALUE_HEAD).collect(); let tail: String = v.chars().skip(n - VALUE_TAIL).collect(); format!("{head} …(中間省略 {} 字)… {tail}", n - VALUE_HEAD - VALUE_TAIL) } /// The centre of a node's box, when it has a real one on the screen. fn centre(n: &Value) -> Option<(f64, f64)> { let b = n.get("bounds")?; let (x, y, w, h) = (b["x"].as_f64()?, b["y"].as_f64()?, b["width"].as_f64()?, b["height"].as_f64()?); if w < 1.0 || h < 1.0 || x + w <= 0.0 || y + h <= 0.0 { return None; } Some((x + w / 2.0, y + h / 2.0)) } /// A snapshot as short lines, and each number's spot. pub fn lines_of(tree: &Value, app: &str) -> (Vec, Spots) { let mut lines = vec![]; let mut spots = std::collections::HashMap::new(); fn walk(n: &Value, depth: usize, lines: &mut Vec, spots: &mut Spots, parent: &str, app: &str) { if lines.len() >= 400 { return; } let name = n["name"].as_str().unwrap_or("").trim(); let value = n["value"].as_str().map(str::to_string).or_else(|| n["value"].as_f64().map(|f| f.to_string())).unwrap_or_default(); let value = shorten(value.trim()); let role = n["role"].as_str().unwrap_or(""); let text = if !name.is_empty() && !value.is_empty() && value != name { format!("「{name}」:{value}") } else if !name.is_empty() { format!("「{name}」") } else if !value.is_empty() { format!(":{value}") } else { String::new() }; let mut flags = vec![]; let is_field = matches!(role, "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield"); for (k, word) in [("checked", "已勾"), ("selected", "已選"), ("disabled", "停用"), ("focused", "游標在這")] { // Only text fields report focus usefully (Finder marks every sidebar cell focused). if k == "focused" && !is_field { continue; } if n["states"].as_array().map(|a| a.iter().any(|x| x == k)).unwrap_or(false) || n[k] == true { flags.push(word); } } let flags = if flags.is_empty() { String::new() } else { format!("({})", flags.join("、")) }; let indent = " ".repeat(depth.min(6)); // A cell or label that only repeats its parent's name adds nothing. let repeats = !name.is_empty() && name == parent; if repeats && n["ref_id"].is_null() { // plain text under a control with the same name } else if repeats && lines.last().map(|l| l.contains(&format!("「{name}」"))).unwrap_or(false) { // e.g. Finder's row → cell with the same name: keep the row only } else if let Some(r) = n["ref_id"].as_str() { let short = r.rsplit(':').next().unwrap_or(r).to_string(); if !text.is_empty() || matches!(role, "button" | "textfield" | "textarea" | "checkbox" | "searchfield" | "popupbutton") { let what = format!("{}{}", role_name(role), if name.is_empty() { String::new() } else { format!("「{}」", name.chars().take(30).collect::()) }); spots.insert(short.clone(), centre(n).map(|(x, y)| Spot { app: app.to_string(), x, y, what })); lines.push(format!("{indent}{short} {}{text}{flags}", role_name(role))); } } else if !text.is_empty() && matches!(role, "statictext" | "text" | "heading" | "cell" | "value" | "") { lines.push(format!("{indent}{}{text}", role_name(role))); } let next_parent = if name.is_empty() { parent } else { name }; for c in n["children"].as_array().into_iter().flatten() { walk(c, depth + 1, lines, spots, next_parent, app); } } walk(tree, 0, &mut lines, &mut spots, "", app); let pad = lines.iter().map(|l| l.len() - l.trim_start().len()).min().unwrap_or(0); (lines.into_iter().map(|l| l[pad..].to_string()).collect(), spots) } /// An app's controls (numbered) and visible text, as short lines. `app` empty = the app in front. /// The numbers and where they are on the screen are remembered for the next click or typing. pub fn read_app(app: &str) -> Result<(String, Vec), String> { let mut args = serde_json::json!({"interactive_only": false, "max_depth": 40, "bounds": true}); if !app.trim().is_empty() { args["app"] = Value::String(app.trim().to_string()); } let v = match run("snapshot", &args) { // Running without a window (closed, or minimised to the Dock): bring it forward once. Err(e) if e.contains("WINDOW_NOT_FOUND") && !app.trim().is_empty() => { let _ = run("launch", &serde_json::json!({"app": app.trim()})); std::thread::sleep(std::time::Duration::from_millis(900)); match run("snapshot", &args) { Ok(v) => v, Err(e) => { // macOS: "open" sends the reopen event, like clicking the app in the Dock. if !cfg!(target_os = "macos") { return Err(e); } let _ = std::process::Command::new("open").args(["-a", app.trim()]).status(); std::thread::sleep(std::time::Duration::from_millis(1200)); run("snapshot", &args)? } } } r => r?, }; let app_name = v["app"].as_str().unwrap_or(app).to_string(); let (lines, spots) = lines_of(&v["tree"], &app_name); *LAST.lock().unwrap_or_else(|e| e.into_inner()) = Some((app_name.clone(), spots)); Ok((app_name, lines)) } /// Open an app, or bring it to the front when it runs. pub fn launch(app: &str) -> Result<(), String> { run("launch", &serde_json::json!({"app": app})).map(|_| ()) } #[cfg(test)] mod tests { use super::*; #[test] fn numbers_keep_where_they_are() { let tree = serde_json::json!({"role": "window", "name": "計算機", "children": [ {"role": "statictext", "value": "54"}, {"ref_id": "@s1:e7", "role": "button", "name": "7", "bounds": {"x": 100.0, "y": 200.0, "width": 40.0, "height": 30.0}}, {"ref_id": "@s1:e8", "role": "button", "name": "8", "bounds": {"x": 0.0, "y": 0.0, "width": 0.0, "height": 0.0}}, {"ref_id": "@s1:e9", "role": "textarea", "value": format!("{}END", "x".repeat(2000))} ]}); let (lines, spots) = lines_of(&tree, "Calculator"); assert!(lines.iter().any(|l| l == "e7 按鈕「7」")); assert_eq!(spots["e7"], Some(Spot { app: "Calculator".into(), x: 120.0, y: 215.0, what: "按鈕「7」".into() })); // No real box (hidden, scrolled away): known, but not clickable. assert_eq!(spots["e8"], None); // A terminal's whole scrollback: the end (prompt, newest output) stays. let e9 = lines.iter().find(|l| l.starts_with("e9")).unwrap(); assert!(e9.ends_with("END") && e9.contains("中間省略") && e9.chars().count() < 700); assert_eq!(short_ref("[e7]").unwrap(), "e7"); assert_eq!(short_ref("@s1:e7").unwrap(), "e7"); assert_eq!(short_ref("12").unwrap(), "e12"); assert!(short_ref("ok").is_err()); } }