// Reading other apps through the OS accessibility APIs, with vercel-labs/agent-desktop compiled // in (AX on macOS, AT-SPI on Linux, UI Automation on Windows): an app's controls and text as // numbered lines (e1, e2…), each number remembered with where it is on the screen. That is all // it does. Acting on a number is a real click or keystroke at that spot (computer.rs), the same // way as acting on screenshot coordinates: one way to act on a screen, whatever the bot read it // with. (agent-desktop's own "semantic" actions were a second way with its own key names and // failures: typing into a terminal, "/" and "`" from a background thread, focus rules.) use agent_desktop_core as ad; use serde_json::Value; fn adapter() -> Box { #[cfg(target_os = "macos")] { Box::new(agent_desktop_macos::MacOSAdapter::new()) } #[cfg(target_os = "linux")] { Box::new(agent_desktop_linux::LinuxAdapter::new()) } #[cfg(target_os = "windows")] { Box::new(agent_desktop_windows::WindowsAdapter::new()) } } fn err(e: ad::AppError) -> String { let mut s = format!("{e}({})", e.code()); if let Some(hint) = e.suggestion() { s.push_str(&format!(":{hint}")); } s } fn st(v: &Value, k: &str) -> String { v.get(k).map(|x| x.as_str().map(String::from).unwrap_or_else(|| if x.is_null() { String::new() } else { x.to_string() })).unwrap_or_default().trim().to_string() } fn opt(v: &Value, k: &str) -> Option { Some(st(v, k)).filter(|s| !s.is_empty()) } /// One agent-desktop command (the CLI's names): apps, windows, snapshot, launch, permissions. pub fn run(cmd: &str, a: &Value) -> Result { use ad::commands as c; let ad = adapter(); let ad = ad.as_ref(); let ctx = ad::CommandContext::default(); let out = match cmd { "apps" => c::list_apps::execute(c::list_apps::ListAppsArgs { app: opt(a, "app"), timeout_ms: None }, ad), "windows" => c::list_windows::execute(c::list_windows::ListWindowsArgs { app: opt(a, "app"), timeout_ms: None }, ad), "snapshot" => c::snapshot::execute( c::snapshot::SnapshotArgs { app: opt(a, "app"), window_id: opt(a, "window_id"), max_depth: a["max_depth"].as_u64().unwrap_or(30).min(60) as u8, include_bounds: a["bounds"].as_bool().unwrap_or(true), interactive_only: a["interactive_only"].as_bool().unwrap_or(true), compact: true, surface: a["surface"].as_str().and_then(|n| serde_json::from_value(Value::String(n.to_string())).ok()).unwrap_or_default(), skeleton: false, root_ref: opt(a, "root"), timeout_ms: None, force_electron_a11y: true, }, ad, &ctx, ), "launch" => c::launch::execute(c::launch::LaunchArgs { app: st(a, "app"), options: ad::system::launch_options::LaunchOptions { activate: true, ..Default::default() } }, ad), "permissions" => { let d = ad::Deadline::standard().map_err(|e| err(e.into()))?; return ad.permission_report(d).map(|r| Value::String(format!("{r:?}"))).map_err(|e| err(e.into())); } other => return Err(format!("agent-desktop 沒有「{other}」這個動作")), }; out.map_err(err) } // ---------- what the bot uses ---------- fn role_name(role: &str) -> &str { match role { "button" => "按鈕", "link" => "連結", "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield" => "輸入框", "checkbox" | "switch" | "toggle" => "勾選框", "radiobutton" => "選項", "popupbutton" | "menubutton" => "選單按鈕", "disclosure" => "展開按鈕", "colorwell" => "顏色", "datefield" | "timefield" => "日期", "menuitem" => "選單項目", "tab" => "分頁", "row" | "cell" | "listitem" | "treeitem" | "outlinerow" => "項目", "slider" => "滑桿", "statictext" | "text" => "文字", "image" => "圖片", "heading" => "標題", other => other, } } /// Roles a person clicks or types into (the rest is text to read). fn active(role: &str) -> bool { matches!( role, "button" | "link" | "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield" | "checkbox" | "switch" | "toggle" | "radiobutton" | "popupbutton" | "menubutton" | "menuitem" | "tab" | "row" | "cell" | "listitem" | "treeitem" | "outlinerow" | "slider" | "incrementor" | "disclosure" | "colorwell" | "datefield" | "timefield" ) } /// Longest value shown whole; longer ones (a terminal's scrollback, a document) keep their /// start and their end, where a terminal's prompt and newest output are. const VALUE_HEAD: usize = 160; const VALUE_TAIL: usize = 400; fn shorten(v: &str) -> String { let n = v.chars().count(); if n <= VALUE_HEAD + VALUE_TAIL + 20 { return v.to_string(); } let head: String = v.chars().take(VALUE_HEAD).collect(); let tail: String = v.chars().skip(n - VALUE_TAIL).collect(); format!("{head} …(中間省略 {} 字)… {tail}", n - VALUE_HEAD - VALUE_TAIL) } /// The first words inside a node (a row's label). fn words_inside(n: &Value) -> String { n["children"].as_array().into_iter().flatten().find_map(|c| { let w = c["name"].as_str().or_else(|| c["value"].as_str()).unwrap_or("").trim().to_string(); if w.is_empty() { Some(words_inside(c)).filter(|w| !w.is_empty()) } else { Some(w) } }).unwrap_or_default() } /// Something inside this node is a control with a name. fn named_inside(n: &Value) -> bool { n["children"].as_array().into_iter().flatten().any(|c| (c["ref_id"].is_string() && active(c["role"].as_str().unwrap_or("")) && !c["name"].as_str().unwrap_or("").trim().is_empty()) || named_inside(c)) } /// The controls and text of a snapshot, with where each is on the screen (nodes the app gave /// no place are left out: they can't be shown or clicked). pub fn elements_of(tree: &Value, app: &str) -> Vec { fn walk(n: &Value, out: &mut Vec, app: &str) { if out.len() >= 600 { return; } let role = n["role"].as_str().unwrap_or(""); let mut name = n["name"].as_str().unwrap_or("").trim().to_string(); // An icon-only control often has its words only as a description (the save panel's // triangle: 「顯示較多選項」). if name.is_empty() && n["ref_id"].is_string() && active(role) { name = n["description"].as_str().unwrap_or("").trim().to_string(); } // A nameless row, cell or list item goes by the words inside it (an icon-only button // keeps its number without a name: it can still be clicked). let row_like = matches!(role, "row" | "cell" | "treeitem" | "outlinerow" | "listitem"); // …unless a named control inside it is what gets clicked (System Settings' sidebar). let wraps_named = name.is_empty() && named_inside(n); if name.is_empty() && row_like && !wraps_named { name = words_inside(n); } let value = n["value"].as_str().map(str::to_string).or_else(|| n["value"].as_f64().map(|f| f.to_string())).unwrap_or_default(); let b = &n["bounds"]; if let (Some(x), Some(y), Some(w), Some(h)) = (b["x"].as_f64(), b["y"].as_f64(), b["width"].as_f64(), b["height"].as_f64()) { let is_active = n["ref_id"].is_string() && active(role); let has = |k: &str| n["states"].as_array().map(|a| a.iter().any(|x| x == k)).unwrap_or(false) || n[k] == true; // Hidden or scrolled out of view: nothing a click could reach. A nameless row whose // own button has the name (System Settings' sidebar) needs no number of its own: a // list of 「項目」 told the bot nothing. let empty = name.is_empty() && value.trim().is_empty(); let unseen = has("hidden") || has("offscreen") || (is_active && wraps_named) || (row_like && empty); if !unseen && w >= 1.0 && h >= 1.0 && x + w > 0.0 && y + h > 0.0 && (is_active || !name.is_empty() || !value.trim().is_empty()) && role != "window" && role != "group" { let mut flags = vec![]; for (k, word) in [("checked", "已勾"), ("selected", "已選"), ("disabled", "停用"), ("focused", "游標在這")] { if has(k) && (k != "focused" || role.contains("text") || role.contains("field")) { flags.push(word); } } out.push(crate::marks::Element { kind: role_name(role).to_string(), name, value: shorten(value.trim()), x, y, w, h, active: is_active, flags, window: app.to_string() }); } } for c in n["children"].as_array().into_iter().flatten() { walk(c, out, app); } } let mut out = vec![]; walk(tree, &mut out, app); out } /// The app in front (or `app`): its name, its window's number (macOS: the CGWindowID) and its /// controls and text, in screen coordinates. pub fn elements(app: &str) -> Result<(String, Option, Vec), String> { let mut args = serde_json::json!({"interactive_only": false, "max_depth": 40, "bounds": true}); if !app.trim().is_empty() { args["app"] = Value::String(app.trim().to_string()); } let v = run("snapshot", &args)?; let name = v["app"].as_str().unwrap_or(app).to_string(); let els = elements_of(&v["tree"], &name); let window = v["window"]["id"].as_str().and_then(|w| w.strip_prefix("w-")).and_then(|n| n.parse().ok()); Ok((name, window, els)) } /// Open an app, or bring it to the front when it runs (macOS uses LaunchServices instead). #[cfg(not(target_os = "macos"))] pub fn launch(app: &str) -> Result<(), String> { run("launch", &serde_json::json!({"app": app})).map(|_| ()) } #[cfg(test)] mod tests { use super::*; #[test] fn hidden_and_offscreen_controls_get_no_number() { let tree = serde_json::json!({"role": "window", "children": [ {"role": "button", "name": "Close", "ref_id": "a", "bounds": {"x": 8.0, "y": 38.0, "width": 16.0, "height": 16.0}}, {"role": "scrollbar", "name": "bar", "ref_id": "b", "states": ["disabled", "hidden"], "bounds": {"x": 1902.0, "y": 61.0, "width": 19.0, "height": 1000.0}}, {"role": "link", "name": "Below the fold", "ref_id": "c", "states": ["offscreen"], "bounds": {"x": 20.0, "y": 2400.0, "width": 80.0, "height": 20.0}} ]}); let names: Vec = elements_of(&tree, "App").into_iter().map(|e| e.name).collect(); assert_eq!(names, vec!["Close".to_string()]); // A nameless row around a named button: only the button is numbered. let rows = serde_json::json!({"role": "window", "children": [ {"role": "row", "name": "", "ref_id": "r", "bounds": {"x": 0.0, "y": 100.0, "width": 200.0, "height": 30.0}, "children": [ {"role": "button", "name": "Wi-Fi", "ref_id": "w", "bounds": {"x": 10.0, "y": 105.0, "width": 80.0, "height": 20.0}}]}, {"role": "row", "name": "", "ref_id": "r2", "bounds": {"x": 0.0, "y": 140.0, "width": 200.0, "height": 30.0}} ]}); let kept: Vec<(String, String)> = elements_of(&rows, "App").into_iter().map(|e| (e.kind, e.name)).collect(); assert_eq!(kept[0].1, "Wi-Fi"); assert_eq!(kept.len(), 1, "an empty spacer row gets no number: {kept:?}"); } #[test] fn a_save_panels_expand_triangle_gets_a_number() { // TextEdit's save sheet: the bot zoomed and clicked beside a triangle it had no number for. let sheet = serde_json::json!({"role": "sheet", "children": [ {"role": "disclosure", "description": "顯示較多選項", "ref_id": "d", "bounds": {"x": 340.0, "y": 180.0, "width": 21.0, "height": 21.0}} ]}); let els = elements_of(&sheet, "文字編輯"); assert_eq!(els.len(), 1); assert!(els[0].active && els[0].kind == "展開按鈕" && els[0].name == "顯示較多選項", "{:?} {:?}", els[0].kind, els[0].name); } #[test] fn controls_and_text_come_with_their_place() { let tree = serde_json::json!({"role": "window", "name": "計算機", "bounds": {"x": 0.0, "y": 0.0, "width": 300.0, "height": 400.0}, "children": [ {"role": "statictext", "value": "54", "bounds": {"x": 10.0, "y": 10.0, "width": 100.0, "height": 30.0}}, {"ref_id": "@s1:e7", "role": "button", "name": "7", "bounds": {"x": 100.0, "y": 200.0, "width": 40.0, "height": 30.0}}, {"ref_id": "@s1:e8", "role": "button", "name": "8", "bounds": {"x": 0.0, "y": 0.0, "width": 0.0, "height": 0.0}}, {"ref_id": "@s1:e9", "role": "textarea", "value": format!("{}END", "x".repeat(2000)), "bounds": {"x": 0.0, "y": 300.0, "width": 300.0, "height": 90.0}} ]}); let els = elements_of(&tree, "Calculator"); // The window itself and a control with no place are left out. assert_eq!(els.len(), 3); assert_eq!((els[0].kind.as_str(), els[0].value.as_str(), els[0].active), ("文字", "54", false)); assert_eq!((els[1].kind.as_str(), els[1].name.as_str(), els[1].active, els[1].centre()), ("按鈕", "7", true, (120.0, 215.0))); // A terminal's whole scrollback: the end (prompt, newest output) stays. assert!(els[2].value.ends_with("END") && els[2].value.contains("中間省略") && els[2].active); } }