243 lines
12 KiB
Rust
243 lines
12 KiB
Rust
// Reading other apps through the OS accessibility APIs, with vercel-labs/agent-desktop compiled
|
||
// in (AX on macOS, AT-SPI on Linux, UI Automation on Windows): an app's controls and text as
|
||
// numbered lines (e1, e2…), each number remembered with where it is on the screen. That is all
|
||
// it does. Acting on a number is a real click or keystroke at that spot (computer.rs), the same
|
||
// way as acting on screenshot coordinates: one way to act on a screen, whatever the bot read it
|
||
// with. (agent-desktop's own "semantic" actions were a second way with its own key names and
|
||
// failures: typing into a terminal, "/" and "`" from a background thread, focus rules.)
|
||
|
||
use agent_desktop_core as ad;
|
||
use serde_json::Value;
|
||
|
||
fn adapter() -> Box<dyn ad::PlatformAdapter> {
|
||
#[cfg(target_os = "macos")]
|
||
{
|
||
Box::new(agent_desktop_macos::MacOSAdapter::new())
|
||
}
|
||
#[cfg(target_os = "linux")]
|
||
{
|
||
Box::new(agent_desktop_linux::LinuxAdapter::new())
|
||
}
|
||
#[cfg(target_os = "windows")]
|
||
{
|
||
Box::new(agent_desktop_windows::WindowsAdapter::new())
|
||
}
|
||
}
|
||
|
||
fn err(e: ad::AppError) -> String {
|
||
let mut s = format!("{e}（{}）", e.code());
|
||
if let Some(hint) = e.suggestion() {
|
||
s.push_str(&format!("：{hint}"));
|
||
}
|
||
s
|
||
}
|
||
|
||
fn st(v: &Value, k: &str) -> String {
|
||
v.get(k).map(|x| x.as_str().map(String::from).unwrap_or_else(|| if x.is_null() { String::new() } else { x.to_string() })).unwrap_or_default().trim().to_string()
|
||
}
|
||
|
||
fn opt(v: &Value, k: &str) -> Option<String> {
|
||
Some(st(v, k)).filter(|s| !s.is_empty())
|
||
}
|
||
|
||
/// One agent-desktop command (the CLI's names): apps, windows, snapshot, launch, permissions.
|
||
pub fn run(cmd: &str, a: &Value) -> Result<Value, String> {
|
||
use ad::commands as c;
|
||
let ad = adapter();
|
||
let ad = ad.as_ref();
|
||
let ctx = ad::CommandContext::default();
|
||
let out = match cmd {
|
||
"apps" => c::list_apps::execute(c::list_apps::ListAppsArgs { app: opt(a, "app"), timeout_ms: None }, ad),
|
||
"windows" => c::list_windows::execute(c::list_windows::ListWindowsArgs { app: opt(a, "app"), timeout_ms: None }, ad),
|
||
"snapshot" => c::snapshot::execute(
|
||
c::snapshot::SnapshotArgs {
|
||
app: opt(a, "app"),
|
||
window_id: opt(a, "window_id"),
|
||
max_depth: a["max_depth"].as_u64().unwrap_or(30).min(60) as u8,
|
||
include_bounds: a["bounds"].as_bool().unwrap_or(true),
|
||
interactive_only: a["interactive_only"].as_bool().unwrap_or(true),
|
||
compact: true,
|
||
surface: ad::SnapshotSurface::default(),
|
||
skeleton: false,
|
||
root_ref: opt(a, "root"),
|
||
timeout_ms: None,
|
||
force_electron_a11y: true,
|
||
},
|
||
ad,
|
||
&ctx,
|
||
),
|
||
"launch" => c::launch::execute(c::launch::LaunchArgs { app: st(a, "app"), options: ad::system::launch_options::LaunchOptions { activate: true, ..Default::default() } }, ad),
|
||
"permissions" => {
|
||
let d = ad::Deadline::standard().map_err(|e| err(e.into()))?;
|
||
return ad.permission_report(d).map(|r| Value::String(format!("{r:?}"))).map_err(|e| err(e.into()));
|
||
}
|
||
other => return Err(format!("agent-desktop 沒有「{other}」這個動作")),
|
||
};
|
||
out.map_err(err)
|
||
}
|
||
|
||
// ---------- what the bot uses ----------
|
||
|
||
fn role_name(role: &str) -> &str {
|
||
match role {
|
||
"button" => "按鈕",
|
||
"link" => "連結",
|
||
"textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield" => "輸入框",
|
||
"checkbox" | "switch" | "toggle" => "勾選框",
|
||
"radiobutton" => "選項",
|
||
"popupbutton" | "menubutton" => "選單按鈕",
|
||
"menuitem" => "選單項目",
|
||
"tab" => "分頁",
|
||
"row" | "cell" | "listitem" | "treeitem" | "outlinerow" => "項目",
|
||
"slider" => "滑桿",
|
||
"statictext" | "text" => "文字",
|
||
"image" => "圖片",
|
||
"heading" => "標題",
|
||
other => other,
|
||
}
|
||
}
|
||
|
||
/// Roles a person clicks or types into (the rest is text to read).
|
||
fn active(role: &str) -> bool {
|
||
matches!(
|
||
role,
|
||
"button" | "link" | "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield" | "checkbox" | "switch" | "toggle" | "radiobutton" | "popupbutton" | "menubutton" | "menuitem" | "tab" | "row" | "cell" | "listitem" | "treeitem" | "outlinerow" | "slider" | "incrementor"
|
||
)
|
||
}
|
||
|
||
/// Longest value shown whole; longer ones (a terminal's scrollback, a document) keep their
|
||
/// start and their end, where a terminal's prompt and newest output are.
|
||
const VALUE_HEAD: usize = 160;
|
||
const VALUE_TAIL: usize = 400;
|
||
|
||
fn shorten(v: &str) -> String {
|
||
let n = v.chars().count();
|
||
if n <= VALUE_HEAD + VALUE_TAIL + 20 {
|
||
return v.to_string();
|
||
}
|
||
let head: String = v.chars().take(VALUE_HEAD).collect();
|
||
let tail: String = v.chars().skip(n - VALUE_TAIL).collect();
|
||
format!("{head} …（中間省略 {} 字）… {tail}", n - VALUE_HEAD - VALUE_TAIL)
|
||
}
|
||
|
||
/// The first words inside a node (a row's label).
|
||
fn words_inside(n: &Value) -> String {
|
||
n["children"].as_array().into_iter().flatten().find_map(|c| {
|
||
let w = c["name"].as_str().or_else(|| c["value"].as_str()).unwrap_or("").trim().to_string();
|
||
if w.is_empty() { Some(words_inside(c)).filter(|w| !w.is_empty()) } else { Some(w) }
|
||
}).unwrap_or_default()
|
||
}
|
||
|
||
/// Something inside this node is a control with a name.
|
||
fn named_inside(n: &Value) -> bool {
|
||
n["children"].as_array().into_iter().flatten().any(|c| (c["ref_id"].is_string() && active(c["role"].as_str().unwrap_or("")) && !c["name"].as_str().unwrap_or("").trim().is_empty()) || named_inside(c))
|
||
}
|
||
|
||
/// The controls and text of a snapshot, with where each is on the screen (nodes the app gave
|
||
/// no place are left out: they can't be shown or clicked).
|
||
pub fn elements_of(tree: &Value, app: &str) -> Vec<crate::marks::Element> {
|
||
fn walk(n: &Value, out: &mut Vec<crate::marks::Element>, app: &str) {
|
||
if out.len() >= 600 {
|
||
return;
|
||
}
|
||
let role = n["role"].as_str().unwrap_or("");
|
||
let mut name = n["name"].as_str().unwrap_or("").trim().to_string();
|
||
// A nameless row, cell or list item goes by the words inside it (an icon-only button
|
||
// keeps its number without a name: it can still be clicked).
|
||
let row_like = matches!(role, "row" | "cell" | "treeitem" | "outlinerow" | "listitem");
|
||
// …unless a named control inside it is what gets clicked (System Settings' sidebar).
|
||
let wraps_named = name.is_empty() && named_inside(n);
|
||
if name.is_empty() && row_like && !wraps_named {
|
||
name = words_inside(n);
|
||
}
|
||
let value = n["value"].as_str().map(str::to_string).or_else(|| n["value"].as_f64().map(|f| f.to_string())).unwrap_or_default();
|
||
let b = &n["bounds"];
|
||
if let (Some(x), Some(y), Some(w), Some(h)) = (b["x"].as_f64(), b["y"].as_f64(), b["width"].as_f64(), b["height"].as_f64()) {
|
||
let is_active = n["ref_id"].is_string() && active(role);
|
||
let has = |k: &str| n["states"].as_array().map(|a| a.iter().any(|x| x == k)).unwrap_or(false) || n[k] == true;
|
||
// Hidden or scrolled out of view: nothing a click could reach. A nameless row whose
|
||
// own button has the name (System Settings' sidebar) needs no number of its own: a
|
||
// list of 「項目」 told the bot nothing.
|
||
let empty = name.is_empty() && value.trim().is_empty();
|
||
let unseen = has("hidden") || has("offscreen") || (is_active && wraps_named) || (row_like && empty);
|
||
if !unseen && w >= 1.0 && h >= 1.0 && x + w > 0.0 && y + h > 0.0 && (is_active || !name.is_empty() || !value.trim().is_empty()) && role != "window" && role != "group" {
|
||
let mut flags = vec![];
|
||
for (k, word) in [("checked", "已勾"), ("selected", "已選"), ("disabled", "停用"), ("focused", "游標在這")] {
|
||
if has(k) && (k != "focused" || role.contains("text") || role.contains("field")) {
|
||
flags.push(word);
|
||
}
|
||
}
|
||
out.push(crate::marks::Element { kind: role_name(role).to_string(), name, value: shorten(value.trim()), x, y, w, h, active: is_active, flags, window: app.to_string() });
|
||
}
|
||
}
|
||
for c in n["children"].as_array().into_iter().flatten() {
|
||
walk(c, out, app);
|
||
}
|
||
}
|
||
let mut out = vec![];
|
||
walk(tree, &mut out, app);
|
||
out
|
||
}
|
||
|
||
/// The app in front (or `app`): its name, its window's number (macOS: the CGWindowID) and its
|
||
/// controls and text, in screen coordinates.
|
||
pub fn elements(app: &str) -> Result<(String, Option<u32>, Vec<crate::marks::Element>), String> {
|
||
let mut args = serde_json::json!({"interactive_only": false, "max_depth": 40, "bounds": true});
|
||
if !app.trim().is_empty() {
|
||
args["app"] = Value::String(app.trim().to_string());
|
||
}
|
||
let v = run("snapshot", &args)?;
|
||
let name = v["app"].as_str().unwrap_or(app).to_string();
|
||
let els = elements_of(&v["tree"], &name);
|
||
let window = v["window"]["id"].as_str().and_then(|w| w.strip_prefix("w-")).and_then(|n| n.parse().ok());
|
||
Ok((name, window, els))
|
||
}
|
||
|
||
/// Open an app, or bring it to the front when it runs (macOS uses LaunchServices instead).
|
||
#[cfg(not(target_os = "macos"))]
|
||
pub fn launch(app: &str) -> Result<(), String> {
|
||
run("launch", &serde_json::json!({"app": app})).map(|_| ())
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn hidden_and_offscreen_controls_get_no_number() {
|
||
let tree = serde_json::json!({"role": "window", "children": [
|
||
{"role": "button", "name": "Close", "ref_id": "a", "bounds": {"x": 8.0, "y": 38.0, "width": 16.0, "height": 16.0}},
|
||
{"role": "scrollbar", "name": "bar", "ref_id": "b", "states": ["disabled", "hidden"], "bounds": {"x": 1902.0, "y": 61.0, "width": 19.0, "height": 1000.0}},
|
||
{"role": "link", "name": "Below the fold", "ref_id": "c", "states": ["offscreen"], "bounds": {"x": 20.0, "y": 2400.0, "width": 80.0, "height": 20.0}}
|
||
]});
|
||
let names: Vec<String> = elements_of(&tree, "App").into_iter().map(|e| e.name).collect();
|
||
assert_eq!(names, vec!["Close".to_string()]);
|
||
// A nameless row around a named button: only the button is numbered.
|
||
let rows = serde_json::json!({"role": "window", "children": [
|
||
{"role": "row", "name": "", "ref_id": "r", "bounds": {"x": 0.0, "y": 100.0, "width": 200.0, "height": 30.0}, "children": [
|
||
{"role": "button", "name": "Wi-Fi", "ref_id": "w", "bounds": {"x": 10.0, "y": 105.0, "width": 80.0, "height": 20.0}}]},
|
||
{"role": "row", "name": "", "ref_id": "r2", "bounds": {"x": 0.0, "y": 140.0, "width": 200.0, "height": 30.0}}
|
||
]});
|
||
let kept: Vec<(String, String)> = elements_of(&rows, "App").into_iter().map(|e| (e.kind, e.name)).collect();
|
||
assert_eq!(kept[0].1, "Wi-Fi");
|
||
assert_eq!(kept.len(), 1, "an empty spacer row gets no number: {kept:?}");
|
||
}
|
||
|
||
#[test]
|
||
fn controls_and_text_come_with_their_place() {
|
||
let tree = serde_json::json!({"role": "window", "name": "計算機", "bounds": {"x": 0.0, "y": 0.0, "width": 300.0, "height": 400.0}, "children": [
|
||
{"role": "statictext", "value": "54", "bounds": {"x": 10.0, "y": 10.0, "width": 100.0, "height": 30.0}},
|
||
{"ref_id": "@s1:e7", "role": "button", "name": "7", "bounds": {"x": 100.0, "y": 200.0, "width": 40.0, "height": 30.0}},
|
||
{"ref_id": "@s1:e8", "role": "button", "name": "8", "bounds": {"x": 0.0, "y": 0.0, "width": 0.0, "height": 0.0}},
|
||
{"ref_id": "@s1:e9", "role": "textarea", "value": format!("{}END", "x".repeat(2000)), "bounds": {"x": 0.0, "y": 300.0, "width": 300.0, "height": 90.0}}
|
||
]});
|
||
let els = elements_of(&tree, "Calculator");
|
||
// The window itself and a control with no place are left out.
|
||
assert_eq!(els.len(), 3);
|
||
assert_eq!((els[0].kind.as_str(), els[0].value.as_str(), els[0].active), ("文字", "54", false));
|
||
assert_eq!((els[1].kind.as_str(), els[1].name.as_str(), els[1].active, els[1].centre()), ("按鈕", "7", true, (120.0, 215.0)));
|
||
// A terminal's whole scrollback: the end (prompt, newest output) stays.
|
||
assert!(els[2].value.ends_with("END") && els[2].value.contains("中間省略") && els[2].active);
|
||
}
|
||
}
|