294 lines
14 KiB
Rust
294 lines
14 KiB
Rust
// Reading other apps through the OS accessibility APIs, with vercel-labs/agent-desktop compiled
|
||
// in (AX on macOS, AT-SPI on Linux, UI Automation on Windows): an app's controls and text as
|
||
// numbered lines (e1, e2…), each number remembered with where it is on the screen. That is all
|
||
// it does. Acting on a number is a real click or keystroke at that spot (computer.rs), the same
|
||
// way as acting on screenshot coordinates: one way to act on a screen, whatever the bot read it
|
||
// with. (agent-desktop's own "semantic" actions were a second way with its own key names and
|
||
// failures: typing into a terminal, "/" and "`" from a background thread, focus rules.)
|
||
|
||
use agent_desktop_core as ad;
|
||
use serde_json::Value;
|
||
|
||
fn adapter() -> Box<dyn ad::PlatformAdapter> {
|
||
#[cfg(target_os = "macos")]
|
||
{
|
||
Box::new(agent_desktop_macos::MacOSAdapter::new())
|
||
}
|
||
#[cfg(target_os = "linux")]
|
||
{
|
||
Box::new(agent_desktop_linux::LinuxAdapter::new())
|
||
}
|
||
#[cfg(target_os = "windows")]
|
||
{
|
||
Box::new(agent_desktop_windows::WindowsAdapter::new())
|
||
}
|
||
}
|
||
|
||
fn err(e: ad::AppError) -> String {
|
||
let mut s = format!("{e}（{}）", e.code());
|
||
if let Some(hint) = e.suggestion() {
|
||
s.push_str(&format!("：{hint}"));
|
||
}
|
||
s
|
||
}
|
||
|
||
fn st(v: &Value, k: &str) -> String {
|
||
v.get(k).map(|x| x.as_str().map(String::from).unwrap_or_else(|| if x.is_null() { String::new() } else { x.to_string() })).unwrap_or_default().trim().to_string()
|
||
}
|
||
|
||
fn opt(v: &Value, k: &str) -> Option<String> {
|
||
Some(st(v, k)).filter(|s| !s.is_empty())
|
||
}
|
||
|
||
/// One agent-desktop command (the CLI's names): apps, windows, snapshot, launch, permissions.
|
||
pub fn run(cmd: &str, a: &Value) -> Result<Value, String> {
|
||
use ad::commands as c;
|
||
let ad = adapter();
|
||
let ad = ad.as_ref();
|
||
let ctx = ad::CommandContext::default();
|
||
let out = match cmd {
|
||
"apps" => c::list_apps::execute(c::list_apps::ListAppsArgs { app: opt(a, "app"), timeout_ms: None }, ad),
|
||
"windows" => c::list_windows::execute(c::list_windows::ListWindowsArgs { app: opt(a, "app"), timeout_ms: None }, ad),
|
||
"snapshot" => c::snapshot::execute(
|
||
c::snapshot::SnapshotArgs {
|
||
app: opt(a, "app"),
|
||
window_id: opt(a, "window_id"),
|
||
max_depth: a["max_depth"].as_u64().unwrap_or(30).min(60) as u8,
|
||
include_bounds: a["bounds"].as_bool().unwrap_or(true),
|
||
interactive_only: a["interactive_only"].as_bool().unwrap_or(true),
|
||
compact: true,
|
||
surface: ad::SnapshotSurface::default(),
|
||
skeleton: false,
|
||
root_ref: opt(a, "root"),
|
||
timeout_ms: None,
|
||
force_electron_a11y: true,
|
||
},
|
||
ad,
|
||
&ctx,
|
||
),
|
||
"launch" => c::launch::execute(c::launch::LaunchArgs { app: st(a, "app"), options: ad::system::launch_options::LaunchOptions { activate: true, ..Default::default() } }, ad),
|
||
"permissions" => {
|
||
let d = ad::Deadline::standard().map_err(|e| err(e.into()))?;
|
||
return ad.permission_report(d).map(|r| Value::String(format!("{r:?}"))).map_err(|e| err(e.into()));
|
||
}
|
||
other => return Err(format!("agent-desktop 沒有「{other}」這個動作")),
|
||
};
|
||
out.map_err(err)
|
||
}
|
||
|
||
// ---------- what the bot uses ----------
|
||
|
||
/// Where a numbered control is: its centre in screen coordinates (the space the mouse uses),
|
||
/// and what it is (「按鈕「7」」), for the step's description.
|
||
#[derive(Clone, Debug, PartialEq)]
|
||
pub struct Spot {
|
||
pub app: String,
|
||
pub x: f64,
|
||
pub y: f64,
|
||
pub what: String,
|
||
}
|
||
|
||
/// Each number of a reading and its spot (None when the app didn't say where).
|
||
pub type Spots = std::collections::HashMap<String, Option<Spot>>;
|
||
|
||
/// The latest reading: the app and its numbers.
|
||
static LAST: std::sync::Mutex<Option<(String, Spots)>> = std::sync::Mutex::new(None);
|
||
|
||
/// "e6", "[e6]", "6" or a full "@snap:e6" → "e6".
|
||
pub fn short_ref(r: &str) -> Result<String, String> {
|
||
let r = r.trim().trim_matches(['[', ']']);
|
||
let r = r.rsplit(':').next().unwrap_or(r);
|
||
let n = r.trim_start_matches('e');
|
||
if n.is_empty() || !n.chars().all(|c| c.is_ascii_digit()) {
|
||
return Err(format!("「{r}」不是元素編號，請用 computer_use_read_app 列出的 e 編號"));
|
||
}
|
||
Ok(format!("e{n}"))
|
||
}
|
||
|
||
/// Where to click for a number from the latest reading.
|
||
pub fn spot(r: &str) -> Result<Spot, String> {
|
||
let r = short_ref(r)?;
|
||
let last = LAST.lock().unwrap_or_else(|e| e.into_inner());
|
||
let (_, spots) = last.as_ref().ok_or("請先用 computer_use_read_app 讀一下那個 App 的畫面")?;
|
||
match spots.get(&r) {
|
||
Some(Some(s)) => Ok(s.clone()),
|
||
// Wayland doesn't tell apps where other windows' controls are: none has a place.
|
||
Some(None) if spots.values().all(Option::is_none) => Err(format!("這台電腦讀得到 {r}，但系統不告訴 App 它在螢幕上的位置：請用 computer_use_screenshot 看，用截圖上的 x、y 點。")),
|
||
Some(None) => Err(format!("{r} 現在不在畫面上（被捲走或藏起來了）。先捲動或改用截圖上的座標。")),
|
||
None => Err(format!("最近一次讀到的畫面裡沒有 {r}：先用 computer_use_read_app 再讀一次。")),
|
||
}
|
||
}
|
||
|
||
/// The app the latest reading was of (for the per-app permission).
|
||
pub fn last_app() -> Option<String> {
|
||
LAST.lock().unwrap_or_else(|e| e.into_inner()).as_ref().map(|l| l.0.clone())
|
||
}
|
||
|
||
fn role_name(role: &str) -> &str {
|
||
match role {
|
||
"button" => "按鈕",
|
||
"link" => "連結",
|
||
"textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield" => "輸入框",
|
||
"checkbox" | "switch" | "toggle" => "勾選框",
|
||
"radiobutton" => "選項",
|
||
"popupbutton" | "menubutton" => "選單按鈕",
|
||
"menuitem" => "選單項目",
|
||
"tab" => "分頁",
|
||
"row" | "cell" | "listitem" | "treeitem" | "outlinerow" => "項目",
|
||
"slider" => "滑桿",
|
||
"statictext" | "text" => "文字",
|
||
"image" => "圖片",
|
||
"scrollarea" => "捲動區",
|
||
"group" => "群組",
|
||
"outline" | "list" | "table" | "tree" | "grid" => "清單",
|
||
"toolbar" => "工具列",
|
||
"menubar" | "menu" => "選單",
|
||
"heading" => "標題",
|
||
"window" => "視窗",
|
||
other => other,
|
||
}
|
||
}
|
||
|
||
/// Longest value shown whole; longer ones (a terminal's scrollback, a document) keep their
|
||
/// start and their end, where a terminal's prompt and newest output are.
|
||
const VALUE_HEAD: usize = 160;
|
||
const VALUE_TAIL: usize = 400;
|
||
|
||
fn shorten(v: &str) -> String {
|
||
let n = v.chars().count();
|
||
if n <= VALUE_HEAD + VALUE_TAIL + 20 {
|
||
return v.to_string();
|
||
}
|
||
let head: String = v.chars().take(VALUE_HEAD).collect();
|
||
let tail: String = v.chars().skip(n - VALUE_TAIL).collect();
|
||
format!("{head} …（中間省略 {} 字）… {tail}", n - VALUE_HEAD - VALUE_TAIL)
|
||
}
|
||
|
||
/// The centre of a node's box, when it has a real one on the screen.
|
||
fn centre(n: &Value) -> Option<(f64, f64)> {
|
||
let b = n.get("bounds")?;
|
||
let (x, y, w, h) = (b["x"].as_f64()?, b["y"].as_f64()?, b["width"].as_f64()?, b["height"].as_f64()?);
|
||
if w < 1.0 || h < 1.0 || x + w <= 0.0 || y + h <= 0.0 {
|
||
return None;
|
||
}
|
||
Some((x + w / 2.0, y + h / 2.0))
|
||
}
|
||
|
||
/// A snapshot as short lines, and each number's spot.
|
||
pub fn lines_of(tree: &Value, app: &str) -> (Vec<String>, Spots) {
|
||
let mut lines = vec![];
|
||
let mut spots = std::collections::HashMap::new();
|
||
fn walk(n: &Value, depth: usize, lines: &mut Vec<String>, spots: &mut Spots, parent: &str, app: &str) {
|
||
if lines.len() >= 400 {
|
||
return;
|
||
}
|
||
let name = n["name"].as_str().unwrap_or("").trim();
|
||
let value = n["value"].as_str().map(str::to_string).or_else(|| n["value"].as_f64().map(|f| f.to_string())).unwrap_or_default();
|
||
let value = shorten(value.trim());
|
||
let role = n["role"].as_str().unwrap_or("");
|
||
let text = if !name.is_empty() && !value.is_empty() && value != name { format!("「{name}」：{value}") } else if !name.is_empty() { format!("「{name}」") } else if !value.is_empty() { format!("：{value}") } else { String::new() };
|
||
let mut flags = vec![];
|
||
let is_field = matches!(role, "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield");
|
||
for (k, word) in [("checked", "已勾"), ("selected", "已選"), ("disabled", "停用"), ("focused", "游標在這")] {
|
||
// Only text fields report focus usefully (Finder marks every sidebar cell focused).
|
||
if k == "focused" && !is_field {
|
||
continue;
|
||
}
|
||
if n["states"].as_array().map(|a| a.iter().any(|x| x == k)).unwrap_or(false) || n[k] == true {
|
||
flags.push(word);
|
||
}
|
||
}
|
||
let flags = if flags.is_empty() { String::new() } else { format!("（{}）", flags.join("、")) };
|
||
let indent = " ".repeat(depth.min(6));
|
||
// A cell or label that only repeats its parent's name adds nothing.
|
||
let repeats = !name.is_empty() && name == parent;
|
||
if repeats && n["ref_id"].is_null() {
|
||
// plain text under a control with the same name
|
||
} else if repeats && lines.last().map(|l| l.contains(&format!("「{name}」"))).unwrap_or(false) {
|
||
// e.g. Finder's row → cell with the same name: keep the row only
|
||
} else if let Some(r) = n["ref_id"].as_str() {
|
||
let short = r.rsplit(':').next().unwrap_or(r).to_string();
|
||
if !text.is_empty() || matches!(role, "button" | "textfield" | "textarea" | "checkbox" | "searchfield" | "popupbutton") {
|
||
let what = format!("{}{}", role_name(role), if name.is_empty() { String::new() } else { format!("「{}」", name.chars().take(30).collect::<String>()) });
|
||
spots.insert(short.clone(), centre(n).map(|(x, y)| Spot { app: app.to_string(), x, y, what }));
|
||
lines.push(format!("{indent}{short} {}{text}{flags}", role_name(role)));
|
||
}
|
||
} else if !text.is_empty() && matches!(role, "statictext" | "text" | "heading" | "cell" | "value" | "") {
|
||
lines.push(format!("{indent}{}{text}", role_name(role)));
|
||
}
|
||
let next_parent = if name.is_empty() { parent } else { name };
|
||
for c in n["children"].as_array().into_iter().flatten() {
|
||
walk(c, depth + 1, lines, spots, next_parent, app);
|
||
}
|
||
}
|
||
walk(tree, 0, &mut lines, &mut spots, "", app);
|
||
let pad = lines.iter().map(|l| l.len() - l.trim_start().len()).min().unwrap_or(0);
|
||
(lines.into_iter().map(|l| l[pad..].to_string()).collect(), spots)
|
||
}
|
||
|
||
/// An app's controls (numbered) and visible text, as short lines. `app` empty = the app in front.
|
||
/// The numbers and where they are on the screen are remembered for the next click or typing.
|
||
pub fn read_app(app: &str) -> Result<(String, Vec<String>), String> {
|
||
let mut args = serde_json::json!({"interactive_only": false, "max_depth": 40, "bounds": true});
|
||
if !app.trim().is_empty() {
|
||
args["app"] = Value::String(app.trim().to_string());
|
||
}
|
||
let v = match run("snapshot", &args) {
|
||
// Running without a window (closed, or minimised to the Dock): bring it forward once.
|
||
Err(e) if e.contains("WINDOW_NOT_FOUND") && !app.trim().is_empty() => {
|
||
let _ = run("launch", &serde_json::json!({"app": app.trim()}));
|
||
std::thread::sleep(std::time::Duration::from_millis(900));
|
||
match run("snapshot", &args) {
|
||
Ok(v) => v,
|
||
Err(e) => {
|
||
// macOS: "open" sends the reopen event, like clicking the app in the Dock.
|
||
if !cfg!(target_os = "macos") {
|
||
return Err(e);
|
||
}
|
||
let _ = std::process::Command::new("open").args(["-a", app.trim()]).status();
|
||
std::thread::sleep(std::time::Duration::from_millis(1200));
|
||
run("snapshot", &args)?
|
||
}
|
||
}
|
||
}
|
||
r => r?,
|
||
};
|
||
let app_name = v["app"].as_str().unwrap_or(app).to_string();
|
||
let (lines, spots) = lines_of(&v["tree"], &app_name);
|
||
*LAST.lock().unwrap_or_else(|e| e.into_inner()) = Some((app_name.clone(), spots));
|
||
Ok((app_name, lines))
|
||
}
|
||
|
||
/// Open an app, or bring it to the front when it runs (macOS uses LaunchServices instead).
|
||
#[cfg(not(target_os = "macos"))]
|
||
pub fn launch(app: &str) -> Result<(), String> {
|
||
run("launch", &serde_json::json!({"app": app})).map(|_| ())
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn numbers_keep_where_they_are() {
|
||
let tree = serde_json::json!({"role": "window", "name": "計算機", "children": [
|
||
{"role": "statictext", "value": "54"},
|
||
{"ref_id": "@s1:e7", "role": "button", "name": "7", "bounds": {"x": 100.0, "y": 200.0, "width": 40.0, "height": 30.0}},
|
||
{"ref_id": "@s1:e8", "role": "button", "name": "8", "bounds": {"x": 0.0, "y": 0.0, "width": 0.0, "height": 0.0}},
|
||
{"ref_id": "@s1:e9", "role": "textarea", "value": format!("{}END", "x".repeat(2000))}
|
||
]});
|
||
let (lines, spots) = lines_of(&tree, "Calculator");
|
||
assert!(lines.iter().any(|l| l == "e7 按鈕「7」"));
|
||
assert_eq!(spots["e7"], Some(Spot { app: "Calculator".into(), x: 120.0, y: 215.0, what: "按鈕「7」".into() }));
|
||
// No real box (hidden, scrolled away): known, but not clickable.
|
||
assert_eq!(spots["e8"], None);
|
||
// A terminal's whole scrollback: the end (prompt, newest output) stays.
|
||
let e9 = lines.iter().find(|l| l.starts_with("e9")).unwrap();
|
||
assert!(e9.ends_with("END") && e9.contains("中間省略") && e9.chars().count() < 700);
|
||
assert_eq!(short_ref("[e7]").unwrap(), "e7");
|
||
assert_eq!(short_ref("@s1:e7").unwrap(), "e7");
|
||
assert_eq!(short_ref("12").unwrap(), "e12");
|
||
assert!(short_ref("ok").is_err());
|
||
}
|
||
}
|