267 lines
13 KiB
Rust
267 lines
13 KiB
Rust
|
// Seeing and using other apps through the OS accessibility APIs, with vercel-labs/agent-desktop
|
|||
|
|
// compiled in (AX on macOS, AT-SPI on Linux, UI Automation on Windows). A snapshot gives every
|
|||
|
|
// control of an app a ref; actions take a ref. No screenshots needed, and by default no pointer
|
|||
|
|
// movement or focus change (the person can keep working).
|
|||
|
|
|
|||
|
|
use agent_desktop_core as ad;
|
|||
|
|
use serde_json::Value;
|
|||
|
|
|
|||
|
|
fn adapter() -> Box<dyn ad::PlatformAdapter> {
|
|||
|
|
#[cfg(target_os = "macos")]
|
|||
|
|
{
|
|||
|
|
Box::new(agent_desktop_macos::MacOSAdapter::new())
|
|||
|
|
}
|
|||
|
|
#[cfg(target_os = "linux")]
|
|||
|
|
{
|
|||
|
|
Box::new(agent_desktop_linux::LinuxAdapter::new())
|
|||
|
|
}
|
|||
|
|
#[cfg(target_os = "windows")]
|
|||
|
|
{
|
|||
|
|
Box::new(agent_desktop_windows::WindowsAdapter::new())
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
fn err(e: ad::AppError) -> String {
|
|||
|
|
let mut s = format!("{e}（{}）", e.code());
|
|||
|
|
if let Some(hint) = e.suggestion() {
|
|||
|
|
s.push_str(&format!("：{hint}"));
|
|||
|
|
}
|
|||
|
|
s
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
fn st(v: &Value, k: &str) -> String {
|
|||
|
|
v.get(k).map(|x| x.as_str().map(String::from).unwrap_or_else(|| if x.is_null() { String::new() } else { x.to_string() })).unwrap_or_default().trim().to_string()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
fn opt(v: &Value, k: &str) -> Option<String> {
|
|||
|
|
Some(st(v, k)).filter(|s| !s.is_empty())
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// One agent-desktop command (the CLI's names): apps, windows, snapshot, click, double_click,
|
|||
|
|
/// right_click, type, set_value, select, press, scroll, focus_window, launch, screenshot.
|
|||
|
|
pub fn run(cmd: &str, a: &Value) -> Result<Value, String> {
|
|||
|
|
use ad::commands as c;
|
|||
|
|
let ad = adapter();
|
|||
|
|
let ad = ad.as_ref();
|
|||
|
|
let ctx = ad::CommandContext::default();
|
|||
|
|
let r = |rf: &Value| c::helpers::RefArgs { ref_id: st(rf, "ref"), timeout_ms: None };
|
|||
|
|
let out = match cmd {
|
|||
|
|
"apps" => c::list_apps::execute(c::list_apps::ListAppsArgs { app: opt(a, "app"), timeout_ms: None }, ad),
|
|||
|
|
"windows" => c::list_windows::execute(c::list_windows::ListWindowsArgs { app: opt(a, "app"), timeout_ms: None }, ad),
|
|||
|
|
"snapshot" => c::snapshot::execute(
|
|||
|
|
c::snapshot::SnapshotArgs {
|
|||
|
|
app: opt(a, "app"),
|
|||
|
|
window_id: opt(a, "window_id"),
|
|||
|
|
max_depth: a["max_depth"].as_u64().unwrap_or(30).min(60) as u8,
|
|||
|
|
include_bounds: false,
|
|||
|
|
interactive_only: a["interactive_only"].as_bool().unwrap_or(true),
|
|||
|
|
compact: true,
|
|||
|
|
surface: ad::SnapshotSurface::default(),
|
|||
|
|
skeleton: false,
|
|||
|
|
root_ref: opt(a, "root"),
|
|||
|
|
timeout_ms: None,
|
|||
|
|
force_electron_a11y: true,
|
|||
|
|
},
|
|||
|
|
ad,
|
|||
|
|
&ctx,
|
|||
|
|
),
|
|||
|
|
"click" => c::click::execute(r(a), ad, &ctx),
|
|||
|
|
"double_click" => c::double_click::execute(r(a), ad, &ctx),
|
|||
|
|
"right_click" => c::right_click::execute(r(a), ad, &ctx),
|
|||
|
|
"type" => c::type_text::execute(c::type_text::TypeArgs { ref_id: st(a, "ref"), text: st(a, "text"), timeout_ms: None, interval_ms: None }, ad, &ctx),
|
|||
|
|
"set_value" => c::set_value::execute(c::set_value::SetValueArgs { ref_id: st(a, "ref"), value: st(a, "value"), timeout_ms: None }, ad, &ctx),
|
|||
|
|
"select" => c::select::execute(c::select::SelectArgs { ref_id: st(a, "ref"), value: st(a, "value"), timeout_ms: None }, ad, &ctx),
|
|||
|
|
"press" => c::press::execute(c::press::PressArgs { combo: st(a, "combo"), app: opt(a, "app"), window_id: opt(a, "window_id"), force: false }, ad, &ctx),
|
|||
|
|
"scroll" => {
|
|||
|
|
let direction = match st(a, "direction").as_str() {
|
|||
|
|
"up" => ad::Direction::Up,
|
|||
|
|
"left" => ad::Direction::Left,
|
|||
|
|
"right" => ad::Direction::Right,
|
|||
|
|
_ => ad::Direction::Down,
|
|||
|
|
};
|
|||
|
|
c::scroll::execute(c::scroll::ScrollArgs { ref_id: st(a, "ref"), direction, amount: a["amount"].as_u64().unwrap_or(3) as u32, timeout_ms: None }, ad, &ctx)
|
|||
|
|
}
|
|||
|
|
"focus_window" => c::focus_window::execute(c::focus_window::FocusWindowArgs { window_id: opt(a, "window_id"), app: opt(a, "app"), title: opt(a, "title") }, ad),
|
|||
|
|
"launch" => c::launch::execute(c::launch::LaunchArgs { app: st(a, "app"), options: ad::system::launch_options::LaunchOptions { activate: true, ..Default::default() } }, ad),
|
|||
|
|
"permissions" => {
|
|||
|
|
let d = ad::Deadline::standard().map_err(|e| err(e.into()))?;
|
|||
|
|
return ad.permission_report(d).map(|r| Value::String(format!("{r:?}"))).map_err(|e| err(e.into()));
|
|||
|
|
}
|
|||
|
|
other => return Err(format!("agent-desktop 沒有「{other}」這個動作")),
|
|||
|
|
};
|
|||
|
|
out.map_err(err)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- what the bot uses ----------
|
|||
|
|
|
|||
|
|
/// The latest snapshot (id, app) so the bot can say "e6" instead of the full ref.
|
|||
|
|
static LAST: std::sync::Mutex<Option<(String, String)>> = std::sync::Mutex::new(None);
|
|||
|
|
|
|||
|
|
/// "e6", "[e6]", "6" or a full "@snap:e6" → the full ref of the latest snapshot.
|
|||
|
|
pub fn full_ref(r: &str) -> Result<String, String> {
|
|||
|
|
let r = r.trim().trim_matches(['[', ']']);
|
|||
|
|
if r.starts_with('@') {
|
|||
|
|
return Ok(r.to_string());
|
|||
|
|
}
|
|||
|
|
let n = r.trim_start_matches('e');
|
|||
|
|
if n.is_empty() || !n.chars().all(|c| c.is_ascii_digit()) {
|
|||
|
|
return Err(format!("「{r}」不是元素編號，請用 computer_use_read_app 列出的 e 編號"));
|
|||
|
|
}
|
|||
|
|
let last = LAST.lock().unwrap().clone().ok_or("請先用 computer_use_read_app 看一下 App 的畫面")?;
|
|||
|
|
Ok(format!("@{}:e{n}", last.0))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// The app the latest snapshot was of (for the per-app permission).
|
|||
|
|
pub fn last_app() -> Option<String> {
|
|||
|
|
LAST.lock().unwrap().as_ref().map(|l| l.1.clone())
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
fn role_name(role: &str) -> &str {
|
|||
|
|
match role {
|
|||
|
|
"button" => "按鈕",
|
|||
|
|
"link" => "連結",
|
|||
|
|
"textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield" => "輸入框",
|
|||
|
|
"checkbox" | "switch" | "toggle" => "勾選框",
|
|||
|
|
"radiobutton" => "選項",
|
|||
|
|
"popupbutton" | "menubutton" => "選單按鈕",
|
|||
|
|
"menuitem" => "選單項目",
|
|||
|
|
"tab" => "分頁",
|
|||
|
|
"row" | "cell" | "listitem" | "treeitem" | "outlinerow" => "項目",
|
|||
|
|
"slider" => "滑桿",
|
|||
|
|
"statictext" | "text" => "文字",
|
|||
|
|
"image" => "圖片",
|
|||
|
|
"scrollarea" => "捲動區",
|
|||
|
|
"group" => "群組",
|
|||
|
|
"outline" | "list" | "table" | "tree" | "grid" => "清單",
|
|||
|
|
"toolbar" => "工具列",
|
|||
|
|
"menubar" | "menu" => "選單",
|
|||
|
|
"heading" => "標題",
|
|||
|
|
"window" => "視窗",
|
|||
|
|
other => other,
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// An app's controls (numbered) and visible text, as short lines. `app` empty = the app in front.
|
|||
|
|
pub fn read_app(app: &str) -> Result<(String, Vec<String>), String> {
|
|||
|
|
read_app_as(app, true)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
fn read_app_as(app: &str, remember: bool) -> Result<(String, Vec<String>), String> {
|
|||
|
|
let mut args = serde_json::json!({"interactive_only": false, "max_depth": 40});
|
|||
|
|
if !app.trim().is_empty() {
|
|||
|
|
args["app"] = Value::String(app.trim().to_string());
|
|||
|
|
}
|
|||
|
|
let v = match run("snapshot", &args) {
|
|||
|
|
// Running without a window (closed, or minimised to the Dock): bring it forward once.
|
|||
|
|
Err(e) if e.contains("WINDOW_NOT_FOUND") && !app.trim().is_empty() => {
|
|||
|
|
let _ = run("launch", &serde_json::json!({"app": app.trim()}));
|
|||
|
|
std::thread::sleep(std::time::Duration::from_millis(900));
|
|||
|
|
match run("snapshot", &args) {
|
|||
|
|
Ok(v) => v,
|
|||
|
|
Err(e) => {
|
|||
|
|
// macOS: "open" sends the reopen event, like clicking the app in the Dock.
|
|||
|
|
if !cfg!(target_os = "macos") {
|
|||
|
|
return Err(e);
|
|||
|
|
}
|
|||
|
|
let _ = std::process::Command::new("open").args(["-a", app.trim()]).status();
|
|||
|
|
std::thread::sleep(std::time::Duration::from_millis(1200));
|
|||
|
|
run("snapshot", &args)?
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
r => r?,
|
|||
|
|
};
|
|||
|
|
let snap = v["snapshot_id"].as_str().unwrap_or("").to_string();
|
|||
|
|
let app_name = v["app"].as_str().unwrap_or(app).to_string();
|
|||
|
|
if remember {
|
|||
|
|
*LAST.lock().unwrap() = Some((snap.clone(), app_name.clone()));
|
|||
|
|
}
|
|||
|
|
let mut lines = vec![];
|
|||
|
|
fn walk(n: &Value, depth: usize, lines: &mut Vec<String>, parent: &str) {
|
|||
|
|
if lines.len() >= 400 {
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
let name = n["name"].as_str().unwrap_or("").trim();
|
|||
|
|
let value = n["value"].as_str().map(str::to_string).or_else(|| n["value"].as_f64().map(|f| f.to_string())).unwrap_or_default();
|
|||
|
|
let role = n["role"].as_str().unwrap_or("");
|
|||
|
|
let text = if !name.is_empty() && !value.is_empty() && value != name { format!("「{name}」：{value}") } else if !name.is_empty() { format!("「{name}」") } else if !value.is_empty() { format!("：{value}") } else { String::new() };
|
|||
|
|
let mut flags = vec![];
|
|||
|
|
let is_field = matches!(role, "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield");
|
|||
|
|
for (k, word) in [("checked", "已勾"), ("selected", "已選"), ("disabled", "停用"), ("focused", "游標在這")] {
|
|||
|
|
// Only text fields report focus usefully (Finder marks every sidebar cell focused).
|
|||
|
|
if k == "focused" && !is_field {
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
if n["states"].as_array().map(|a| a.iter().any(|x| x == k)).unwrap_or(false) || n[k] == true {
|
|||
|
|
flags.push(word);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
let flags = if flags.is_empty() { String::new() } else { format!("（{}）", flags.join("、")) };
|
|||
|
|
let indent = " ".repeat(depth.min(6));
|
|||
|
|
// A cell or label that only repeats its parent's name adds nothing.
|
|||
|
|
let repeats = !name.is_empty() && name == parent;
|
|||
|
|
if repeats && n["ref_id"].is_null() {
|
|||
|
|
// plain text under a control with the same name
|
|||
|
|
} else if repeats && lines.last().map(|l| l.contains(&format!("「{name}」"))).unwrap_or(false) {
|
|||
|
|
// e.g. Finder's row → cell with the same name: keep the row only
|
|||
|
|
} else if let Some(r) = n["ref_id"].as_str() {
|
|||
|
|
let short = r.rsplit(':').next().unwrap_or(r);
|
|||
|
|
if !text.is_empty() || matches!(role, "button" | "textfield" | "textarea" | "checkbox" | "searchfield" | "popupbutton") {
|
|||
|
|
lines.push(format!("{indent}{short} {}{text}{flags}", role_name(role)));
|
|||
|
|
}
|
|||
|
|
} else if !text.is_empty() && matches!(role, "statictext" | "text" | "heading" | "cell" | "value" | "") {
|
|||
|
|
lines.push(format!("{indent}{}{text}", role_name(role)));
|
|||
|
|
}
|
|||
|
|
let next_parent = if name.is_empty() { parent } else { name };
|
|||
|
|
for c in n["children"].as_array().into_iter().flatten() {
|
|||
|
|
walk(c, depth + 1, lines, next_parent);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
walk(&v["tree"], 0, &mut lines, "");
|
|||
|
|
let pad = lines.iter().map(|l| l.len() - l.trim_start().len()).min().unwrap_or(0);
|
|||
|
|
let lines = lines.into_iter().map(|l| l[pad..].to_string()).collect();
|
|||
|
|
Ok((app_name, lines))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// Act on an element by its number: click, double_click, right_click, type, set_value, select,
|
|||
|
|
/// scroll. Returns what the app shows now (new text since the action).
|
|||
|
|
pub fn act(cmd: &str, r: &str, extra: &Value) -> Result<String, String> {
|
|||
|
|
let full = full_ref(r)?;
|
|||
|
|
let app = last_app().unwrap_or_default();
|
|||
|
|
let before = read_app_as(&app, false).map(|x| x.1).unwrap_or_default();
|
|||
|
|
let mut a = extra.clone();
|
|||
|
|
if !a.is_object() {
|
|||
|
|
a = serde_json::json!({});
|
|||
|
|
}
|
|||
|
|
a["ref"] = Value::String(full.clone());
|
|||
|
|
let res = run(cmd, &a);
|
|||
|
|
// Typing into some fields fails as keystrokes; setting the value works there.
|
|||
|
|
let res = match (cmd, res) {
|
|||
|
|
("type", Err(_)) => run("set_value", &serde_json::json!({"ref": full, "value": a["text"]})),
|
|||
|
|
(_, r) => r,
|
|||
|
|
};
|
|||
|
|
res?;
|
|||
|
|
std::thread::sleep(std::time::Duration::from_millis(400));
|
|||
|
|
let (_, after) = read_app_as(&app, false).unwrap_or_default();
|
|||
|
|
let old: std::collections::HashSet<String> = before.iter().map(|l| l.split_whitespace().skip(1).collect::<Vec<_>>().join(" ")).collect();
|
|||
|
|
let new: Vec<&String> = after.iter().filter(|l| !old.contains(&l.split_whitespace().skip(1).collect::<Vec<_>>().join(" "))).take(30).collect();
|
|||
|
|
// Show what changed without numbers (they belong to a different reading); if controls came
|
|||
|
|
// or went, the bot's numbers may now point elsewhere.
|
|||
|
|
let strip = |l: &str| -> String {
|
|||
|
|
let t = l.trim();
|
|||
|
|
match t.split_once(' ') {
|
|||
|
|
Some((first, rest)) if first.starts_with('e') && first[1..].chars().all(|c| c.is_ascii_digit()) => rest.to_string(),
|
|||
|
|
_ => t.to_string(),
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
let controls_changed = new.iter().any(|l| l.trim().starts_with('e')) || before.iter().filter(|l| l.trim().starts_with('e')).count() != after.iter().filter(|l| l.trim().starts_with('e')).count();
|
|||
|
|
let mut out = format!("好了（在「{app}」）。");
|
|||
|
|
if !new.is_empty() {
|
|||
|
|
out.push_str(&format!("\n現在畫面上新出現：\n{}", new.iter().map(|l| strip(l)).collect::<Vec<_>>().join("\n")));
|
|||
|
|
}
|
|||
|
|
if controls_changed {
|
|||
|
|
out.push_str("\n（按鈕或欄位有變動，編號可能已經不同：下一步前先用 computer_use_read_app 看新的清單。）");
|
|||
|
|
}
|
|||
|
|
Ok(out)
|
|||
|
|
}
|