From 28c043617e1a2fdc35eecc41066a29902b2796a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=80=A7=E9=A9=8A?= Date: Sun, 8 Nov 2026 00:58:18 +0800 Subject: [PATCH] Linux: the front app is found by its program, and accessibility is switched on when it's off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit agent-desktop knows an app by its program's name, but the X11 window says its class: a Snap Firefox is "firefox_firefox", so the front app was never found and the bot got a bare picture (「看不出最前面是哪個 App（WINDOW_NOT_FOUND）」). The window's process now gives the name. When there are no controls at all and the desktop's accessibility switch (org.a11y.Status IsEnabled) is off — most apps then show none — AnyBot turns it on, as a screen reader would, and tells the bot that apps already open show their numbers after a restart. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4 --- src/computer.rs | 17 ++++++++++++++++- src/desk.rs | 25 +++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/computer.rs b/src/computer.rs index cd7f69b..4022e07 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -768,7 +768,12 @@ fn look(after: Option, pointer: bool) -> Result<(String, Option) let picture = if mac::screen_capture_preflight() { Some(capture_screen().map_err(|e| format!("看不到畫面：{e}"))?) } else { None }; // The app in front: on Linux the window system's own answer when it gives one (agent-desktop // alone goes by AT-SPI's "active" window, which some desktops never mark: no numbers, 「」). - let front = if cfg!(target_os = "linux") { mac::front_app().map(|(n, _)| n).unwrap_or_default() } else { String::new() }; + // agent-desktop knows an app by its program's name (a Snap Firefox's window says + // "firefox_firefox", its program is "firefox"). + #[cfg(target_os = "linux")] + let front = mac::front_app().map(|(n, pid)| crate::desk::program_of(pid).unwrap_or(n)).unwrap_or_default(); + #[cfg(not(target_os = "linux"))] + let front = String::new(); let (found, why) = if controls_allowed() { match crate::desktop::elements(&front).or_else(|e| if front.is_empty() { Err(e) } else { crate::desktop::elements("") }) { Ok(f) => (Some(f), String::new()), @@ -781,6 +786,15 @@ fn look(after: Option, pointer: bool) -> Result<(String, Option) ask_permission("Privacy_ScreenCapture"); return Err(NO_SCREEN_RECORDING.into()); } + // No controls at all, and the desktop's accessibility switch was off: turn it on (Linux). + #[cfg(target_os = "linux")] + let a11y_note = if controls_allowed() && found.as_ref().is_none_or(|f| f.2.is_empty()) && crate::desk::accessibility_switched_on() { + "\n（這台的無障礙介面原本是關著的，AnyBot 剛把它打開：之後打開的 App 才會有編號，已經開著的 App 要關掉重開。網頁請改用 browser_* 工具。）" + } else { + "" + }; + #[cfg(not(target_os = "linux"))] + let a11y_note = ""; let (app, window, mut els) = found.unwrap_or_default(); // Only what is on this screen gets a number (a link below the fold was numbered at the // picture's edge, and a click there landed on the edge). @@ -828,6 +842,7 @@ fn look(after: Option, pointer: bool) -> Result<(String, Option) } else { text.push_str(&format!("\n「{app}」上可以操作的東西（圖上同樣的編號，用 ref 點）：\n{}", lines.join("\n"))); } + text.push_str(a11y_note); Ok((text, img_b64)) } diff --git a/src/desk.rs b/src/desk.rs index be4fafe..d9169d8 100644 --- a/src/desk.rs +++ b/src/desk.rs @@ -262,6 +262,31 @@ pub fn front_app() -> Option<(String, i32)> { } } +// ---------- accessibility (the numbered controls) ---------- + +/// The program a window belongs to, the way agent-desktop names apps (the executable's name): a +/// Snap Firefox's window says "firefox_firefox", its program is "firefox". +pub fn program_of(pid: i32) -> Option { + if pid <= 0 { + return None; + } + let exe = std::fs::read_link(format!("/proc/{pid}/exe")).ok()?; + let name = exe.file_name()?.to_str()?.trim_end_matches(" (deleted)").to_string(); + (!name.is_empty()).then_some(name) +} + +/// Turn the desktop's accessibility switch on when it is off (what a screen reader does): +/// without it most apps never show their controls, so the bot gets no numbers. True when it was +/// off and is on now; apps already open show theirs once restarted. +pub fn accessibility_switched_on() -> bool { + let Ok(conn) = zbus::blocking::Connection::session() else { return false }; + let Ok(p) = zbus::blocking::Proxy::new(&conn, "org.a11y.Bus", "/org/a11y/bus", "org.a11y.Status") else { return false }; + if p.get_property::("IsEnabled").unwrap_or(true) { + return false; + } + p.set_property("IsEnabled", true).is_ok() +} + // ---------- idle and screensaver ---------- fn dbus_call(dest: &str, path: &str, iface: &str, method: &str) -> Option {