From 22eaf031b00085f231b0a0098b0a838b0f0d3d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=80=A7=E9=A9=8A?= Date: Mon, 9 Nov 2026 14:58:40 +0800 Subject: [PATCH] The quick model runs again, the Mac's own menus get numbers, and an unread window is said MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - quick_tools indexed serde_json Values mutably, which inserted actions.items.properties = null into click/typing/scroll; xAI refused the whole tool list, so since ba1fce5 every step fell back to the main model. It now uses get_mut and the test checks nothing but ref changes. - The front app's menu bar titles and any open menu (submenus, right-click menus) are listed with numbers and their shortcuts; an option-key alternate in the same place is noted on the visible item. - Disclosure triangles, colour wells and date fields get numbers; an icon-only control without a name goes by its description. - When the front window can't be read the list says so (with the reason) instead of 「」 above the menu bar alone. - Computer facts add each system's conventions (Mac: cmd+shift+G, save panel, default format, /bin/sh echo -n; Linux: GTK ctrl+L). - `anybot --help` and unknown -- flags print the usage and exit instead of opening the window on the real data folder. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS --- FEATURES.md | 6 +- src/agent.rs | 20 ++++-- src/axread.rs | 179 ++++++++++++++++++++++++++++++++++++++++++++++++ src/computer.rs | 48 ++++++++++++- src/desktop.rs | 21 +++++- src/facts.rs | 10 +++ src/main.rs | 18 +++++ 7 files changed, 295 insertions(+), 7 deletions(-) diff --git a/FEATURES.md b/FEATURES.md index a59f8b7..587cfef 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -592,7 +592,7 @@ AnyBot 是一個桌面聊天 App，介面和運作方式照 Grok Bot 重做。 - 快速模型叫不動（帳號沒有這個模型、出錯）時，這一回合剩下的步驟全部改用 Bot 的模型。 - 快速模型的步驟不即時顯示文字（它的答案可能被擱置），工具步驟照常顯示；每一則回覆標的是實際用的模型。 - 依據：量測時九成以上的時間是會思考的模型在每一步之前思考（grok-4.7 每步約 10 秒，快速模型約 1 秒）；快速模型單獨做時，簡單的工作全對，難的工作會太早收工，所以由會思考的模型規劃和收尾。 -- **電腦現況**：每個回合開始時，系統提示會附上每台電腦的現況，讓模型直接用那台最直接的做法，不用花步驟試探：系統和版本、桌面、有沒有 systemd（沒有時會說 systemctl、timedatectl 不能用，改設定直接改檔案）、能不能 sudo、有沒有網路、用什麼裝套件、有哪些常用指令和 App，以及固定的 TZ 環境變數（版本 7 以前的 box 映像檔設了 TZ=UTC，會蓋過 /etc/localtime，改了系統時區 date 還是 UTC；版本 8 拿掉了，舊的 box 更新重建前現況會提醒）。 +- **電腦現況**：每個回合開始時，系統提示會附上每台電腦的現況，讓模型直接用那台最直接的做法，不用花步驟試探：系統和版本、桌面、有沒有 systemd（沒有時會說 systemctl、timedatectl 不能用，改設定直接改檔案）、能不能 sudo、有沒有網路、用什麼裝套件、有哪些常用指令和 App，以及固定的 TZ 環境變數（版本 7 以前的 box 映像檔設了 TZ=UTC，會蓋過 /etc/localtime，改了系統時區 date 還是 UTC；版本 8 拿掉了，舊的 box 更新重建前現況會提醒）。也附上那個系統的操作慣例，沒用過的 App 也照這些做：Mac 在存檔、開檔對話框和 Finder 按 cmd+shift+G 直接打完整路徑；存檔對話框在「儲存為」打檔名按 Return 就存好；cmd+W 關視窗、cmd+Q 結束 App；檔案格式、副檔名使用者沒說就照預設，不要另外改（Bot 曾經存好 note.rtf 之後又花了 6 分鐘想把格式改掉）。Linux 桌面：GTK 對話框按 ctrl+L 打路徑；ctrl+S、ctrl+W、ctrl+Q。Mac 另外提醒：指令用 /bin/sh 跑，echo -n 會把「-n」寫出去，要用 printf；讀到的和預期不同就照實說（Bot 曾經把寫成「-n hello」的檔案說成「hello」）。 - 一台電腦讀一次（一段唯讀的小腳本，約 0.2 秒），30 分鐘內不再讀；一個回合最多等 6 秒，來不及的下一回合用。 - Bot 的電腦一律讀；使用者的電腦只有 Bot 已經可以在上面做事時才讀；AnyBot 所在的這台由 AnyBot 自己讀，不算 Bot 執行的指令。 - **做事的方式**（系統提示）：先選路（專用工具、指令、最後才畫面）；動手前想好要做哪幾步、做完怎樣算成功；選定一條路做到底；同一個做法失敗兩次就換；做完只用一種最直接的方法確認一次；做不到就照實說，不用替代品假裝做到（例如只改環境變數讓時鐘看起來對）；會切斷連線的事先問。 @@ -1160,6 +1160,9 @@ computer_use_screenshot、click、mouse_move、mouse_button（按住／放開） - 只把最近 3 張截圖給模型看（估計值），較早的改成「（較早的截圖已省略）」。截圖不會存到硬碟。 - 成功：「你的螢幕截圖（寬×高）。之後的座標請用這張圖上的像素位置。」 - Mac 選單列右上角的圖示（Wi‑Fi、控制中心、時鐘、Tailscale、LINE…）也有編號，叫「選單列圖示」：從每個 App 的 AXExtrasMenuBar 讀，系統自己的圖示用它裡面那一層的字（「Wi‑Fi，已連接，3格」），同時問 8 個 App，一次約 0.3 秒；15 秒內用上次讀的，過期就在背景重讀，不讓截圖等。 +- Mac 最前面那個 App 自己的選單（「檔案」「編輯」「格式」…）也有編號，叫「選單」；拉下來的選單、從它打開的子選單、按右鍵出來的選單，裡面的項目叫「選單項目」，有快捷鍵的會寫「快捷鍵 option+shift+cmd+s」，下次可以直接按。同一個位置按住 option 才會出現的項目（「複製」底下的「儲存為⋯」）不另外編號，寫在看得到的那個旁邊：「按住 option 時這裡是「儲存為⋯」」。每次看畫面直接讀（文字編輯約 0.05–0.13 秒），App 0.25 秒不回應就略過。以前這些都沒有編號，Bot 要放大截圖或自己寫 osascript 去找。 +- 讀不到最前面 App 視窗裡的按鈕時（App 沒回應、正好在切換），清單照樣列出選單列，但會先說「讀不到最前面「某某」視窗裡的按鈕（原因）：下面只有選單列」。以前只寫「「」上可以操作的東西」，Bot 不知道視窗沒讀到，猜了 8 分鐘的名字。 +- 只有圖示的控制項沒有名字時，用它的說明當名字（存檔對話框的展開鈕「顯示較多選項」）；展開鈕、顏色、日期這幾種控制項也有編號了（以前沒有，Bot 只能照座標點，常點不到）。 ### 「放大看」（computer_use_zoom） @@ -1789,6 +1792,7 @@ computer_use_screenshot、click、mouse_move、mouse_button（按住／放開） ### 不開視窗執行（`anybot --serve`） +- `anybot --help`（或 `-h`）印出用法就結束；打錯的 `--` 選項也一樣（說「不認得」，結束碼 2）。以前這兩種都會直接打開視窗，用的是真正的資料夾。macOS 自己傳的單橫線參數不受影響。 - Bot、例行任務、事件觸發、訊息管道和手機網頁照常運作，你用手機操作。 - 可以跑在 Mac 或 Linux。Linux 版已在 Docker 裡編譯並測試過。 - 環境變數： diff --git a/src/agent.rs b/src/agent.rs index bba401a..4887768 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -866,11 +866,16 @@ fn quick_tools(tools: &[serde_json::Value]) -> Vec { let mut t = t.clone(); let screen = t["name"].as_str().is_some_and(|n| matches!(n, "computer_use_click" | "computer_use_typing" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_batch")); if screen { - if let Some(p) = t["parameters"]["properties"].as_object_mut() { - p.remove("ref"); - } - if let Some(p) = t["parameters"]["properties"]["actions"]["items"]["properties"].as_object_mut() { + // `get_mut`, not `t[..]`: indexing a Value mutably inserts every missing key, which + // once gave click and typing an `actions` with `properties: null` — the provider then + // refused the whole tool list and the quick model never ran. + let props = t.get_mut("parameters").and_then(|p| p.get_mut("properties")).and_then(|p| p.as_object_mut()); + if let Some(p) = props { p.remove("ref"); + let items = p.get_mut("actions").and_then(|a| a.get_mut("items")).and_then(|i| i.get_mut("properties")).and_then(|p| p.as_object_mut()); + if let Some(items) = items { + items.remove("ref"); + } } if let Some(d) = t["description"].as_str() { t["description"] = serde_json::Value::String(format!("{d} Name the control exactly as the latest list shows it (name); there are no numbers here.")); @@ -2025,6 +2030,13 @@ mod tests { assert!(t["parameters"]["properties"]["actions"]["items"]["properties"].get("ref").is_none()); } } + // Nothing else changes shape: only `ref` is gone (a stray key here makes the provider refuse + // the whole request). + for (a, b) in specs.iter().zip(&quick) { + let keys = |t: &serde_json::Value| t["parameters"]["properties"].as_object().map(|p| p.keys().filter(|k| *k != "ref").cloned().collect::>()); + assert_eq!(keys(a), keys(b), "{}", a["name"]); + } + assert!(!serde_json::to_string(&quick).unwrap().contains(":null"), "a key with no value"); // The bot's own model keeps both. assert!(specs.iter().any(|t| t["name"] == "computer_use_click" && t["parameters"]["properties"].get("ref").is_some())); } diff --git a/src/axread.rs b/src/axread.rs index ef47b71..0d3de8a 100644 --- a/src/axread.rs +++ b/src/axread.rs @@ -114,6 +114,8 @@ mod mac { fn CFStringGetMaximumSizeForEncoding(len: isize, enc: u32) -> isize; fn CFStringGetCString(s: CFTypeRef, buf: *mut c_char, size: isize, enc: u32) -> bool; fn CFURLGetString(u: CFTypeRef) -> CFTypeRef; + fn CFNumberGetTypeID() -> usize; + fn CFNumberGetValue(n: CFTypeRef, kind: isize, out: *mut c_void) -> bool; fn CFDictionaryCreate(alloc: CFTypeRef, keys: *const CFTypeRef, values: *const CFTypeRef, n: isize, kcb: *const c_void, vcb: *const c_void) -> CFTypeRef; static kCFBooleanTrue: CFTypeRef; static kCFTypeDictionaryKeyCallBacks: [usize; 6]; @@ -328,6 +330,122 @@ mod mac { attr_string(el, "AXIdentifier").map(|id| super::menu_extra_name(&id)).unwrap_or_default() } + /// One entry of an app's menus: a title in its menu bar (「檔案」), or an item of a menu that is + /// open now, with its keyboard shortcut when it has one. + pub struct MenuEntry { + pub label: String, + pub rect: [f64; 4], + /// 0: in the menu bar; 1: in the open menu; 2 and more: in a submenu opened from it. + pub depth: u8, + pub enabled: bool, + pub shortcut: String, + /// The item macOS shows in this place while option is held (「複製」 hides 「儲存為⋯」). + pub alternate: Option<(String, String)>, + } + + fn rect_of(el: CFTypeRef) -> Option<[f64; 4]> { + #[repr(C)] + #[derive(Default, Clone, Copy)] + struct Xy { + a: f64, + b: f64, + } + let get = |k: &str, kind: u32| -> Option { + let v = attr(el, k)?; + let mut xy = Xy::default(); + unsafe { AXValueGetValue(v.0, kind, &mut xy as *mut Xy as *mut c_void) }.then_some(xy) + }; + // kAXValueCGPointType = 1, kAXValueCGSizeType = 2 + let (p, z) = (get("AXPosition", 1)?, get("AXSize", 2)?); + (z.a >= 1.0 && z.b >= 1.0).then_some([p.a, p.b, z.a, z.b]) + } + + fn attr_bool(el: CFTypeRef, name: &str) -> Option { + attr(el, name).map(|v| v.0 == unsafe { kCFBooleanTrue }) + } + + fn attr_int(el: CFTypeRef, name: &str) -> Option { + let v = attr(el, name)?; + let mut n: i64 = 0; + // kCFNumberSInt64Type = 4 + unsafe { (CFGetTypeID(v.0) == CFNumberGetTypeID() && CFNumberGetValue(v.0, 4, &mut n as *mut i64 as *mut c_void)).then_some(n) } + } + + /// The menus of the app `pid`: its menu bar's titles, and the items of whichever menu is open + /// (one pulled down from the bar, a submenu opened from it, or a right-click menu). Nothing + /// when the app doesn't answer within a quarter second. + pub fn app_menus(pid: i32) -> Vec { + let app = Cf(unsafe { AXUIElementCreateApplication(pid) }); + if app.0.is_null() { + return vec![]; + } + unsafe { AXUIElementSetMessagingTimeout(app.0, 0.25) }; + let mut out = vec![]; + if let Some(bar) = attr(app.0, "AXMenuBar") { + let (_keep, items) = children(bar.0); + for it in items { + let Some(rect) = rect_of(it) else { continue }; + let label = attr_string(it, "AXTitle").unwrap_or_default().trim().to_string(); + if label.is_empty() || rect[1] > 60.0 { + continue; + } + out.push(MenuEntry { label, rect, depth: 0, enabled: attr_bool(it, "AXEnabled").unwrap_or(true), shortcut: String::new(), alternate: None }); + if attr_bool(it, "AXSelected") == Some(true) { + let (_k, menus) = children(it); + for m in menus { + menu_items(m, 1, &mut out); + } + } + } + } + // A right-click menu hangs off the app itself. + let (_keep, kids) = children(app.0); + for k in kids { + if attr_string(k, "AXRole").as_deref() == Some("AXMenu") { + menu_items(k, 1, &mut out); + } + } + out + } + + fn menu_items(menu: CFTypeRef, depth: u8, out: &mut Vec) { + if depth > 3 || out.len() > 300 { + return; + } + let (_keep, items) = children(menu); + for it in items { + let label = attr_string(it, "AXTitle").unwrap_or_default().trim().to_string(); + // Separators have no title; a closed menu's items have no place on the screen. + let Some(rect) = rect_of(it) else { continue }; + if label.is_empty() { + continue; + } + let shortcut = attr_string(it, "AXMenuItemCmdChar").filter(|c| !c.is_empty()).map(|c| shortcut_words(&c, attr_int(it, "AXMenuItemCmdModifiers").unwrap_or(0))).unwrap_or_default(); + let entry = MenuEntry { label, rect, depth, enabled: attr_bool(it, "AXEnabled").unwrap_or(true), shortcut, alternate: None }; + // Two items in one place: one of them is the option-key alternate of the other, and + // only one shows. AX says nothing about which (AXMenuItemPrimaryUIElement is empty), + // so the one that needs option is taken for the hidden one. + if let Some(last) = out.last_mut().filter(|l| l.depth == depth && l.rect == entry.rect && l.alternate.is_none()) { + let needs_option = |e: &MenuEntry| e.shortcut.contains("option"); + if needs_option(last) && !needs_option(&entry) { + let hidden = std::mem::replace(last, entry); + last.alternate = Some((hidden.label, hidden.shortcut)); + } else { + last.alternate = Some((entry.label, entry.shortcut)); + } + continue; + } + let entry_selected = attr_bool(it, "AXSelected") == Some(true); + out.push(entry); + if entry_selected { + let (_k, subs) = children(it); + for m in subs { + menu_items(m, depth + 1, out); + } + } + } + } + fn status_items_of(apps: &[(i32, String)]) -> Vec { #[repr(C)] #[derive(Default, Clone, Copy)] @@ -363,11 +481,48 @@ mod mac { } } +#[cfg(target_os = "macos")] +pub use mac::app_menus; #[cfg(target_os = "macos")] pub use mac::trusted; #[cfg(target_os = "macos")] pub use mac::status_items; +/// A menu item's shortcut as the bot presses it (computer_use_press_key): AXMenuItemCmdModifiers +/// is 0 for ⌘ alone, plus 1 shift, 2 option, 4 control, 8 without ⌘. +#[cfg_attr(not(target_os = "macos"), allow(dead_code))] +pub fn shortcut_words(key: &str, mods: i64) -> String { + let mut parts = vec![]; + if mods & 4 != 0 { + parts.push("ctrl"); + } + if mods & 2 != 0 { + parts.push("option"); + } + if mods & 1 != 0 { + parts.push("shift"); + } + if mods & 8 == 0 { + parts.push("cmd"); + } + // macOS writes arrows and function keys as private-use characters (NSUpArrowFunctionKey…). + let key = match key { + "\u{8}" | "\u{7f}" => "delete".to_string(), + "\r" | "\u{3}" => "return".to_string(), + "\u{1b}" => "escape".to_string(), + "\t" => "tab".to_string(), + " " => "space".to_string(), + "\u{f700}" => "up".to_string(), + "\u{f701}" => "down".to_string(), + "\u{f702}" => "left".to_string(), + "\u{f703}" => "right".to_string(), + k if k.chars().count() == 1 && ('\u{f704}'..='\u{f726}').contains(&k.chars().next().unwrap_or(' ')) => format!("f{}", k.chars().next().unwrap_or(' ') as u32 - 0xf703), + k => k.trim().to_lowercase(), + }; + parts.push(&key); + parts.join("+") +} + /// A system menu-bar icon's words from its identifier (com.apple.menuextra.wifi → Wi-Fi): the /// system's own icons carry no title. #[cfg_attr(not(target_os = "macos"), allow(dead_code))] @@ -445,6 +600,16 @@ mod tests { assert_eq!(pick_person_pid(" 1 /Applications/Safari.app/Contents/MacOS/Safari", exe), None); } + #[test] + fn a_menu_items_shortcut_reads_as_keys_to_press() { + assert_eq!(shortcut_words("S", 0), "cmd+s"); + assert_eq!(shortcut_words("S", 3), "option+shift+cmd+s"); + assert_eq!(shortcut_words("F", 4), "ctrl+cmd+f"); + assert_eq!(shortcut_words("\u{8}", 0), "cmd+delete"); + assert_eq!(shortcut_words("\u{f701}", 2), "option+cmd+down"); + assert_eq!(shortcut_words("\u{f704}", 8), "f1"); + } + #[test] fn bundle_is_found_from_the_executable() { assert_eq!(bundle_of("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"), Some("/Applications/Google Chrome.app")); @@ -454,6 +619,20 @@ mod tests { #[cfg(test)] mod live { + /// By hand: `ANYBOT_MENU_PID= cargo test live_app_menus -- --ignored --nocapture`. + #[cfg(target_os = "macos")] + #[test] + #[ignore] + fn live_app_menus() { + let pid: i32 = std::env::var("ANYBOT_MENU_PID").ok().and_then(|p| p.parse().ok()).unwrap_or(0); + let t = std::time::Instant::now(); + let m = super::app_menus(pid); + println!("{} entries in {} ms", m.len(), t.elapsed().as_millis()); + for e in m.iter().take(40) { + println!("{} {} [{}] {:?} {} {:?}", e.depth, e.label, e.shortcut, e.rect, if e.enabled { "" } else { "停用" }, e.alternate); + } + } + /// By hand: `cargo test live_person_chrome -- --ignored --nocapture`. Prints counts only. #[test] #[ignore] diff --git a/src/computer.rs b/src/computer.rs index 40ab4bc..e3a9b6e 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -686,6 +686,38 @@ fn menu_bar_controls() -> Vec { els } +/// The front app's menus as controls: its menu bar's titles, and the items of the menu that is +/// open now (with their shortcuts, so the next time a key does it). +#[cfg(target_os = "macos")] +fn app_menu_controls(pid: i32) -> Vec { + crate::axread::app_menus(pid) + .into_iter() + .map(|m| { + let mut flags = vec![]; + if !m.enabled { + flags.push("停用"); + } + let mut value = if m.shortcut.is_empty() { String::new() } else { format!("快捷鍵 {}", m.shortcut) }; + if let Some((label, keys)) = &m.alternate { + let keys = if keys.is_empty() { String::new() } else { format!("（快捷鍵 {keys}）") }; + value = format!("{value}{}按住 option 時這裡是「{label}」{keys}", if value.is_empty() { "" } else { "；" }); + } + crate::marks::Element { + kind: if m.depth == 0 { "選單".into() } else { "選單項目".into() }, + name: m.label, + value, + x: m.rect[0], + y: m.rect[1], + w: m.rect[2], + h: m.rect[3], + active: m.enabled, + flags, + window: if m.depth == 0 { "選單列".into() } else { "打開的選單".into() }, + } + }) + .collect() +} + #[cfg(target_os = "macos")] fn read_menu_bar() -> Vec { crate::axread::status_items(&app_processes()) @@ -950,6 +982,10 @@ fn look(after: Option, pointer: bool) -> Result<(String, Option) #[cfg(not(target_os = "linux"))] let a11y_note = ""; let (app, window, mut els) = found.unwrap_or_default(); + // The window's own controls couldn't be read: the menu bar's icons and menus still get + // numbers, and the bot must hear that the window itself is missing (it once saw only + // 「」 and a menu bar, and guessed names for eight minutes). + let window_unread = app.is_empty(); // Only what is on this screen gets a number (a link below the fold was numbered at the // picture's edge, and a click there landed on the edge). let (sw, sh) = mac::screen_points(); @@ -960,12 +996,18 @@ fn look(after: Option, pointer: bool) -> Result<(String, Option) // A number is a promise that a click there lands in this app: a control another window // covers (or one on another desktop) gets none. let hidden = cover_check(window, &mut els); - *LOOKED_AT.lock().unwrap_or_else(|e| e.into_inner()) = mac::front_app().map(|(_, pid)| pid); + let front_now = mac::front_app(); + *LOOKED_AT.lock().unwrap_or_else(|e| e.into_inner()) = front_now.as_ref().map(|(_, pid)| *pid); // The menu bar's status icons (Tailscale, Wi-Fi, a VPN…) belong to no window in front, so // they had no numbers and a bot cropped and measured screenshots to find them. #[cfg(target_os = "macos")] if controls_allowed() { els.extend(menu_bar_controls()); + // The front app's own menus (「檔案」「編輯」…) and whatever menu is open: a person + // reaches half of an app through them, and the bot had nothing to click there. + if let Some(pid) = *LOOKED_AT.lock().unwrap_or_else(|e| e.into_inner()) { + els.extend(app_menu_controls(pid)); + } } // Numbers kept from the last look where the same control is still there (marks::Numbering). static NUMBERS: Mutex> = Mutex::new(None); @@ -1005,6 +1047,10 @@ fn look(after: Option, pointer: bool) -> Result<(String, Option) text.push_str(&format!("\n看不出最前面是哪個 App{why}，所以沒有編號：只好照圖上的位置點 x、y。")); } else if lines.is_empty() { text.push_str(&format!("\n「{app}」沒有可以照編號點的東西（有些 App 不提供）：只好照圖上的位置點 x、y。")); + } else if window_unread { + let front = front_now.map(|(n, _)| n).unwrap_or_default(); + let why = if why.is_empty() { String::new() } else { format!("（{}）", why.lines().next().unwrap_or("")) }; + text.push_str(&format!("\n讀不到最前面「{front}」視窗裡的按鈕{why}：下面只有選單列。視窗裡的東西照圖上的位置點 x、y，或等一下再截一次圖。\n{}", lines.join("\n"))); } else { text.push_str(&format!("\n「{app}」上可以操作的東西（圖上同樣的編號，用 ref 點）：\n{}", lines.join("\n"))); } diff --git a/src/desktop.rs b/src/desktop.rs index 1b7ac3f..3180e1e 100644 --- a/src/desktop.rs +++ b/src/desktop.rs @@ -86,6 +86,9 @@ fn role_name(role: &str) -> &str { "checkbox" | "switch" | "toggle" => "勾選框", "radiobutton" => "選項", "popupbutton" | "menubutton" => "選單按鈕", + "disclosure" => "展開按鈕", + "colorwell" => "顏色", + "datefield" | "timefield" => "日期", "menuitem" => "選單項目", "tab" => "分頁", "row" | "cell" | "listitem" | "treeitem" | "outlinerow" => "項目", @@ -101,7 +104,7 @@ fn role_name(role: &str) -> &str { fn active(role: &str) -> bool { matches!( role, - "button" | "link" | "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield" | "checkbox" | "switch" | "toggle" | "radiobutton" | "popupbutton" | "menubutton" | "menuitem" | "tab" | "row" | "cell" | "listitem" | "treeitem" | "outlinerow" | "slider" | "incrementor" + "button" | "link" | "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield" | "checkbox" | "switch" | "toggle" | "radiobutton" | "popupbutton" | "menubutton" | "menuitem" | "tab" | "row" | "cell" | "listitem" | "treeitem" | "outlinerow" | "slider" | "incrementor" | "disclosure" | "colorwell" | "datefield" | "timefield" ) } @@ -142,6 +145,11 @@ pub fn elements_of(tree: &Value, app: &str) -> Vec { } let role = n["role"].as_str().unwrap_or(""); let mut name = n["name"].as_str().unwrap_or("").trim().to_string(); + // An icon-only control often has its words only as a description (the save panel's + // triangle: 「顯示較多選項」). + if name.is_empty() && n["ref_id"].is_string() && active(role) { + name = n["description"].as_str().unwrap_or("").trim().to_string(); + } // A nameless row, cell or list item goes by the words inside it (an icon-only button // keeps its number without a name: it can still be clicked). let row_like = matches!(role, "row" | "cell" | "treeitem" | "outlinerow" | "listitem"); @@ -223,6 +231,17 @@ mod tests { assert_eq!(kept.len(), 1, "an empty spacer row gets no number: {kept:?}"); } + #[test] + fn a_save_panels_expand_triangle_gets_a_number() { + // TextEdit's save sheet: the bot zoomed and clicked beside a triangle it had no number for. + let sheet = serde_json::json!({"role": "sheet", "children": [ + {"role": "disclosure", "description": "顯示較多選項", "ref_id": "d", "bounds": {"x": 340.0, "y": 180.0, "width": 21.0, "height": 21.0}} + ]}); + let els = elements_of(&sheet, "文字編輯"); + assert_eq!(els.len(), 1); + assert!(els[0].active && els[0].kind == "展開按鈕" && els[0].name == "顯示較多選項", "{:?} {:?}", els[0].kind, els[0].name); + } + #[test] fn controls_and_text_come_with_their_place() { let tree = serde_json::json!({"role": "window", "name": "計算機", "bounds": {"x": 0.0, "y": 0.0, "width": 300.0, "height": 400.0}, "children": [ diff --git a/src/facts.rs b/src/facts.rs index 8f87651..d405ab7 100644 --- a/src/facts.rs +++ b/src/facts.rs @@ -130,6 +130,14 @@ pub fn describe(name: &str, f: &Facts, own_box: bool, mac_sandbox: bool) -> Stri if mac_sandbox { parts.push("指令在 AnyBot 的沙盒裡跑：ps、top、App Store 版 App 自己的指令列、單獨一條 osascript -e 會在沙盒外跑，其他碰不到祕密資料夾".into()); } + // What anyone used to the system knows about its standard windows and dialogs: an app the bot + // never saw still uses them (it lost minutes to TextEdit's save panel, then changed the format + // nobody asked about). + if f.kernel == "Darwin" { + parts.push("Mac 的操作慣例：存檔、開檔對話框和 Finder 裡按 cmd+shift+G 可以直接打資料夾的完整路徑再按 Return；存檔對話框在「儲存為」打檔名後按 Return 就存好了；cmd+W 關掉目前的視窗、cmd+Q 結束 App（有沒存的會先問）；檔案格式、副檔名這類選項使用者沒說就照預設，不要另外改；指令是用 /bin/sh 跑的，echo -n 會把「-n」也寫出去，要不換行就用 printf；讀到的結果和預期不同（多了字、少了字）就照實說，不要說成預期的樣子".into()); + } else if f.kernel == "Linux" && (!f.wm.is_empty() || !f.desktop.is_empty()) { + parts.push("Linux 桌面的操作慣例：GTK 的開檔、存檔對話框按 ctrl+L 可以直接打完整路徑再按 Return；ctrl+S 存檔、ctrl+W 關視窗、ctrl+Q 結束 App；檔案格式這類選項使用者沒說就照預設".into()); + } if !f.apps.is_empty() { let more = if f.apps.len() > MAX_APPS { "…" } else { "" }; parts.push(format!("裝了的 App：{}{more}", f.apps.iter().take(MAX_APPS).cloned().collect::>().join("、"))); @@ -245,6 +253,8 @@ mod tests { fn a_mac_says_what_its_sandbox_lets_through() { let line = describe("王小明的Mac mini", &parse(MAC), false, true); assert!(line.contains("macOS 26.1") && line.contains("沙盒") && line.contains("osascript") && line.contains("裝東西用 brew") && line.contains("Tailscale"), "{line}"); + assert!(line.contains("cmd+shift+G") && line.contains("照預設") && line.contains("printf"), "{line}"); + assert!(describe("Bot 的電腦", &parse(BOX), true, false).contains("ctrl+L")); assert!(!line.contains("systemd"), "{line}"); // Offline and no sudo, said plainly so the bot doesn't try. let off = describe("x", &parse("kernel=Linux\nnet=no\nsudo=password\n"), false, false); diff --git a/src/main.rs b/src/main.rs index 40f6703..4ada87b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -229,6 +229,14 @@ impl eframe::App for Busy { } } +const USAGE: &str = "用法： + anybot 打開 AnyBot 視窗 + anybot --serve 不開視窗，讓 Bot 繼續工作、接手機 + anybot --ask 問題… 問第一個 Bot 一句，印出回答（測試用） + anybot setup … 在沒有螢幕的機器上設定 + anybot --screenshot 檔名 把視窗畫成一張圖 +資料夾：ANYBOT_DATA_DIR（沒設就用這個使用者的 AnyBot 資料夾）"; + fn main() -> eframe::Result<()> { let args: Vec = std::env::args().collect(); let screenshot = args.iter().position(|a| a == "--screenshot").and_then(|i| args.get(i + 1)).map(PathBuf::from); @@ -304,6 +312,16 @@ fn main() -> eframe::Result<()> { let _folder = take_folder_or_exit(); app::App::serve(&ctx); } + // `--help` or a flag this build doesn't know: say what there is and stop. (It used to open the + // window on the real data folder, next to the AnyBot already using it.) Single-dash arguments + // are macOS's own (-psn_…, -NSDocumentRevisionsDebugMode) and are left alone. + if let Some(bad) = args.iter().skip(1).find(|a| a.as_str() == "-h" || (a.starts_with("--") && !matches!(a.as_str(), "--screenshot" | "--serve"))) { + if bad != "--help" && bad != "-h" { + eprintln!("AnyBot：不認得「{bad}」。"); + } + eprintln!("{USAGE}"); + std::process::exit(if bad == "--help" || bad == "-h" { 0 } else { 2 }); + } // (A screenshot run is a tool next to the AnyBot that owns the folder; it doesn't take it.) let _folder = if screenshot.is_some() { None