From 1202de93ad04ee56a5a331e66cf585f506e23002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=80=A7=E9=A9=8A?= Date: Mon, 9 Nov 2026 09:01:03 +0800 Subject: [PATCH] Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS --- FEATURES.md | 28 ++- box/Dockerfile | 6 +- docs/architecture.md | 7 +- src/agent.rs | 576 ++++++++++++++++++++++++++++++++++++++++++- src/app.rs | 52 +++- src/appctl.rs | 3 +- src/axread.rs | 90 +++++++ src/browser.rs | 93 ++++++- src/cloudbox.rs | 155 ++++++++++-- src/computer.rs | 215 +++++++++++++++- src/desktop.rs | 3 +- src/facts.rs | 261 ++++++++++++++++++++ src/main.rs | 1 + src/marks.rs | 167 ++++++++++++- src/memory.rs | 124 +++++++++- src/model.rs | 6 + src/providers.rs | 14 ++ src/tools.rs | 371 +++++++++++++++++++++++++++- 18 files changed, 2099 insertions(+), 73 deletions(-) create mode 100644 src/facts.rs diff --git a/FEATURES.md b/FEATURES.md index 58b61bb..4138edd 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -587,6 +587,15 @@ AnyBot 是一個桌面聊天 App，介面和運作方式照 Grok Bot 重做。 - 每 25 步，模型會在最新的工具結果後面看到一句提醒：方向對就繼續；最近幾步都沒進展就停下來，用一兩句話告訴你卡在哪、要你做什麼（聊天記錄裡看不到這句）。 - 鬼打牆另外有攔截：同一個畫面或網頁動作第 3 次會被警告，第 4 次起不執行。 - 系統提示要模型每一步做多一點：互不相依的工具同一步一起呼叫、查東西的指令合成一條、等待寫在同一條指令裡。 +- **想的跟做的分開**（Bot 設定 › AI 供應商與模型 › 執行步驟用的模型）：Bot 的模型負責第一步（想好計畫、開始做）、出問題後的下一步（有步驟失敗、被拒絕、被判定重複、點了之後畫面完全沒變，或快速模型自己從指令改去點畫面、從畫面改去網頁這類換路）、每連續 4 個快速步驟之後看一次進度，以及回合結尾：快速模型在改過東西之後說「做好了」，那句話先擱著不顯示，由 Bot 的模型看過結果再回答。其他照計畫做的步驟交給同一家的快速模型（不長時間思考）。 + - 「自動」：xAI 用 grok-4.20-0309-non-reasoning、Anthropic 用 claude-haiku-5-5、Gemini 用 gemini-2.5-flash、OpenAI 用 gpt-5-mini；Bot 的模型本身就是快速型（名字有 non-reasoning、fast、mini、haiku、flash）時不分開。其他供應商不分開。也可以選「不分開」或指定一個模型。 + - 快速模型叫不動（帳號沒有這個模型、出錯）時，這一回合剩下的步驟全部改用 Bot 的模型。 + - 快速模型的步驟不即時顯示文字（它的答案可能被擱置），工具步驟照常顯示；每一則回覆標的是實際用的模型。 + - 依據：量測時九成以上的時間是會思考的模型在每一步之前思考（grok-4.7 每步約 10 秒，快速模型約 1 秒）；快速模型單獨做時，簡單的工作全對，難的工作會太早收工，所以由會思考的模型規劃和收尾。 +- **電腦現況**：每個回合開始時，系統提示會附上每台電腦的現況，讓模型直接用那台最直接的做法，不用花步驟試探：系統和版本、桌面、有沒有 systemd（沒有時會說 systemctl、timedatectl 不能用，改設定直接改檔案）、能不能 sudo、有沒有網路、用什麼裝套件、有哪些常用指令和 App，以及固定的 TZ 環境變數（版本 7 以前的 box 映像檔設了 TZ=UTC，會蓋過 /etc/localtime，改了系統時區 date 還是 UTC；版本 8 拿掉了，舊的 box 更新重建前現況會提醒）。 + - 一台電腦讀一次（一段唯讀的小腳本，約 0.2 秒），30 分鐘內不再讀；一個回合最多等 6 秒，來不及的下一回合用。 + - Bot 的電腦一律讀；使用者的電腦只有 Bot 已經可以在上面做事時才讀；AnyBot 所在的這台由 AnyBot 自己讀，不算 Bot 執行的指令。 +- **做事的方式**（系統提示）：先選路（專用工具、指令、最後才畫面）；動手前想好要做哪幾步、做完怎樣算成功；選定一條路做到底；同一個做法失敗兩次就換；做完只用一種最直接的方法確認一次；做不到就照實說，不用替代品假裝做到（例如只改環境變數讓時鐘看起來對）；會切斷連線的事先問。 - **每一步開始前會依序做**： 1. 從第 2 步起，檢查要不要整理摘要（門檻 75%）。 2. 把你中途打的話加進去。 @@ -856,12 +865,18 @@ AnyBot 是一個桌面聊天 App，介面和運作方式照 Grok Bot 重做。 ### 沙盒（macOS 有提供時） -- 所有指令都在沙盒裡跑。 +- 所有指令都在沙盒裡跑，下面三種例外（只在你的電腦上；Bot 自己的電腦一律在沙盒裡）。 - 指令碰不到這些地方：AnyBot 的資料、.ssh、.gnupg、Library/Keychains、.aws、.config/gcloud。 +- 例外，在沙盒外跑： + - App Store 版 App 自己的指令列（`/Applications/X.app/Contents/MacOS/…`，單獨一條指令、程式帶 Apple 的 App 沙盒）：在 AnyBot 的沙盒裡兩層沙盒會衝突、程式一開就當（例如 Tailscale 的指令列），它自己的 App 沙盒一樣讀不到上面那些地方。 + - `ps`、`top`，單獨或只接 grep、head、awk 這類不讀檔的過濾：它們要系統權限（setuid），沙盒裡一律「Operation not permitted」。 + - AppleScript：單獨一條 `osascript -e '…'`（不能跑腳本檔、JavaScript、heredoc），而且不含 do shell script、run script、終端機、keystroke、讀檔、ObjC、祕密資料夾的名字。在沙盒裡 AppleScript 指揮別的 App 一律被拒（-10004）。第一次會先問你，直接操作模式也問，選「一律允許」之後不再問。 +- 被沙盒擋下時（setuid 程式、-10004、App 的指令列當掉），結果後面會附一句說明是沙盒擋的、改用什麼，Bot 不會當成電腦壞了一直重試。 ### 在「Bot 的電腦」上 - 指令只檢查上面那 10 種一律拒絕的，其他都直接做。沙盒會把它關在自己的資料夾裡，看不到也寫不到家目錄的其他地方。 +- Docker 的 Bot 電腦（box）上 sudo 是 Bot 自己的（box 給它免密碼 sudo，可以裝套件、改系統設定）：只檢查 sudo 後面接的指令，`sudo rm -rf /`、`sudo mkfs`、`sudo su` 一樣拒絕；含 sudo 的指令用容器裡的 root 執行（gVisor 裡 setuid 不起作用）。以前 sudo 在 box 上也一律拒絕，和「可以 sudo 安裝套件」的說明互相矛盾，Bot 會一直繞路。 - 自己資料夾裡的檔案可以自由讀和改。 - 資料夾外：拒絕，「{Bot} 看不到這台電腦上的其他檔案（路徑）。要處理你的檔案，請用「你的電腦」。」 @@ -908,6 +923,7 @@ AnyBot 是一個桌面聊天 App，介面和運作方式照 Grok Bot 重做。 - 「在瀏覽器打開」（browser_open）：參數是網址和設定檔。等頁面載入完成（最多 12 秒，看載入狀態、網址和文字停下來，不是固定秒數），直接回傳和 browser_read 一樣的內容（最多 12,000 字，加上可以點的編號），所以打開後不用再讀一次。 - 點擊、輸入、按鍵、選擇、捲動、上一頁之後：等頁面停下來就回來（頁面有變且穩定，或 0.5–0.9 秒內沒有開始變），最多 4 秒；以前是固定 1–2.5 秒。 - 「讀取瀏覽器分頁」（browser_read）：最多讀多少字（預設 20,000，最多 40,000），加上設定檔。 +- 編號也包含 web component 裡的東西（打開的 shadow DOM，一層層往下找，例如 Chromium 的設定頁、YouTube、很多用元件框架做的網站），排列順序照畫面上的位置：元件本身、它 shadow DOM 裡顯示的東西、再來是它自己的子元素。照編號點、照文字找、畫編號框都看得到這些。 - 「瀏覽器設定檔」（browser_profiles）：列出、新增、切換。 ### 用哪個瀏覽器 @@ -1142,6 +1158,13 @@ computer_use_screenshot、click、mouse_move、mouse_button（按住／放開） - JPEG 品質依序試 85、70、55，取第一個小於 1 MB（1,048,576 位元組）的。這些都取自原版。 - 只把最近 3 張截圖給模型看（估計值），較早的改成「（較早的截圖已省略）」。截圖不會存到硬碟。 - 成功：「你的螢幕截圖（寬×高）。之後的座標請用這張圖上的像素位置。」 +- Mac 選單列右上角的圖示（Wi‑Fi、控制中心、時鐘、Tailscale、LINE…）也有編號，叫「選單列圖示」：從每個 App 的 AXExtrasMenuBar 讀，系統自己的圖示用它裡面那一層的字（「Wi‑Fi，已連接，3格」），同時問 8 個 App，一次約 0.3 秒；15 秒內用上次讀的，過期就在背景重讀，不讓截圖等。 + +### 「放大看」（computer_use_zoom） + +- 參數：最新截圖上的 x、y、width、height。從螢幕原本的清晰度切那一塊、放大（長邊最多 1,024，最多放大 4 倍），給選單列圖示、小字、小按鈕用。 +- 結果說明放大圖上的點怎麼換回一般截圖的座標；有編號的東西還是照編號點。 +- 你的電腦、Bot 的電腦、帳號裡其他電腦都可以用；算「看」，和截圖用同一個允許。 ### 「點一下」（computer_use_click） @@ -1275,6 +1298,9 @@ computer_use_screenshot、click、mouse_move、mouse_button（按住／放開） - **記憶**：跟 Bot 說「記住我喜歡靠窗的座位」，它會用 remember 記下來；「忘記…」用 forget。每個 Bot 最多 200 條、每條 500 字（估計值），每次聊天都帶著。Bot 設定裡的「記得的事」可以一條條刪。 - **技能**：教 Bot 一套做法後請它「存成技能」（save_skill），所有 Bot 共用；系統指示只列名稱和一句說明，Bot 要用時先用 use_skill 讀完整做法。設定 →「連接器」頁的「技能」卡可以刪除。 - **自己進化**：一件事做了 5 步以上（操作電腦、網頁、指令，不算記憶和設定），回答完後在背景回頭看一次：這是在網站或程式上動手操作的事（填表、發文、回留言、上傳下載、改設定），就把這次真的成功的最短做法存成技能（用畫面上的名字，不寫編號和座標；繞路和失敗寫成「注意」；使用者說過的偏好寫進去）；已經有同一件事的技能就合併成更好的一版，照著做而且很順就不動。回覆下面會多一行「（學會了新做法「…」）」或「（把做法「…」更新了）」。系統指示要 Bot 開始做事前先看技能清單、有符合的先 use_skill。實測（httpbin 訂 pizza，真的 Grok）：第一次 11 步學到技能；第二次先讀技能、10 步 24 秒一次做完，回頭看後判斷不用改。 +- **重播學會的做法**（replay_skill）：存技能時，模型會挑出這次真的成功、必要、會改變東西的步驟（不含截圖、讀取這類只是看的步驟、不含繞路和失敗），連同每一步點的是哪個東西（「勾選框「自動更新」」，不是編號）一起存下來；內容每次不同的工作（發文的文字、回覆的留言）不存步驟。技能清單裡標「可重播，N 步」。 + - 要做的事和技能一模一樣時，Bot 叫 replay_skill，AnyBot 就一步一步照做，中間不問模型：畫面上的步驟在最近的畫面清單裡找同一種、同名的東西換成現在的編號（值和勾選狀態不算），網頁上的步驟改用字找；找不到時先自動截一次圖再找，還是找不到就停下來，告訴 Bot 停在第幾步、找不到什麼，由它接手。 + - 每一步照樣經過核准（有風險的照樣問你）；有一步失敗就停，後面不做；做完由 Bot 自己的模型看結果再回答。重播的步驟在聊天裡標「（重播）」。 - **每件事用自己的分頁**：幾個 Bot 共用一個瀏覽器（Bot 的電腦裡的 Chromium）時，每件事只在自己打開或停在的分頁上做（點連結開出的新視窗也算它的），別的事的分頁不會被讀到、點到、列出或關掉；做完只收自己的分頁、留最後那一頁。 - **範本**：側邊欄「＋」按右鍵 →「從範本建立」：研究助理、寫作夥伴、電腦助手、客服助理、開發助手，各有角色說明、指示、頭像和顏色。Bot 也能用 create_bot 的 template 參數建立。 diff --git a/box/Dockerfile b/box/Dockerfile index 70c91a0..33a640d 100644 --- a/box/Dockerfile +++ b/box/Dockerfile @@ -3,7 +3,9 @@ # the screen and the VNC server; AT-SPI is on, so every app's controls (web pages included) can # be read with their places and numbered on the bot's picture (box/a11y, after XBot's box). FROM debian:bookworm-slim -ENV DEBIAN_FRONTEND=noninteractive LANG=C.UTF-8 TZ=UTC +# No TZ here: a TZ variable overrides /etc/localtime for every program, so setting the timezone +# the normal way (/etc/localtime, /etc/timezone) seemed to do nothing. /etc/localtime is UTC. +ENV DEBIAN_FRONTEND=noninteractive LANG=C.UTF-8 RUN apt-get update && apt-get install -y --no-install-recommends \ tigervnc-standalone-server tigervnc-tools xdotool x11-utils x11-xserver-utils imagemagick fluxbox xterm \ at-spi2-core libatk-adaptor python3-gi gir1.2-atspi-2.0 libglib2.0-bin gsettings-desktop-schemas dconf-gsettings-backend \ @@ -35,6 +37,6 @@ RUN chmod +x /usr/local/bin/anybot-box-start /usr/local/bin/anybot-a11y /usr/loc # Accessibility on for every app, including those started over `docker exec`. ENV DISPLAY=:1 SCREEN_SIZE=1280x800x24 GTK_MODULES=atk-bridge GTK_A11Y=atspi GNOME_ACCESSIBILITY=1 NO_AT_BRIDGE=0 EXPOSE 6080 -LABEL anybot.box.version="7" +LABEL anybot.box.version="8" ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/usr/local/bin/anybot-box-start"] diff --git a/docs/architecture.md b/docs/architecture.md index bb16680..8be9262 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -37,7 +37,8 @@ AnyBot 是一個 Rust 程式（`anybot`），用 egui/eframe 畫介面。同一 | 檔案 | 做什麼 | |---|---| | `model.rs` | 資料：State、Bot、Message、設定；讀寫 `state.json` | -| `agent.rs` | 一個回合：組系統指示、送模型、收串流、跑工具、核准關卡、長對話摘要 | +| `agent.rs` | 一個回合：組系統指示、送模型、收串流、跑工具、核准關卡、長對話摘要；每一步用哪個模型（`StepPlan`：Bot 的模型規劃、出問題時和收尾，快速模型照計畫執行） | +| `facts.rs` | 電腦現況：每台電腦一段唯讀小腳本讀出系統、桌面、systemd、sudo、網路、套件管理、常用指令和 App，30 分鐘快取，回合開始時放進系統指示 | | `providers.rs` | 各家模型的 API（OpenAI 相容、Anthropic、OpenCode Go…）、串流、重試、看不到圖時的看圖小幫手 | | `tools.rs` | 檔案、指令、背景指令；指令在 macOS 沙盒裡跑（見「安全」） | | `policy.rs` | 哪些動作直接做、先問、拒絕 | @@ -72,8 +73,8 @@ AnyBot 是一個 Rust 程式（`anybot`），用 egui/eframe 畫介面。同一 ## 安全 - **核准關卡**：工具動作照規則直接做、先問或拒絕（`policy.rs`，FEATURES.md 第 16、19 節）。「直接操作模式」可以跳過詢問。 -- **指令的沙盒（macOS）**：Bot 執行的指令在 `sandbox-exec` 裡跑，讀不到也寫不到 AnyBot 的資料資料夾、`.gnupg`、鑰匙圈、`.aws` 等；只有 `attachments/` 和 `.ssh` 可以讀（都不能寫，`.ssh` 可讀讓 git 和 ssh 連得動）。只有沙盒管得到的才是祕密防線，核准層不再逐字掃路徑。環境變數裡的模型金鑰會先拿掉，PATH 會補齊 Homebrew 等位置（從 Finder 或 launchd 啟動也要找得到指令）。 -- **Bot 的電腦**：放在容器裡，和你的電腦分開。標準（runc）容器和主機共用 Linux 核心，Chromium 在裡面關掉自己的沙盒，容器就是唯一那道牆。Docker 主機裝了 gVisor 時，設定 › 電腦 › 容器隔離可改「強隔離（gVisor）」：容器有自己的虛擬核心，Chromium 的沙盒重新開啟，牆變成兩道（虛擬核心＋瀏覽器沙盒）。改完要按電腦的「更新」重建才生效。在 macOS 上沒有能裝 gVisor 的 Docker，用 `deploy/mac/gvisor-vm.sh` 建一台自帶 Docker＋gVisor 的 Lima 虛擬機，再把 Docker 主機指過去；那台虛擬機沒開時（例如 Mac 重開機後）AnyBot 會自己開。 +- **指令的沙盒（macOS）**：Bot 執行的指令在 `sandbox-exec` 裡跑，讀不到也寫不到 AnyBot 的資料資料夾、`.gnupg`、鑰匙圈、`.aws` 等；只有 `attachments/` 和 `.ssh` 可以讀（都不能寫，`.ssh` 可讀讓 git 和 ssh 連得動）。只有沙盒管得到的才是祕密防線，核准層不再逐字掃路徑。例外（`tools::runs_outside_sandbox`，只在你的電腦上）：App Store 版 App 的指令列（自帶 App 沙盒，在這裡會兩層衝突而當掉）、`ps`／`top`（setuid，沙盒裡不能跑）、單獨一條且不含危險寫法的 `osascript -e`（沙盒裡指揮別的 App 一律 -10004；第一次先問，直接操作模式也問，`osascript:outside` 授權後不再問）。AppleScript 在沙盒外沒有東西擋它讀檔，所以靠寫法過濾加第一次詢問，不是密不透風。環境變數裡的模型金鑰會先拿掉，PATH 會補齊 Homebrew 等位置（從 Finder 或 launchd 啟動也要找得到指令）。 +- **Bot 的電腦**：放在容器裡，和你的電腦分開。標準（runc）容器和主機共用 Linux 核心，Chromium 在裡面關掉自己的沙盒，容器就是唯一那道牆。Docker 主機裝了 gVisor 時，設定 › 電腦 › 容器隔離可改「強隔離（gVisor）」：容器有自己的虛擬核心，Chromium 的沙盒重新開啟，牆變成兩道（虛擬核心＋瀏覽器沙盒）。改完要按電腦的「更新」重建才生效。gVisor 裡 setuid 不起作用，bot 使用者用 sudo 會失敗（「effective uid is not 0」），所以含 sudo 的指令改用容器裡的 root 執行（`cloudbox::exec_wrapper`），跑完把它在 bot 家目錄留下的檔案還給 bot。在 macOS 上沒有能裝 gVisor 的 Docker，用 `deploy/mac/gvisor-vm.sh` 建一台自帶 Docker＋gVisor 的 Lima 虛擬機，再把 Docker 主機指過去；那台虛擬機沒開時（例如 Mac 重開機後）AnyBot 會自己開。 - **保險庫**：密碼、卡號、CVC 不會交給模型，由 AnyBot 直接填進網頁。 - **多台電腦**：金鑰只留在大腦（下面說明）；手腳只接受帳號裡的 AnyBot，用配對碼加入的只接受那一個大腦。 diff --git a/src/agent.rs b/src/agent.rs index 63935aa..a72323f 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -37,6 +37,9 @@ const SUMMARY_SNIPPET_CHARS: usize = 2_000; pub struct Job { pub req: ChatRequest, + /// The quick model that carries out planned steps (same provider), or None: every step uses + /// `req.model`. See `step_plan`. + pub step_model: Option, /// Computers this bot may work on (empty when computer tools are off). pub computers: Vec, pub default_computer: String, @@ -128,7 +131,8 @@ fn learn(job: &Job, history: &[Message], cancel: &AtomicBool, send: &dyn Fn(Stre } }; match (crate::memory::learned_skill(&out.text), turn.last()) { - (Some(k), Some(reply)) => { + (Some(mut k), Some(reply)) => { + k.steps = crate::memory::replay_steps(turn, &crate::memory::learned_replay(&out.text)); eprintln!("AnyBot：學到做法「{}」", k.name); send(StreamEvent::Learned(k, reply.at)); } @@ -630,6 +634,156 @@ fn wait_answer(rx: std::sync::mpsc::Receiver, cancel: &AtomicBoo } } +/// The tool name for doing a skill's steps again. +pub const REPLAY_TOOL: &str = "replay_skill"; + +/// A control's kind and name from its line in a list, without its value or state, which may +/// differ the next time (「勾選框「自動更新」：1（已勾）」 → 「勾選框「自動更新」」). +fn control_key(desc: &str) -> &str { + match desc.find('」') { + Some(i) => &desc[..i + '」'.len_utf8()], + None => desc.split(['：', '（']).next().unwrap_or(desc).trim(), + } +} + +/// A replay step's arguments for now: each control it acted on found again by what it is, in the +/// newest lists (screen steps get today's number; browser steps find it by its words). Err: the +/// control that isn't there. +fn resolve_replay(step: &crate::memory::ReplayStep, history: &[Message]) -> Result { + let mut v: serde_json::Value = serde_json::from_str(&step.args).unwrap_or_default(); + let find = |desc: &str| -> Option { + let want = control_key(desc); + history.iter().rev().filter(|m| m.role == Role::Tool).take(6).find_map(|m| { + m.text.lines().find_map(|l| { + let (num, rest) = l.trim_start().strip_prefix('[')?.split_once("] ")?; + (control_key(rest.trim()) == want && num.chars().all(|c| c.is_ascii_digit())).then(|| num.to_string()) + }) + }) + }; + let words = |desc: &str| desc.split_once('「').and_then(|(_, r)| r.rsplit_once('」')).map(|(t, _)| t.to_string()).unwrap_or_else(|| desc.to_string()); + if step.tool == "computer_use_batch" { + if let Some(actions) = v["actions"].as_array_mut() { + for (i, a) in actions.iter_mut().enumerate() { + if let Some(desc) = step.targets.get(i).filter(|d| !d.is_empty()) { + a["ref"] = serde_json::Value::String(find(desc).ok_or_else(|| desc.clone())?); + } + } + } + } else if let Some(desc) = step.targets.first().filter(|d| !d.is_empty()) { + if step.tool.starts_with("browser_") { + if let Some(o) = v.as_object_mut() { + o.remove("ref"); + } + v["text"] = serde_json::Value::String(words(desc)); + } else { + v["ref"] = serde_json::Value::String(find(desc).ok_or_else(|| desc.clone())?); + } + } + Ok(v.to_string()) +} + +/// A skill being done again: what is left, which step this is, and how many there are. +struct Replay { + name: String, + left: std::collections::VecDeque, + total: usize, + /// The step that already had a fresh look taken for it (one look per step, then hand back). + looked: Option, +} + +/// Quick steps in a row before the bot's own model looks at the work again (estimated). +pub const QUICK_RUN_MAX: usize = 4; + +/// Steps that only look (nothing on any computer changes): a turn of only these needs no check +/// at the end. +fn only_looks(name: &str) -> bool { + matches!( + name, + "computer_use_screenshot" | "computer_use_zoom" | "computer_use_show" | "computer_use_check_permissions" | "browser_read" | "browser_screenshot" | "browser_elements" | "browser_find" | "browser_table" | "browser_tabs" | "browser_wait" | "web_fetch" | "read_file" | "list_dir" | "search" | "read_terminal" | "list_computers" | "list_bots" | "list_routines" | "read_page" | "use_skill" | "react" | "show_card" + ) +} + +/// 想的跟做的分開: which model takes a step. The bot's own (thinking) model takes the first step +/// (it plans and starts), the step after anything went wrong, one in every QUICK_RUN_MAX, and the +/// end of a turn that changed something (it checks the work and answers). A quick model carries +/// out the planned steps in between: most of a turn's time was the thinking model thinking +/// before each click. +pub struct StepPlan { + quick: Option, + main_next: bool, + quick_run: usize, + /// The way the last acting step went (commands, the screen, a web page). + last_route: Option<&'static str>, + /// Something on a computer may have changed this turn (not only looking). + pub changed: bool, +} + +impl StepPlan { + pub fn new(quick: Option) -> Self { + StepPlan { quick, main_next: true, quick_run: 0, last_route: None, changed: false } + } + + /// The quick model for this step, or None for the bot's own. + pub fn pick(&mut self) -> Option { + if self.quick.is_none() || self.main_next || self.quick_run >= QUICK_RUN_MAX { + self.quick_run = 0; + self.main_next = false; + return None; + } + self.quick_run += 1; + self.quick.clone() + } + + /// After a step: trouble (a step failed, was refused, or repeated) sends the next one back + /// to the bot's own model. + pub fn after(&mut self, trouble: bool) { + if trouble { + self.main_next = true; + } + } + + /// The quick model answered without acting: when the turn changed something, the bot's own + /// model checks and answers instead. Returns whether to set that answer aside. + pub fn recheck_answer(&mut self, was_quick: bool) -> bool { + let recheck = was_quick && self.changed; + if recheck { + self.main_next = true; + } + recheck + } + + /// The quick model couldn't be used (not on this account, an error): the rest of the turn + /// goes without it. + pub fn quick_failed(&mut self) { + self.quick = None; + } + + pub fn quick_is_off(&self) -> bool { + self.quick.is_none() + } + + /// The way a step's acting calls went. The quick model switching ways on its own (commands + /// to clicking the screen, say) is the bot's own model's call: true sends the next step there. + pub fn changed_route(&mut self, calls: &[crate::model::ToolCall], was_quick: bool) -> bool { + let route = calls.iter().filter(|c| !only_looks(&c.name)).map(|c| route_of(&c.name)).next(); + let Some(route) = route else { return false }; + let switched = was_quick && self.last_route.is_some_and(|r| r != route); + self.last_route = Some(route); + switched + } +} + +/// Which way a tool goes: commands and files, the screen, or a web page. +fn route_of(name: &str) -> &'static str { + if computer::is_computer_tool(name) || crate::macapps::is_mac_tool(name) { + "screen" + } else if web::is_web_tool(name) { + "web" + } else { + "command" + } +} + pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc, ctx: eframe::egui::Context) { std::thread::spawn(move || { let mut job = job; @@ -645,8 +799,48 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< let mut tried: Vec = Vec::new(); let mut local_on = job.local_execution; let mut browsers = UsedBrowsers::new(&job.via_name, bot_id); + // 電腦現況: what someone at each computer already knows (system, sudo, network, apps…). + let may_use_persons = job.tools_enabled || job.direct.load(Ordering::Relaxed); + job.req.system.push_str(&crate::facts::for_turn(&job.computers, may_use_persons, &job.via_name)); maybe_compact(&job, &mut history, COMPACT_THRESHOLD_PERCENT, &cancel, &send); + let mut plan = StepPlan::new(job.step_model.clone()); + // A quick step's answer was set aside: its (empty) message in the chat is reused. + let mut reuse_slot = false; + // 重播: a skill's steps done again as they were, with no model in between. + let mut replay: Option = None; for step in 0..MAX_STEPS { + // The next replay step, its controls found again; a control that isn't there gets one + // fresh look, then the bot takes over (told where and why). + let mut scripted: Option = None; + if let Some(r) = replay.as_mut() { + if let Some(next) = r.left.front().cloned() { + let k = r.total - r.left.len() + 1; + match resolve_replay(&next, &history) { + Ok(args) => { + r.left.pop_front(); + scripted = Some(crate::model::ToolCall { id: format!("replay-{k}-{}", crate::model::now_ms()), name: next.tool.clone(), args, status: ToolStatus::Running }); + } + Err(_) if r.looked != Some(k) && !next.tool.starts_with("browser_") => { + r.looked = Some(k); + let computer = tools::args_of(&next.args)["computer"].as_str().map(|c| serde_json::json!({"computer": c})).unwrap_or(serde_json::json!({})); + scripted = Some(crate::model::ToolCall { id: format!("replay-look-{k}-{}", crate::model::now_ms()), name: "computer_use_screenshot".into(), args: computer.to_string(), status: ToolStatus::Running }); + } + Err(missing) => { + if let Some(m) = history.last_mut().filter(|m| m.role == Role::Tool) { + m.text.push_str(&format!("\n\n（重播「{}」停在第 {k} 步：畫面上找不到「{missing}」。前面 {} 步已經照做了；接下來由你看畫面決定怎麼做完。）", r.name, k - 1)); + } + replay = None; + plan.after(true); + } + } + } + } + let quick = if scripted.is_some() { None } else { plan.pick() }; + let via = match (&quick, &scripted) { + (_, Some(_)) => format!("{}（重播）", job.via), + (Some(q), None) => job.via.replacen(&job.req.model, q, 1), + (None, None) => job.via.clone(), + }; if step > 0 { maybe_compact(&job, &mut history, COMPACT_THRESHOLD_PERCENT + COMPACT_TOLERANCE_PERCENT, &cancel, &send); drain_steer(&job, &mut history, &send); @@ -656,10 +850,33 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< m.text.push_str(&format!("\n\n{note}")); } } - send(StreamEvent::NextStep(job.via.clone())); + if !reuse_slot { + send(StreamEvent::NextStep(via.clone())); + } } - let req = ChatRequest { history: history.clone(), ..job.req.clone() }; - let out = match providers::run_step(&req, &cancel, &send) { + reuse_slot = false; + let main_req = ChatRequest { history: history.clone(), ..job.req.clone() }; + // Quick steps don't stream their words: an answer may be set aside for the check. + let quiet = |ev: StreamEvent| { + if !matches!(ev, StreamEvent::Text(_) | StreamEvent::Thinking(_)) { + send(ev) + } + }; + let tried_quick = quick.clone(); + let out = match (&quick, scripted.clone()) { + (_, Some(c)) => Ok(providers::StepOutput { text: String::new(), thinking: String::new(), tool_calls: vec![c], blocks: vec![] }), + (q, None) => match q { + Some(q) => providers::run_step(&ChatRequest { model: q.clone(), ..main_req.clone() }, &cancel, &quiet).or_else(|e| { + eprintln!("AnyBot：快速模型 {q} 不能用（{e}），這一回合改用 {}", job.req.model); + plan.quick_failed(); + providers::run_step(&main_req, &cancel, &send) + }), + None => providers::run_step(&main_req, &cancel, &send), + }, + }; + let quick = if plan.quick_is_off() { None } else { tried_quick }; + let via = if quick.is_none() && scripted.is_none() { job.via.clone() } else { via }; + let out = match out { Ok(o) => o, Err(e) => { send(StreamEvent::Error(e)); @@ -683,14 +900,22 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< role: Role::Assistant, text: crate::i18n::reply_text(&out.text), thinking: out.thinking, - via: job.via.clone(), + via, tool_calls: out.tool_calls, blocks: out.blocks, at: crate::model::now_ms(), ..Default::default() }; + // The quick model says it's done after changing things: the bot's own model looks + // at the results and answers (the quick answer is set aside, never shown). + if msg.tool_calls.is_empty() && job.steer.lock().unwrap().is_empty() && plan.recheck_answer(quick.is_some()) { + reuse_slot = true; + continue; + } send(StreamEvent::StepDone(msg.clone())); let calls = msg.tool_calls.clone(); + plan.changed |= calls.iter().any(|c| !only_looks(&c.name)); + let switched = plan.changed_route(&calls, quick.is_some()); history.push(msg); if calls.is_empty() { // Something typed while it answered keeps the same turn going. @@ -703,6 +928,8 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< continue; } // (Each call's computer was already kept to the chosen one, see keep_to_chosen.) + // Whether any step this round failed, was refused or repeated (see StepPlan::after). + let trouble = std::cell::Cell::new(false); for call in calls { // 鬼打牆: the same screen or browser step again and again (a click on the same // empty spot ten times). The third time it runs with a warning; after that it isn't @@ -713,6 +940,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< tried.push(key); if times >= 4 { let text = format!("沒有執行：這一步你已經試了 {times} 次，畫面都沒有變成你要的樣子。不要再試同一步。改用別的方法：照截圖上的編號點、網頁改用 browser_read 拿連結直接 browser_open，或用 hand_over_computer 請使用者幫忙。"); + trouble.set(true); send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Failed)); let result = Message::tool_result(&call.id, &call.name, &text); send(StreamEvent::ToolResult(result.clone())); @@ -729,6 +957,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< let same = same_answers(&history, &call.name, &call.args); if same >= 3 { let text = format!("沒有執行：同一個工具、同樣的參數你已經做了 {same} 次，每次結果都一模一樣，再做也不會不同。換一個做法（例如用 browser_open 打開要看的網址），或直接告訴使用者你看到了什麼。"); + trouble.set(true); send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Failed)); let result = Message::tool_result(&call.id, &call.name, &text); send(StreamEvent::ToolResult(result.clone())); @@ -742,12 +971,42 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< // Every step ends the same way: its status, then its result (with the notes) and // any picture, shown and kept. let finish = |history: &mut Vec, output: &str, status: ToolStatus, image: Option| { + // A click that changed nothing on the screen is trouble too, though it "worked". + if matches!(status, ToolStatus::Failed | ToolStatus::Denied) || !repeat_note.is_empty() || output.contains("畫面和上一步完全一樣") { + trouble.set(true); + } send(StreamEvent::ToolStatus(call.id.clone(), status)); let mut result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, output))); result.images.extend(image); send(StreamEvent::ToolResult(result.clone())); history.push(result); }; + if call.name == REPLAY_TOOL { + let v = tools::args_of(&call.args); + let name = v["name"].as_str().unwrap_or("").trim().to_string(); + let skill = job.skills.iter().find(|k| k.name == name).cloned(); + match skill { + Some(k) if !k.steps.is_empty() => { + // Another computer named for the replay: every step goes there. + let mut steps: std::collections::VecDeque<_> = k.steps.clone().into(); + if let Some(c) = v["computer"].as_str().filter(|c| !c.trim().is_empty()) { + for st in steps.iter_mut() { + let mut a = tools::args_of(&st.args); + if a.get("computer").is_some() || computer::is_computer_tool(&st.tool) || web::is_web_tool(&st.tool) || st.tool == "shell" { + a["computer"] = serde_json::Value::String(c.to_string()); + st.args = a.to_string(); + } + } + } + let total = steps.len(); + finish(&mut history, &format!("開始重播「{name}」，共 {total} 步：AnyBot 會一步一步照做，每一步先在畫面上找到同一個東西，對不上會停下來交給你。"), ToolStatus::Done, None); + replay = Some(Replay { name, left: steps, total, looked: None }); + } + Some(_) => finish(&mut history, &format!("「{name}」沒有存可以重播的步驟：用 use_skill 讀出做法，照著自己做。"), ToolStatus::Failed, None), + None => finish(&mut history, &format!("沒有叫「{name}」的技能。"), ToolStatus::Failed, None), + } + continue; + } if appctl::is_app_tool(&call.name) { // App actions run on the window's side, which also asks for deletions. send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running)); @@ -1211,7 +1470,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< (Ok(_), Ok(())) if is_screen && call.name == "computer_use_check_permissions" => Decision::Allow, // 裝置連結: another of the person's computers; seeing and operating it ask once each. (Ok(t), Ok(())) if is_screen && on_device.is_some() => { - let looks = matches!(call.name.as_str(), "computer_use_screenshot" | "computer_use_show"); + let looks = matches!(call.name.as_str(), "computer_use_screenshot" | "computer_use_zoom" | "computer_use_show"); let grant = if looks { format!("device-see:{}", t.name) } else { format!("device-control:{}", t.name) }; if grants.contains(&grant) { Decision::Allow @@ -1220,7 +1479,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< } } (Ok(_), Ok(())) if is_screen => { - if matches!(call.name.as_str(), "computer_use_screenshot" | "computer_use_show") { + if matches!(call.name.as_str(), "computer_use_screenshot" | "computer_use_zoom" | "computer_use_show") { if grants.iter().any(|g| g == "computer:see") { Decision::Allow } else { @@ -1261,7 +1520,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< // Only looking (a screenshot, a page or file read) changes nothing, so no // "always ask" rule (deleting, paying, posting…) can be about it: no model // call to check (it cost every look at the person's Mac 10–20 s). - let looks = matches!(call.name.as_str(), "computer_use_screenshot" | "computer_use_show" | "browser_read" | "read_file" | "list_dir" | "search"); + let looks = matches!(call.name.as_str(), "computer_use_screenshot" | "computer_use_zoom" | "computer_use_show" | "browser_read" | "read_file" | "list_dir" | "search"); let verdict = match review { Some(rv) if !rv.ask.is_empty() && !looks => review_action(&job, &Review { allow: vec![], ask: rv.ask.clone() }, &reason, &cancel), _ => ReviewVerdict::NoMatch, @@ -1292,6 +1551,17 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< } else { decision }; + // AppleScript that runs outside the sandbox can drive the person's apps: asked the + // first time even in direct mode, until they choose 「一律允許」. + let on_persons = matches!(target.as_ref().map(|t| &t.place), Ok(tools::Place::Local { .. } | tools::Place::Device { .. })); + let script = tools::args_of(&call.args)["command"].as_str().unwrap_or("").to_string(); + let decision = if call.name == "shell" && on_persons && gate.is_ok() && !matches!(decision, Decision::Deny(_)) && tools::osascript_outside(&script).is_ok() && !grants.iter().any(|g| g == tools::OSASCRIPT_GRANT) { + review_note.clear(); + let short: String = script.chars().take(200).collect(); + Decision::Ask { reason: format!("讓 {} 在沙盒外執行 AppleScript（可以指揮你電腦上的 App）：{short}", job.via_name), grant: tools::OSASCRIPT_GRANT.into() } + } else { + decision + }; let approved = match decision { _ if gate.is_err() => Err((gate.clone().unwrap_err(), ToolStatus::Denied)), _ if target.is_err() => Err((target.clone().err().unwrap_or_default(), ToolStatus::Failed)), @@ -1367,6 +1637,26 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< return; } } + plan.after(trouble.get() || switched); + if scripted.is_some() { + if let Some(r) = replay.as_ref() { + let k = r.total - r.left.len(); + let note = if trouble.get() { + Some(format!("\n\n（重播「{}」在第 {k} 步出了問題，後面的步驟沒有做。接下來由你判斷怎麼做完。）", r.name)) + } else if r.left.is_empty() { + Some(format!("\n\n（重播「{}」做完了，共 {} 步。看一下結果，確認真的做到了再回答使用者。）", r.name, r.total)) + } else { + None + }; + if let Some(n) = note { + if let Some(m) = history.last_mut().filter(|m| m.role == Role::Tool) { + m.text.push_str(&n); + } + replay = None; + plan.after(true); + } + } + } } send(StreamEvent::NextStep(job.via.clone())); send(StreamEvent::StepDone(Message::assistant(&format!("（這次已經用了 {MAX_STEPS} 個步驟，先停在這裡。要繼續的話，回覆「繼續」。）"), &job.via))); @@ -1385,6 +1675,7 @@ mod tests { fn test_job() -> Job { Job { req: ChatRequest { kind: ProviderKind::Custom, base_url: String::new(), api_key: String::new(), model: String::new(), system: String::new(), history: vec![], tools: vec![], session: String::new() }, + step_model: None, computers: vec![], default_computer: String::new(), roles: tools::Roles::default(), @@ -1501,6 +1792,271 @@ mod tests { port } + /// A model server answering `replies` (each a stream delta) in order, recording the model + /// each request asked for. + fn recording_model(replies: Vec) -> (u16, Arc>>) { + use std::io::{Read, Write}; + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = l.local_addr().unwrap().port(); + let asked = Arc::new(Mutex::new(Vec::::new())); + let seen = asked.clone(); + std::thread::spawn(move || { + for delta in replies { + let Ok((mut s, _)) = l.accept() else { return }; + let mut req = Vec::new(); + let mut buf = [0u8; 16384]; + let body = loop { + let n = s.read(&mut buf).unwrap_or(0); + if n == 0 { + break String::new(); + } + req.extend_from_slice(&buf[..n]); + let t = String::from_utf8_lossy(&req).to_string(); + if let Some(end) = t.find("\r\n\r\n") { + let len = t[..end].lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::().unwrap_or(0))).unwrap_or(0); + if req.len() >= end + 4 + len { + break t[end + 4..].to_string(); + } + } + }; + let v: serde_json::Value = serde_json::from_str(&body).unwrap_or_default(); + seen.lock().unwrap().push(v["model"].as_str().unwrap_or("").to_string()); + let out = format!("data: {}\n\ndata: [DONE]\n\n", serde_json::json!({"choices": [{"delta": delta}]})); + let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n"); + let _ = s.write_all(out.as_bytes()); + } + }); + (port, asked) + } + + fn tool_delta(id: &str, name: &str, args: serde_json::Value) -> serde_json::Value { + serde_json::json!({"tool_calls": [{"index": 0, "id": id, "function": {"name": name, "arguments": args.to_string()}}]}) + } + + #[test] + fn a_control_is_found_again_by_what_it_is() { + assert_eq!(control_key("勾選框「自動更新」：1（已勾）"), "勾選框「自動更新」"); + assert_eq!(control_key("按鈕「Close」"), "按鈕「Close」"); + let list = Message::tool_result("x", "computer_use_screenshot", "螢幕截圖。\n[3] 按鈕「Close」\n[9] 勾選框「自動更新」：0"); + let step = crate::memory::ReplayStep { tool: "computer_use_click".into(), args: r#"{"ref":"5","computer":"C"}"#.into(), targets: vec!["勾選框「自動更新」：1（已勾）".into()] }; + let args = resolve_replay(&step, std::slice::from_ref(&list)).unwrap(); + assert_eq!(tools::args_of(&args)["ref"], "9"); + assert_eq!(tools::args_of(&args)["computer"], "C"); + // A browser step finds its control by its words. + let web = crate::memory::ReplayStep { tool: "browser_click".into(), args: r#"{"ref":"4"}"#.into(), targets: vec!["按鈕「開始」".into()] }; + let a = tools::args_of(&resolve_replay(&web, &[]).unwrap()); + assert!(a.get("ref").is_none() && a["text"] == "開始"); + // Not on the screen: says which. + let gone = crate::memory::ReplayStep { tool: "computer_use_click".into(), args: r#"{"ref":"5"}"#.into(), targets: vec!["按鈕「Save」".into()] }; + assert_eq!(resolve_replay(&gone, std::slice::from_ref(&list)).unwrap_err(), "按鈕「Save」"); + } + + /// A skill with saved steps is done again by AnyBot: no model between the steps, then the + /// bot's own model checks the result. + #[test] + fn a_skill_is_replayed_without_the_model_in_between() { + let (port, asked) = recording_model(vec![tool_delta("r", REPLAY_TOOL, serde_json::json!({"name": "兩步"})), serde_json::json!({"content": "確認做完了"})]); + let w = std::env::temp_dir().join(format!("anybot-replay-{}", std::process::id())); + std::fs::create_dir_all(&w).unwrap(); + let skill = crate::memory::Skill { + name: "兩步".into(), + description: "做兩件事".into(), + instructions: "…".into(), + steps: vec![ + crate::memory::ReplayStep { tool: "shell".into(), args: r#"{"command":"echo first > r1.txt"}"#.into(), targets: vec![] }, + crate::memory::ReplayStep { tool: "shell".into(), args: r#"{"command":"echo second > r2.txt"}"#.into(), targets: vec![] }, + ], + }; + let job = Job { + req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "main".into(), system: String::new(), history: vec![Message::user("照上次那樣做")], tools: tools::specs(), session: String::new() }, + step_model: Some("quick".into()), + computers: vec![tools::Target { name: crate::model::THIS_COMPUTER_NAME.into(), place: tools::Place::Local { workdir: w.clone() } }], + default_computer: crate::model::THIS_COMPUTER_NAME.into(), + roles: tools::Roles { own: crate::model::BOT_COMPUTER_NAME.into(), person: Some(crate::model::THIS_COMPUTER_NAME.into()), server: false }, + local_execution: true, + tools_enabled: true, + direct: Arc::new(AtomicBool::new(true)), + skills: vec![skill], + ..test_job() + }; + let (tx, rx) = channel(); + spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default()); + let mut results = vec![]; + let mut last = String::new(); + loop { + match rx.recv_timeout(Duration::from_secs(20)).expect("loop stalled") { + (_, StreamEvent::ToolResult(m)) => results.push(m.text), + (_, StreamEvent::StepDone(m)) => last = m.text, + (_, StreamEvent::NeedApproval(a)) => { + let _ = a.reply.send(ApprovalAnswer::Once); + } + (_, StreamEvent::Done) => break, + (_, StreamEvent::Error(e)) => panic!("{e}"), + _ => {} + } + } + // Two model calls: the one that chose to replay, and the bot's own check at the end. + assert_eq!(*asked.lock().unwrap(), vec!["main", "main"]); + assert_eq!(std::fs::read_to_string(w.join("r1.txt")).unwrap().trim(), "first"); + assert_eq!(std::fs::read_to_string(w.join("r2.txt")).unwrap().trim(), "second"); + assert!(results[0].contains("開始重播「兩步」，共 2 步"), "{results:?}"); + // (「重播做完了」 is added for the model only, like the check-in line: not in the chat.) + assert_eq!(results.len(), 3, "{results:?}"); + assert_eq!(last, "確認做完了"); + } + + #[test] + fn the_quick_model_carries_out_steps_between_the_bots_own() { + let mut p = StepPlan::new(Some("q".into())); + assert_eq!(p.pick(), None, "the first step plans: the bot's own model"); + assert_eq!(p.pick().as_deref(), Some("q")); + p.after(true); + assert_eq!(p.pick(), None, "after trouble: the bot's own model"); + for _ in 0..QUICK_RUN_MAX { + assert_eq!(p.pick().as_deref(), Some("q")); + } + assert_eq!(p.pick(), None, "a look from the bot's own model every QUICK_RUN_MAX steps"); + assert!(!p.recheck_answer(true), "nothing changed: the quick answer stands"); + p.changed = true; + assert!(p.recheck_answer(true) && p.pick().is_none(), "changed: the bot's own model checks and answers"); + p.quick_failed(); + assert_eq!(p.pick(), None); + assert_eq!(StepPlan::new(None).pick(), None); + // The quick model leaving commands for the screen on its own: the bot's own model decides. + let tc = |n: &str| crate::model::ToolCall { id: "x".into(), name: n.into(), args: "{}".into(), status: ToolStatus::Done }; + let mut r = StepPlan::new(Some("q".into())); + assert!(!r.changed_route(&[tc("shell")], false)); + assert!(!r.changed_route(&[tc("computer_use_screenshot")], true), "looking isn't switching"); + assert!(!r.changed_route(&[tc("shell")], true)); + assert!(r.changed_route(&[tc("computer_use_click")], true)); + assert!(!r.changed_route(&[tc("browser_click")], false), "the bot's own model may switch"); + assert_eq!(ProviderKind::Xai.quick_model("grok-4.7"), Some("grok-4.20-0309-non-reasoning")); + assert_eq!(ProviderKind::Xai.quick_model("grok-4.20-0309-non-reasoning"), None); + assert_eq!(ProviderKind::Ollama.quick_model("llama3.2"), None); + } + + /// End to end with a model server that records which model each request asked for: plan with + /// the bot's model, act with the quick one, back to the bot's after a failed step, and the + /// quick model's 「done」 after changing things replaced by the bot's own check. + #[test] + fn a_turn_switches_models_as_planned() { + use std::io::{Read, Write}; + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = l.local_addr().unwrap().port(); + let asked = Arc::new(Mutex::new(Vec::::new())); + let seen = asked.clone(); + std::thread::spawn(move || { + let tool = |id: &str, cmd: &str| serde_json::json!({"tool_calls": [{"index": 0, "id": id, "function": {"name": "shell", "arguments": serde_json::json!({"command": cmd}).to_string()}}]}); + let replies = [ + tool("a", "echo plan-and-start"), + tool("b", "exit 3"), + tool("c", "echo fixed"), + serde_json::json!({"content": "做好了（快速模型）"}), + serde_json::json!({"content": "確認做好了"}), + ]; + for delta in replies { + let (mut s, _) = l.accept().unwrap(); + let mut req = Vec::new(); + let mut buf = [0u8; 16384]; + let body = loop { + let n = s.read(&mut buf).unwrap_or(0); + if n == 0 { + break String::new(); + } + req.extend_from_slice(&buf[..n]); + let t = String::from_utf8_lossy(&req).to_string(); + if let Some(end) = t.find("\r\n\r\n") { + let len = t[..end].lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::().unwrap_or(0))).unwrap_or(0); + if req.len() >= end + 4 + len { + break t[end + 4..].to_string(); + } + } + }; + let v: serde_json::Value = serde_json::from_str(&body).unwrap_or_default(); + seen.lock().unwrap().push(v["model"].as_str().unwrap_or("").to_string()); + let out = format!("data: {}\n\ndata: [DONE]\n\n", serde_json::json!({"choices": [{"delta": delta}]})); + let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n"); + let _ = s.write_all(out.as_bytes()); + } + }); + let w = std::env::temp_dir().join(format!("anybot-steps-{}", std::process::id())); + std::fs::create_dir_all(&w).unwrap(); + let job = Job { + req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "main".into(), system: String::new(), history: vec![Message::user("做一件事")], tools: tools::specs(), session: String::new() }, + step_model: Some("quick".into()), + computers: vec![tools::Target { name: crate::model::THIS_COMPUTER_NAME.into(), place: tools::Place::Local { workdir: w.clone() } }], + default_computer: crate::model::THIS_COMPUTER_NAME.into(), + roles: tools::Roles { own: crate::model::BOT_COMPUTER_NAME.into(), person: Some(crate::model::THIS_COMPUTER_NAME.into()), server: false }, + local_execution: true, + tools_enabled: true, + direct: Arc::new(AtomicBool::new(true)), + via: "Custom · main".into(), + ..test_job() + }; + let (tx, rx) = channel(); + spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default()); + let mut shown = vec![]; + loop { + match rx.recv_timeout(Duration::from_secs(20)).expect("loop stalled") { + (_, StreamEvent::StepDone(m)) => shown.push((m.via.clone(), m.text.clone())), + (_, StreamEvent::NeedApproval(a)) => { + let _ = a.reply.send(ApprovalAnswer::Once); + } + (_, StreamEvent::Done) => break, + (_, StreamEvent::Error(e)) => panic!("{e}"), + _ => {} + } + } + assert_eq!(*asked.lock().unwrap(), vec!["main", "quick", "main", "quick", "main"]); + assert_eq!(shown.iter().map(|(v, _)| v.as_str()).collect::>(), vec!["Custom · main", "Custom · quick", "Custom · main", "Custom · main"]); + assert_eq!(shown.last().unwrap().1, "確認做好了"); + assert!(!shown.iter().any(|(_, t)| t.contains("快速模型")), "the set-aside answer was never shown: {shown:?}"); + } + + /// AppleScript run outside the sandbox drives the person's apps: asked the first time even + /// in direct mode, and not again once they chose 「一律允許」. + #[test] + fn applescript_outside_the_sandbox_is_asked_once_even_in_direct_mode() { + for (granted, expect_ask) in [(false, true), (true, false)] { + let port = fake_model(vec![ + ("shell", serde_json::json!({"command": "osascript -e 'return 1+1'", "computer": crate::model::THIS_COMPUTER_NAME})), + ("", serde_json::json!("好了")), + ]); + let w = std::env::temp_dir().join(format!("anybot-osa-{}-{granted}", std::process::id())); + std::fs::create_dir_all(&w).unwrap(); + let job = Job { + req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("算一下")], tools: tools::specs(), session: String::new() }, + computers: vec![tools::Target { name: crate::model::THIS_COMPUTER_NAME.into(), place: tools::Place::Local { workdir: w.clone() } }], + default_computer: crate::model::THIS_COMPUTER_NAME.into(), + roles: tools::Roles { own: crate::model::BOT_COMPUTER_NAME.into(), person: Some(crate::model::THIS_COMPUTER_NAME.into()), server: false }, + local_execution: true, + tools_enabled: true, + direct: Arc::new(AtomicBool::new(true)), + grants: if granted { vec![tools::OSASCRIPT_GRANT.into()] } else { vec![] }, + ..test_job() + }; + let (tx, rx) = channel(); + spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default()); + let mut asked = vec![]; + loop { + match rx.recv_timeout(Duration::from_secs(20)).expect("loop stalled") { + (_, StreamEvent::NeedApproval(a)) => { + asked.push(a.grant.clone()); + let _ = a.reply.send(ApprovalAnswer::Reject); + } + (_, StreamEvent::Done) => break, + (_, StreamEvent::Error(e)) => panic!("{e}"), + _ => {} + } + } + if expect_ask { + assert_eq!(asked, vec![tools::OSASCRIPT_GRANT.to_string()]); + } else { + assert!(asked.is_empty(), "{asked:?}"); + } + } + } + /// A reply with nothing in it (a dropped stream) is asked again, not taken as the answer. #[test] fn an_empty_reply_is_asked_again() { @@ -1967,6 +2523,7 @@ echo 'done送出了' > done.html; (setsid pyt std::fs::write(w.join("a.txt"), "hi").unwrap(); let job = Job { req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("有什麼檔案？")], tools: tools::specs(), session: String::new() }, + step_model: None, computers: vec![tools::Target { name: crate::model::THIS_COMPUTER_NAME.into(), place: tools::Place::Local { workdir: w } }], default_computer: crate::model::THIS_COMPUTER_NAME.into(), roles: tools::Roles { own: crate::model::BOT_COMPUTER_NAME.into(), person: Some(crate::model::THIS_COMPUTER_NAME.into()), server: false }, @@ -2040,6 +2597,7 @@ echo 'done送出了' > done.html; (setsid pyt let steer = Arc::new(Mutex::new(vec!["順便用英文".to_string()])); let job = Job { req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("打招呼")], tools: vec![], session: String::new() }, + step_model: None, computers: vec![], default_computer: crate::model::BOT_COMPUTER_NAME.into(), roles: tools::Roles { own: crate::model::BOT_COMPUTER_NAME.into(), person: Some(crate::model::THIS_COMPUTER_NAME.into()), server: false }, @@ -2119,6 +2677,7 @@ echo 'done送出了' > done.html; (setsid pyt std::fs::write(w.join("note.txt"), "hi").unwrap(); let job = Job { req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("看看我的電腦")], tools: tools::specs(), session: String::new() }, + step_model: None, computers: vec![tools::Target { name: crate::model::THIS_COMPUTER_NAME.into(), place: tools::Place::Local { workdir: w } }], default_computer: crate::model::THIS_COMPUTER_NAME.into(), roles: tools::Roles { own: crate::model::BOT_COMPUTER_NAME.into(), person: Some(crate::model::THIS_COMPUTER_NAME.into()), server: false }, @@ -2259,6 +2818,7 @@ echo 'done送出了' > done.html; (setsid pyt std::fs::create_dir_all(&w).unwrap(); let job = Job { req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("做一個檔案")], tools: tools::specs(), session: String::new() }, + step_model: None, computers: vec![tools::Target { name: crate::model::THIS_COMPUTER_NAME.into(), place: tools::Place::Local { workdir: w.clone() } }], default_computer: crate::model::THIS_COMPUTER_NAME.into(), roles: tools::Roles { own: crate::model::BOT_COMPUTER_NAME.into(), person: Some(crate::model::THIS_COMPUTER_NAME.into()), server: false }, diff --git a/src/app.rs b/src/app.rs index 40734ea..ffb25d4 100644 --- a/src/app.rs +++ b/src/app.rs @@ -778,6 +778,16 @@ impl App { if b.demos.iter().any(|d| d.shots.iter().any(|s| !s.is_empty())) { tool_specs.push(crate::teach::view_step_spec()); } + // 重播: skills that kept the steps that worked can be done again as they were. + let replayable: Vec = self.state.skills.iter().filter(|k| !k.steps.is_empty()).map(|k| k.name.clone()).collect(); + if !replayable.is_empty() { + tool_specs.push(serde_json::json!({"name": agent::REPLAY_TOOL, + "description": "Do a skill again exactly as it was done before (only skills marked 可重播): AnyBot carries out its saved steps one by one without you, finding each button or field again by what it is; if something isn't there it stops and hands back to you with where. Use it only when the task is the same as the skill's, with nothing that differs this time (other text, other file, other settings) — otherwise read the skill with use_skill and do it yourself. Risky steps still ask the person. When it finishes, check the result before answering.", + "parameters": {"type": "object", "properties": { + "name": {"type": "string", "enum": replayable}, + "computer": {"type": "string", "description": "Only to do it on a different computer than the one it was saved on."}}, + "required": ["name"]}})); + } if !screens.is_empty() { let default_screen = if screens.contains(&default_computer) { default_computer.clone() } else { screens[0].clone() }; tool_specs.extend(crate::computer::specs_for(&screens, &default_screen, screens.contains(&default_computer))); @@ -820,6 +830,12 @@ impl App { self.stick_bottom = true; let review = self.state.auto_review.then(|| agent::Review { allow: self.state.review_allow_rules.clone(), ask: self.state.review_ask_rules.clone() }); let job = agent::Job { + step_model: match b.step_model.trim() { + "off" => None, + "" => b.provider.quick_model(&b.model).map(String::from), + m if m == b.model => None, + m => Some(m.to_string()), + }, req, computers, default_computer, @@ -1705,6 +1721,7 @@ impl App { color: color.into(), provider, model, + step_model: String::new(), pinned: false, messages: vec![], unread: false, @@ -2884,6 +2901,33 @@ impl App { } }); }); + // 想的跟做的分開: this model plans and checks; a quick one carries out the steps. + ui.horizontal(|ui| { + let auto = b.provider.quick_model(&b.model); + let shown = match b.step_model.trim() { + "" => match auto { + Some(q) => format!("自動（{q}）"), + None => "自動（每一步都用上面的模型）".into(), + }, + "off" => "不分開（每一步都用上面的模型）".into(), + m => m.to_string(), + }; + ui.label(RichText::new(t("執行步驟用的模型")).size(12.0).color(p.text3)); + egui::ComboBox::from_id_salt("bot_step_model").selected_text(shown).width(ui.available_width() - 8.0).show_ui(ui, |ui| { + if ui.selectable_label(b.step_model.is_empty(), t("自動")).clicked() { + b.step_model.clear(); + } + if ui.selectable_label(b.step_model == "off", t("不分開（每一步都用上面的模型）")).clicked() { + b.step_model = "off".into(); + } + for m in self.models_for(b.provider) { + if m != b.model && ui.selectable_label(b.step_model == m, &m).clicked() { + b.step_model = m; + } + } + }); + }); + ui.label(RichText::new(t("上面的模型負責想計畫、遇到問題和最後確認；中間照計畫做的步驟交給這個比較快的模型。")).size(11.0).color(p.text3)); let ps = self.state.provider(b.provider); if b.provider.needs_key() && providers::resolve_key(b.provider, &ps.api_key).is_empty() { ui.horizontal(|ui| { @@ -4078,7 +4122,13 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu - 絕對不要直接呼叫網站的內部 API、下載網站的程式碼來研究，或用程式大量抓資料：網站會把帳號當成機器人（例如跳出拼圖驗證、封鎖帳號）。也絕對不要用指令去讀瀏覽器的 Cookie、瀏覽紀錄、密碼或設定檔資料庫。 - 需要使用者親手做的步驟（讀到登入頁、輸入密碼、驗證碼、拼圖、簡訊或 email 驗證、確認付款）：先在那台電腦上把那一頁打開、點到要輸入的地方，再呼叫 hand_over_computer，instruction 寫清楚要他做什麼。聊天裡會出現那台電腦的即時畫面，他點「接管」做完後按「我完成了」，你再確認、接著做（登入一次之後就會記住）。不要只用文字叫他自己去找視窗，絕對不要替他打密碼，也不要叫他改瀏覽器的設定或選單、或說要從 Chrome 複製登入或 Cookie（都不需要，也沒有這個功能）。\n\ - AnyBot 瀏覽器有設定檔（profile），每個有自己的登入。使用者說要換 profile／設定檔／另一個帳號時，用 browser_profiles：list 看有哪些，use 切換，add 新增（新的要登入一次）。也可以在 browser_open / browser_read 帶 profile 參數。使用者想用自己平常的 Chrome（自己的 Chrome 設定檔、已經登入的帳號）時，用 profile「你的瀏覽器」：請使用者在 Chrome 切到想要的設定檔，再 browser_read。\n\ - - 你也能像人一樣操作螢幕（{screens}），用在檔案、指令和網頁工具做不到的事（例如操作其他 App）。照截圖上的編號操作（工具說明裡有怎麼用）；只有沒有編號的東西才用 x、y。結果說「畫面和上一步完全一樣」就是沒點到：換一個編號或換方法，絕對不要重複同一步。已經從畫面知道接下來幾步時（填一張表、一組捷徑），用 computer_use_batch 一次做完；要看上一步結果才知道下一步的，就一步一步來。每次看截圖用一句話說畫面顯示什麼、這步成功與否。\n\ + - 你也能像人一樣操作螢幕（{screens}），用在檔案、指令和網頁工具做不到的事（例如操作其他 App）。照截圖上的編號操作（工具說明裡有怎麼用），按鈕名字清楚的也可以直接用 name 指定（例如計算機的 name「4」、name「=」），不用自己對編號；只有沒有編號的東西才用 x、y。結果說「畫面和上一步完全一樣」就是沒點到：換一個編號或換方法，絕對不要重複同一步。已經從畫面知道接下來幾步時（填一張表、一組捷徑），用 computer_use_batch 一次做完；要看上一步結果才知道下一步的，就一步一步來。每次看截圖用一句話說畫面顯示什麼、這步成功與否。\n\ + - 每件事動手前先選好路：① 有專用工具就用專用工具（網頁用 browser_*，Mac 的提醒事項、行事曆、郵件等用它們的專門工具）。② 改設定、處理檔案、安裝、開關服務、App 有自己的指令列：用指令。③ 只有畫面上才做得到的（App 裡的按鈕或開關、選單列的圖示、沒有指令可改的設定）：操作畫面，先把那個 App 的視窗打開到最前面（Mac 用 open -a App名稱），再照編號點。\n\ + - 換路的規則：一條指令因為環境失敗（Operation not permitted、程式當掉、結果寫著是沙盒擋的），就不要換寫法再試同一條路，改走畫面；畫面上找不到要的東西，先想有沒有指令或專用工具做得到。絕對不要用指令截圖或分析圖片（screencapture、import、PIL）：看畫面一律用 computer_use_screenshot，看不清楚就用 computer_use_zoom。\n\ + - 像一個熟練的人那樣做事：動手前先想好要做哪幾步、做完怎樣才算成功（照「電腦現況」挑那台電腦上最直接的做法，不用再試探環境）；選定一條路就做到底，不要在指令和畫面之間來回換。出錯先讀錯誤訊息想原因再換方法，同一個做法失敗兩次就換。做完只用一種最直接的方法確認一次（讀設定檔、查狀態、看一眼畫面），確認了就回答，不要用好幾種方法反覆確認。\n\ + - 沒見過的 App 或畫面也一樣：先看清楚（截圖、需要就放大），找選單、設定、按鈕上的字，照編號點；網頁上的東西用 browser_* 的編號。\n\ + - 做不到就照實說哪裡卡住、需要什麼。絕對不要用變通的替代品假裝做到（例如只改環境變數讓時鐘看起來對、自己寫一個同名的視窗或程式），也不要做使用者沒要的額外修改。\n\ + - 會切斷你和那台電腦連線的事（關網路、Wi-Fi、VPN、Tailscale、重開機、登出）：先告訴使用者做了會斷線、問他要不要做，他說好再做，做之前先把之後要確認的事情想好。\n\ {}\ - 不要說「我沒有瀏覽器或工具」：先試這些工具。需要權限時 App 會自己跳出視窗問使用者；工具回報要開什麼設定時，照實轉告。", now.format("%Y-%m-%d %H:%M"), diff --git a/src/appctl.rs b/src/appctl.rs index ba35341..7d10b53 100644 --- a/src/appctl.rs +++ b/src/appctl.rs @@ -429,7 +429,7 @@ pub fn apply(state: &mut State, bot_id: u64, name: &str, args: &str) -> Outcome if name.is_empty() || instructions.trim().is_empty() { return err("技能要有名稱和做法".into()); } - crate::memory::upsert_skill(&mut state.skills, crate::memory::Skill { name: name.clone(), description: s("description").unwrap_or_default(), instructions }); + crate::memory::upsert_skill(&mut state.skills, crate::memory::Skill { name: name.clone(), description: s("description").unwrap_or_default(), instructions, steps: vec![] }); ok(format!("已存成技能「{name}」，所有 Bot 都能用。")) } "use_skill" => { @@ -465,6 +465,7 @@ pub fn apply(state: &mut State, bot_id: u64, name: &str, args: &str) -> Outcome color: template.map(|t| t.color).unwrap_or(color).into(), provider, model: model.clone(), + step_model: String::new(), pinned: false, messages: vec![], unread: false, diff --git a/src/axread.rs b/src/axread.rs index 701f261..ef47b71 100644 --- a/src/axread.rs +++ b/src/axread.rs @@ -95,6 +95,7 @@ mod mac { fn AXUIElementSetAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: CFTypeRef) -> i32; fn AXUIElementCopyAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: *mut CFTypeRef) -> i32; fn AXUIElementSetMessagingTimeout(el: CFTypeRef, secs: f32) -> i32; + fn AXValueGetValue(value: CFTypeRef, kind: u32, out: *mut c_void) -> bool; static kAXTrustedCheckOptionPrompt: CFTypeRef; } @@ -288,10 +289,99 @@ mod mac { Err(format!("讀不到「{last_title}」的頁面內容")) } } + + /// One status item in the menu bar (an app's icon at the top right): its app, its words, + /// and where it is in screen points. + pub struct StatusItem { + pub app: String, + pub label: String, + pub rect: [f64; 4], + } + + /// The status items of these apps (pid, name), read through each app's AXExtrasMenuBar. An + /// app that doesn't answer within a quarter second is skipped. + pub fn status_items(apps: &[(i32, String)]) -> Vec { + // Several apps at once: one that is slow to answer holds up only its share. + let chunk = apps.len().div_ceil(8).max(1); + std::thread::scope(|sc| { + let parts: Vec<_> = apps.chunks(chunk).map(|part| sc.spawn(move || status_items_of(part))).collect(); + parts.into_iter().flat_map(|h| h.join().unwrap_or_default()).collect() + }) + } + + /// An icon's words: its own title or description, else the first found among its children + /// (the system's own icons are a group hosting the real button), else its identifier. + fn words_within(el: CFTypeRef, depth: usize) -> String { + let own = ["AXTitle", "AXDescription", "AXValue", "AXHelp"].iter().filter_map(|k| attr_string(el, k)).map(|t| t.trim().to_string()).find(|t| !t.is_empty()); + if let Some(w) = own { + return w; + } + if depth > 0 { + let (_keep, kids) = children(el); + for k in kids { + let w = words_within(k, depth - 1); + if !w.is_empty() { + return w; + } + } + } + attr_string(el, "AXIdentifier").map(|id| super::menu_extra_name(&id)).unwrap_or_default() + } + + fn status_items_of(apps: &[(i32, String)]) -> Vec { + #[repr(C)] + #[derive(Default, Clone, Copy)] + struct Xy { + a: f64, + b: f64, + } + let mut out = vec![]; + for (pid, name) in apps { + let app = Cf(unsafe { AXUIElementCreateApplication(*pid) }); + if app.0.is_null() { + continue; + } + unsafe { AXUIElementSetMessagingTimeout(app.0, 0.25) }; + let Some(bar) = attr(app.0, "AXExtrasMenuBar") else { continue }; + let (_keep, items) = children(bar.0); + for it in items { + let get = |k: &str, kind: u32| -> Option { + let v = attr(it, k)?; + let mut xy = Xy::default(); + unsafe { AXValueGetValue(v.0, kind, &mut xy as *mut Xy as *mut c_void) }.then_some(xy) + }; + // kAXValueCGPointType = 1, kAXValueCGSizeType = 2 + let (Some(p), Some(z)) = (get("AXPosition", 1), get("AXSize", 2)) else { continue }; + if z.a < 1.0 || z.b < 1.0 || p.b > 60.0 { + continue; + } + let label = words_within(it, 3); + out.push(StatusItem { app: name.clone(), label, rect: [p.a, p.b, z.a, z.b] }); + } + } + out + } } #[cfg(target_os = "macos")] pub use mac::trusted; +#[cfg(target_os = "macos")] +pub use mac::status_items; + +/// A system menu-bar icon's words from its identifier (com.apple.menuextra.wifi → Wi-Fi): the +/// system's own icons carry no title. +#[cfg_attr(not(target_os = "macos"), allow(dead_code))] +pub fn menu_extra_name(id: &str) -> String { + let tail = id.rsplit(['.', '/']).next().unwrap_or(id).to_lowercase(); + let known = [ + ("wifi", "Wi-Fi"), ("airport", "Wi-Fi"), ("battery", "電池"), ("bluetooth", "藍牙"), ("clock", "時鐘"), + ("controlcenter", "控制中心"), ("sound", "聲音"), ("volume", "聲音"), ("focusmodes", "專注模式"), ("focus", "專注模式"), + ("display", "顯示器"), ("nowplaying", "正在播放"), ("screenmirroring", "螢幕鏡像輸出"), ("vpn", "VPN"), + ("textinput", "輸入法"), ("spotlight", "Spotlight"), ("siri", "Siri"), ("user", "使用者"), ("timemachine", "時光機"), + ("keyboardbrightness", "鍵盤亮度"), ("accessibility", "輔助使用"), ("stagemanager", "幕前調度"), ("airdrop", "AirDrop"), + ]; + known.iter().find(|(k, _)| tail.contains(k)).map(|(_, v)| v.to_string()).unwrap_or_else(|| id.to_string()) +} #[cfg(not(target_os = "macos"))] pub fn trusted(_prompt: bool) -> bool { diff --git a/src/browser.rs b/src/browser.rs index 20ec3c2..2bf7456 100644 --- a/src/browser.rs +++ b/src/browser.rs @@ -124,22 +124,32 @@ impl Tab { // ---------- what the page offers ---------- +/// Every element matching `sel`, the page's own and those inside open shadow roots (web +/// components: Chromium's settings pages, YouTube, many app frameworks), in the order they are +/// laid out: a host, then what its shadow root shows, then its own children. Plain +/// querySelectorAll stops at every shadow root, so those buttons had no number at all. +const DEEP_JS: &str = "const deepAll = (sel, root) => { const out = []; const walk = r => { for (const el of r.querySelectorAll('*')) { if (sel === '*' || el.matches(sel)) out.push(el); if (el.shadowRoot) walk(el.shadowRoot); } }; walk(root || document); return out; };"; + /// Numbers everything clickable or fillable (data-anybot-ref) and lists it. const ELEMENTS_JS: &str = r#"(() => { + %DEEP% const vis = el => { const r = el.getBoundingClientRect(); if (r.width < 2 || r.height < 2) return false; const s = getComputedStyle(el); return s.visibility !== 'hidden' && s.display !== 'none' && +s.opacity > 0.05; }; const clean = s => (s || '').replace(/\s+/g, ' ').trim(); const name = el => clean(el.getAttribute('aria-label') || el.innerText || el.value || el.getAttribute('placeholder') || el.title || el.getAttribute('alt') || (el.querySelector('img[alt]') || {}).alt || '').slice(0, 80); - const label = el => { const a = el.getAttribute('aria-label'); if (a) return clean(a).slice(0, 60); if (el.labels && el.labels[0]) return clean(el.labels[0].innerText).slice(0, 60); const id = el.getAttribute('aria-labelledby'); if (id) { const l = document.getElementById(id); if (l) return clean(l.innerText).slice(0, 60); } return clean(el.getAttribute('placeholder') || el.name || el.title || ''); }; + const label = el => { const a = el.getAttribute('aria-label'); if (a) return clean(a).slice(0, 60); if (el.labels && el.labels[0]) return clean(el.labels[0].innerText).slice(0, 60); const id = el.getAttribute('aria-labelledby'); if (id) { const home = el.getRootNode(); const l = (home.getElementById ? home : document).getElementById(id); if (l) return clean(l.innerText).slice(0, 60); } return clean(el.getAttribute('placeholder') || el.name || el.title || ''); }; const base = 'a[href],button,input:not([type=hidden]),select,textarea,summary,[contenteditable=""],[contenteditable=true],[role=button],[role=link],[role=tab],[role=menuitem],[role=menuitemcheckbox],[role=option],[role=checkbox],[role=radio],[role=combobox],[role=switch],[role=treeitem],[role=gridcell],[onclick],[tabindex]:not([tabindex="-1"])'; - const set = new Set(document.querySelectorAll(base)); + const all = deepAll('*'); + const order = new Map(all.map((el, i) => [el, i])); + const set = new Set(all.filter(el => el.matches(base))); // Anything the page shows a pointer cursor for (calendar days, cards, custom buttons…). - for (const el of document.querySelectorAll('div,span,li,td,th,p,img,svg,i,label')) { + for (const el of all) { if (set.size > 4000) break; + if (!el.matches('div,span,li,td,th,p,img,svg,i,label')) continue; if (getComputedStyle(el).cursor === 'pointer' && !(el.parentElement && getComputedStyle(el.parentElement).cursor === 'pointer')) set.add(el); } - document.querySelectorAll('[data-anybot-ref]').forEach(e => e.removeAttribute('data-anybot-ref')); + all.forEach(e => e.hasAttribute('data-anybot-ref') && e.removeAttribute('data-anybot-ref')); const inView = el => { const r = el.getBoundingClientRect(); return r.bottom > 0 && r.top < innerHeight && r.right > 0 && r.left < innerWidth; }; - const ordered = [...set].filter(el => vis(el) && !el.closest('[aria-hidden=true]')).sort((a, b) => (a.compareDocumentPosition(b) & 4) ? -1 : 1); + const ordered = [...set].filter(el => vis(el) && !el.closest('[aria-hidden=true]')).sort((a, b) => order.get(a) - order.get(b)); const out = []; const seen = new Set(); let n = 0; for (const el of ordered) { const tag = el.tagName.toLowerCase(); const role = el.getAttribute('role') || ''; @@ -176,7 +186,7 @@ const ELEMENTS_JS: &str = r#"(() => { /// The numbered list of what can be clicked or filled. pub fn elements(tab: &mut Tab) -> Result<(String, String, Vec, bool), String> { - let v = tab.eval(&ELEMENTS_JS.replace("%MAX%", &MAX_ELEMENTS.to_string()))?; + let v = tab.eval(&ELEMENTS_JS.replace("%DEEP%", DEEP_JS).replace("%MAX%", &MAX_ELEMENTS.to_string()))?; let p: Value = v.as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or(Value::Null); let list = p["list"].as_array().map(|a| a.iter().filter_map(|x| x.as_str().map(String::from)).collect()).unwrap_or_default(); Ok((p["t"].as_str().unwrap_or("").into(), p["u"].as_str().unwrap_or("").into(), list, p["more"].as_bool().unwrap_or(false))) @@ -184,11 +194,12 @@ pub fn elements(tab: &mut Tab) -> Result<(String, String, Vec, bool), St /// Draws a numbered box over every listed element that is on screen (removed after the shot). const MARKS_JS: &str = r#"(() => { + %DEEP% const old = document.getElementById('__anybot_marks'); if (old) old.remove(); const box = document.createElement('div'); box.id = '__anybot_marks'; box.style.cssText = 'position:fixed;inset:0;pointer-events:none;z-index:2147483647'; const colors = ['#e11d48', '#2563eb', '#16a34a', '#d97706', '#9333ea']; - for (const el of document.querySelectorAll('[data-anybot-ref]')) { + for (const el of deepAll('[data-anybot-ref]')) { const r = el.getBoundingClientRect(); if (r.bottom < 0 || r.top > innerHeight || r.right < 0 || r.left > innerWidth || r.width < 2) continue; const n = el.getAttribute('data-anybot-ref'); const c = colors[n % colors.length]; @@ -241,12 +252,13 @@ fn find_js(r#ref: &str, text: &str) -> String { let tx = serde_json::to_string(text.trim()).unwrap_or_default(); format!( r#"(() => {{ + {deep} const rf = {rf}, tx = {tx}; - if (rf) {{ const e = document.querySelector('[data-anybot-ref="' + rf + '"]'); if (e) return e; }} + if (rf) {{ const e = deepAll('[data-anybot-ref="' + rf + '"]')[0]; if (e) return e; }} if (!tx) return null; const t = s => (s || '').replace(/\s+/g, ' ').trim(); const vis = e => {{ const r = e.getBoundingClientRect(); return r.width > 1 && r.height > 1; }}; - const all = [...document.querySelectorAll('a,button,[role],input,select,textarea,label,summary,li,td,th,span,div,p,img')].filter(vis); + const all = deepAll('a,button,[role],input,select,textarea,label,summary,li,td,th,span,div,p,img').filter(vis); const hit = e => [t(e.innerText), t(e.getAttribute('aria-label')), t(e.getAttribute('placeholder')), t(e.title), t(e.value), e.labels && e.labels[0] ? t(e.labels[0].innerText) : '']; // The control the words name, not the box around it: 「Customer name:」 is a

holding a // label and its input, 「Submit order」 a

holding the button (its middle is empty page). @@ -255,14 +267,15 @@ fn find_js(r#ref: &str, text: &str) -> String { const inner = e => {{ if (isCtl(e)) return e; if (e.tagName === 'LABEL' && e.control && vis(e.control)) return e.control; - const c = [...e.querySelectorAll(ctl)].filter(vis); + const c = deepAll(ctl, e).filter(vis); return c.length === 1 ? c[0] : e; }}; const shortest = l => l.slice().sort((a, b) => t(a.innerText).length - t(b.innerText).length)[0]; const pick = l => l.find(isCtl) || l.map(inner).find(isCtl) || shortest(l) || null; const exact = all.filter(e => hit(e).includes(tx)); return exact.length ? pick(exact) : pick(all.filter(e => hit(e).some(h => h.includes(tx)))); -}})()"# +}})()"#, + deep = DEEP_JS ) } @@ -691,8 +704,8 @@ pub fn run(name: &str, args: &Value) -> Result<(String, Option), String> let mut listed = vec![]; if marks { let (_, _, list, _) = elements(&mut tab)?; - tab.eval(MARKS_JS)?; - let in_view: std::collections::HashSet = tab.eval("JSON.stringify([...document.querySelectorAll('[data-anybot-ref]')].filter(e => { const r = e.getBoundingClientRect(); return r.bottom > 0 && r.top < innerHeight && r.right > 0 && r.left < innerWidth && r.width > 1; }).map(e => e.getAttribute('data-anybot-ref')))")?.as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or_default(); + tab.eval(&MARKS_JS.replace("%DEEP%", DEEP_JS))?; + let in_view: std::collections::HashSet = tab.eval(&format!("(() => {{ {DEEP_JS} return JSON.stringify(deepAll('[data-anybot-ref]').filter(e => {{ const r = e.getBoundingClientRect(); return r.bottom > 0 && r.top < innerHeight && r.right > 0 && r.left < innerWidth && r.width > 1; }}).map(e => e.getAttribute('data-anybot-ref'))); }})()"))?.as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or_default(); listed = list.into_iter().filter(|l| l.strip_prefix('[').and_then(|r| r.split(']').next()).map(|n| in_view.contains(n)).unwrap_or(false)).collect(); settle(120); } @@ -1003,6 +1016,60 @@ mod tests { assert_eq!(p[0]["webSocketDebuggerUrl"], "ws://c"); } + /// Controls inside web components (open shadow roots, two levels deep) get numbers and can be + /// clicked and typed into, like Chromium's settings pages and many app frameworks. Headless: + /// `ANYBOT_DATA_DIR= ANYBOT_BROWSER_ENGINE=firefox ANYBOT_FIREFOX_HEADLESS=1 cargo test + /// numbers_controls_inside_web_components -- --ignored --nocapture` (or Chrome, with a window). + #[test] + #[ignore] + fn numbers_controls_inside_web_components() { + use std::io::{Read, Write}; + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = l.local_addr().unwrap().port(); + std::thread::spawn(move || { + for mut c in l.incoming().flatten() { + let mut buf = [0u8; 4096]; + let _ = c.read(&mut buf); + let body = r#"元件 +

設定

還沒按

+"#; + let _ = write!(c, "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + let _ = c.write_all(body.as_bytes()); + } + }); + crate::botbrowser::add("shadow-test").unwrap(); + let p = |v: Value| { + let mut v = v; + v["profile"] = json!("shadow-test"); + v + }; + let go = || -> Result<(), String> { + crate::botbrowser::open(&format!("http://127.0.0.1:{port}/"), Some("shadow-test"))?; + let (els, _) = run("browser_elements", &p(json!({})))?; + println!("{els}"); + for want in ["「名稱」", "「開始」", "「自動更新」"] { + assert!(els.contains(want), "{want} missing: {els}"); + } + let n = els.lines().find(|l| l.contains("「開始」")).and_then(|l| l.trim_start().strip_prefix('[')).and_then(|r| r.split(']').next()).unwrap().to_string(); + run("browser_type", &p(json!({"text": "名稱", "value": "小明"})))?; + let (r, _) = run("browser_click", &p(json!({"ref": n})))?; + assert!(r.contains("按了開始：小明"), "{r}"); + let (r, _) = run("browser_click", &p(json!({"text": "自動更新"})))?; + assert!(r.contains("自動更新：開"), "{r}"); + Ok(()) + }; + let r = go(); + let _ = crate::botbrowser::remove("shadow-test"); + r.unwrap(); + } + /// A dashboard like Shopee's affiliate pages: a React-style date picker that only reacts to /// real clicks, a custom dropdown, a table, a hover menu, an export. Run with ANYBOT_DATA_DIR /// set to a scratch folder; opens a browser window for a moment. diff --git a/src/cloudbox.rs b/src/cloudbox.rs index 54be4e5..ea08cbe 100644 --- a/src/cloudbox.rs +++ b/src/cloudbox.rs @@ -49,7 +49,8 @@ pub fn container() -> &'static str { /// home folder is a volume, kept by name; a rename doesn't stop it). fn adopt_usual_box() { let me = container(); - if me == CONTAINER || dk(&["inspect", "-f", "{{.Id}}", me]).is_ok() { + // A box with a volume of its own (a test box) is not the usual one moved: leave that alone. + if own_volume().is_some() || me == CONTAINER || dk(&["inspect", "-f", "{{.Id}}", me]).is_ok() { return; } if dk(&["inspect", "-f", "{{.Id}}", CONTAINER]).is_ok() && dk(&["rename", CONTAINER, me]).is_ok() { @@ -57,8 +58,40 @@ fn adopt_usual_box() { } } pub const IMAGE: &str = "anybot-box:1"; + +/// The box's image (ANYBOT_BOX_IMAGE, else the usual one): a throwaway box (a benchmark) on the +/// same Docker as the real one gets an image of its own, so building a newer box version for it +/// never changes the image the real box was made from. +pub fn image() -> &'static str { + static I: OnceLock = OnceLock::new(); + I.get_or_init(|| { + std::env::var("ANYBOT_BOX_IMAGE") + .ok() + .map(|n| n.trim().to_string()) + .filter(|n| !n.is_empty() && n.len() <= 80 && n.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '-' | '_' | '.' | ':' | '/'))) + .unwrap_or_else(|| IMAGE.to_string()) + }) +} /// The bot's files live here and survive an update (the original's mounted sand-data tree). pub const VOLUME: &str = "anybot-box-home"; + +/// ANYBOT_BOX_VOLUME: a home volume of its own, for a throwaway box (a benchmark) next to the +/// real one on the same Docker: it never adopts the usual box or touches its volume. +fn own_volume() -> Option<&'static str> { + static V: OnceLock> = OnceLock::new(); + V.get_or_init(|| { + std::env::var("ANYBOT_BOX_VOLUME") + .ok() + .map(|n| n.trim().to_string()) + .filter(|n| !n.is_empty() && n.len() <= 60 && n != VOLUME && n.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')) + }) + .as_deref() +} + +/// This box's home volume (ANYBOT_BOX_VOLUME, else the usual one). +pub fn volume() -> &'static str { + own_volume().unwrap_or(VOLUME) +} pub const HOME: &str = "/home/bot"; pub const DISPLAY: &str = ":1"; /// noVNC inside the box (Ap.primaryNovncWebsockify). @@ -748,7 +781,7 @@ pub fn available() -> bool { /// The box image is here and is the version this AnyBot carries (an old one would make a new /// box without the newer parts: the numbered controls, the browser launcher). fn image_exists() -> bool { - dk(&["image", "inspect", "-f", "{{index .Config.Labels \"anybot.box.version\"}}", IMAGE]).map(|v| v.trim() == wanted_version()).unwrap_or(false) + dk(&["image", "inspect", "-f", "{{index .Config.Labels \"anybot.box.version\"}}", image()]).map(|v| v.trim() == wanted_version()).unwrap_or(false) } /// Build the box image from the files baked into AnyBot. @@ -760,7 +793,7 @@ fn build_image(pull: bool) -> Result<(), String> { std::fs::write(dir.join(name), body).map_err(|e| e.to_string())?; } log("正在建立電腦的映像檔（第一次要幾分鐘）…"); - let mut args = vec!["build", "-q", "-t", IMAGE]; + let mut args = vec!["build", "-q", "-t", image()]; if pull { args.push("--pull"); } @@ -797,7 +830,7 @@ fn create_container() -> Result<(), String> { } let port = free_port(HOST_PORT); let publish = format!("127.0.0.1:{port}:{NOVNC_PORT}"); - let vol = format!("{VOLUME}:{HOME}"); + let vol = format!("{}:{HOME}", volume()); let env = format!("SCREEN_SIZE={SCREEN}"); let vnc = format!("VNC_PASSWORD={}", vnc_password()); log("正在開一台新的電腦…"); @@ -832,11 +865,31 @@ fn create_container() -> Result<(), String> { args.push("-e"); args.push(sandbox_env); } - args.extend(["--label", "anybot.box=1", IMAGE]); + args.extend(["--label", "anybot.box=1", image()]); // host.docker.internal → this Mac (for 網路流量改走你的電腦; Docker Desktop and OrbStack already have it). - dk(&args).map_err(|e| format!("開不了電腦：{}", tail(&e, 400)))?; - ensure_holder(); - Ok(()) + // On a Docker host elsewhere (a VM, a server) this Mac can't tell which ports are taken there: + // another box may hold the usual one ("port is already allocated"). Then the next one. + let mut owned: Vec = args.iter().map(|a| a.to_string()).collect(); + let at = owned.iter().position(|a| *a == publish); + let mut port = port; + for _ in 0..20 { + let argv: Vec<&str> = owned.iter().map(String::as_str).collect(); + match dk(&argv) { + Ok(_) => { + ensure_holder(); + return Ok(()); + } + Err(e) if e.contains("port is already allocated") && at.is_some() => { + let _ = dk(&["rm", "-f", container()]); + port += 1; + if let Some(i) = at { + owned[i] = format!("127.0.0.1:{port}:{NOVNC_PORT}"); + } + } + Err(e) => return Err(format!("開不了電腦：{}", tail(&e, 400))), + } + } + Err("開不了電腦：找不到空的連接埠".into()) } fn holder_name() -> String { @@ -848,15 +901,15 @@ fn holder_name() -> String { /// disk) deletes volumes that are attached to nothing, logins and all. One sleeping holder /// keeps it always attached. fn ensure_holder() { - if dk(&["volume", "inspect", "-f", "{{.Name}}", VOLUME]).is_err() { + if dk(&["volume", "inspect", "-f", "{{.Name}}", volume()]).is_err() { return; } if dk(&["inspect", "-f", "{{.State.Running}}", &holder_name()]).map(|s| s == "true").unwrap_or(false) { return; } let _ = dk(&["rm", "-f", &holder_name()]); - let vol = format!("{VOLUME}:{HOME}"); - let _ = dk(&["run", "-d", "--name", &holder_name(), "--label", "anybot.box=1", "-v", &vol, IMAGE, "sleep", "infinity"]); + let vol = format!("{}:{HOME}", volume()); + let _ = dk(&["run", "-d", "--name", &holder_name(), "--label", "anybot.box=1", "-v", &vol, image(), "sleep", "infinity"]); } fn wait_ready(ms: u64) -> Result<(), String> { @@ -1339,7 +1392,7 @@ fn run_rebuild(kind: Rebuild, phase: &dyn Fn(&str, &str)) -> Result<(), String> let _ = dk(&["rm", "-f", container()]); let _ = dk(&["rm", "-f", &holder_name()]); phase("wiping", "清除電腦裡的資料"); - let _ = dk(&["volume", "rm", "-f", VOLUME]); + let _ = dk(&["volume", "rm", "-f", volume()]); phase("creating", "開一台全新的電腦"); if !image_exists() { build_image(false)?; @@ -1358,7 +1411,7 @@ fn run_rebuild(kind: Rebuild, phase: &dyn Fn(&str, &str)) -> Result<(), String> phase("cleaning-up", "拆掉舊的電腦和它的資料"); let _ = dk(&["rm", "-f", container()]); let _ = dk(&["rm", "-f", &holder_name()]); - let _ = dk(&["volume", "rm", "-f", VOLUME]); + let _ = dk(&["volume", "rm", "-f", volume()]); phase("creating", "從快照開一台新的電腦"); if !image_exists() { build_image(false)?; @@ -1480,10 +1533,36 @@ pub fn exec_wrapper(dir: &str, cmd: &str, interactive: bool, job: &str) -> Strin // The job tag rides in the command line so the app can kill the in-container process later // (killing the local docker client alone leaves the program running in the box). let tagged = if job.is_empty() { cmd.to_string() } else { format!("export ANYBOT_JOB={job}; {cmd}") }; - let inner = format!("mkdir -p {d} && cd {d} && {tagged}", d = sh_quote(dir)); + let mut inner = format!("mkdir -p {d} && cd {d} && {tagged}", d = sh_quote(dir)); + // sudo can't raise the bot user to root under gVisor (setuid bits do nothing there: "effective + // uid is not 0"), though the box gives the bot sudo without a password. A command that asks + // for sudo runs as the box's root instead (root's sudo needs no setuid), and what it left in + // the bot's home is handed back to the bot. Still inside the box, same walls. + let user = if uses_sudo(cmd) { + inner = format!("m=$(mktemp); {inner}; rc=$?; find {HOME} -xdev -user root -newer \"$m\" -exec chown bot:bot {{}} + 2>/dev/null; rm -f \"$m\"; exit $rc"); + "root" + } else { + "bot" + }; let env = env_file_path(); let env_arg = if env.exists() { format!(" --env-file {}", sh_quote(&env.display().to_string())) } else { String::new() }; - format!("{} exec{} -u bot -e DISPLAY={DISPLAY} -e HOME={HOME}{}{env_arg} {box} bash -lc {}", docker_prefix(), if interactive { " -i" } else { "" }, egress_env(), sh_quote(&inner), r#box = container()) + format!("{} exec{} -u {user} -e DISPLAY={DISPLAY} -e HOME={HOME} -e USER=bot{}{env_arg} {box} bash -lc {}", docker_prefix(), if interactive { " -i" } else { "" }, egress_env(), sh_quote(&inner), r#box = container()) +} + +/// Run a read-only script in the box as the bot user, for AnyBot's own use (facts.rs): not one +/// of the bot's commands, so no job tag and no sudo rerouting. +pub fn read_facts(script: &str) -> Result { + let out = dk_with_input(&["exec", "-i", "-u", "bot", "-e", &format!("HOME={HOME}"), container(), "sh", "-c", script], Some(b""), 10_000)?; + Ok(String::from_utf8_lossy(&out).to_string()) +} + +/// The command calls sudo (as a command word, not inside another word like "pseudo"). +pub fn uses_sudo(cmd: &str) -> bool { + cmd.match_indices("sudo").any(|(i, _)| { + let before = cmd[..i].chars().next_back(); + let after = cmd[i + 4..].chars().next(); + before.is_none_or(|c| c.is_whitespace() || matches!(c, ';' | '&' | '|' | '(' | '`' | '{' | '!')) && after.is_none_or(|c| c.is_whitespace()) + }) } /// An argv that kills the box process carrying this job tag (and its children). @@ -1866,6 +1945,7 @@ fn box_look(script: Option<&str>, after: Option, pointer: bool) -> Resul // read is the slow part on web pages); a JPEG costs less to make and to ship than a PNG. let look = "( import -window root -quality 90 jpeg:- 2>/dev/null > /tmp/anybot-look.jpg & ip=$!; \ sleep 0.18; anybot-a11y --budget 3 > /tmp/anybot-look.json 2>/dev/null || echo '{\"elements\":[]}' > /tmp/anybot-look.json; \ + printf '%s\\n' \"$(xdotool getactivewindow getwindowname 2>/dev/null | tr -d '\\n')\"; \ wc -c < /tmp/anybot-look.json; cat /tmp/anybot-look.json; wait $ip; cat /tmp/anybot-look.jpg )"; let full = match script { Some(s) => format!("{{ {s}; }} || exit $?; {SETTLE_SH}; {look}"), @@ -1878,7 +1958,10 @@ fn box_look(script: Option<&str>, after: Option, pointer: bool) -> Resul Err(e) => return Err(format!("xdotool：{e}")), }; let exec_ms = t_all.elapsed().as_millis() as u64; - // "\n" + // "\n\n" + let first = out.iter().position(|b| *b == b'\n').ok_or("這台電腦沒有回傳畫面")?; + let front = String::from_utf8_lossy(&out[..first]).trim().to_string(); + let out = &out[first + 1..]; let nl = out.iter().position(|b| *b == b'\n').ok_or("這台電腦沒有回傳畫面")?; let n: usize = String::from_utf8_lossy(&out[..nl]).trim().parse().map_err(|_| "這台電腦回傳的畫面讀不懂")?; let json_end = (nl + 1 + n).min(out.len()); @@ -1886,8 +1969,18 @@ fn box_look(script: Option<&str>, after: Option, pointer: bool) -> Resul let img = image::load_from_memory(&out[json_end..]).map_err(|e| format!("截圖讀不出來：{e}"))?.to_rgb8(); let (w, h) = img.dimensions(); shared().lock().unwrap_or_else(|e| e.into_inner()).size = Some((w, h)); - let els = elements_of(&tree); - let (numbered, lines, spots) = crate::marks::number(&els, false); + // The window in front comes first (its controls numbered first, never crowded out by a + // window left open behind it), like a person looks at what's in front of them. + let mut els = elements_of(&tree); + if !front.is_empty() { + els.sort_by_key(|e| e.window != front); + } + // Numbers kept from the last look where the same control is still there (marks::Numbering). + static NUMBERS: Mutex> = Mutex::new(None); + let (numbered, lines, spots) = { + let mut m = NUMBERS.lock().unwrap_or_else(|e| e.into_inner()); + crate::marks::number_keeping(&els, false, m.get_or_insert_with(Default::default)) + }; *BOX_SPOTS.lock().unwrap_or_else(|e| e.into_inner()) = Some(spots); let (fw, fh, resize) = crate::computer::fit(w, h); let mut pic = if resize { image::imageops::resize(&img, fw, fh, image::imageops::FilterType::Triangle) } else { img }; @@ -1982,11 +2075,23 @@ static BOX_STEP: Mutex<()> = Mutex::new(()); pub fn computer_run(name: &str, v: &Value) -> Result<(String, Option), String> { let _one = BOX_STEP.lock().unwrap_or_else(|e| e.into_inner()); + // A control named instead of numbered (「按 4」): its number in the latest look. + let mut named = v.clone(); + if matches!(name, "computer_use_click" | "computer_use_typing" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_batch") { + let spots = BOX_SPOTS.lock().unwrap_or_else(|e| e.into_inner()).clone(); + crate::marks::names_to_numbers(&mut named, spots.as_ref())?; + } + let v = &named; ensure_running()?; let _busy = Busy::new(true); match name { "computer_use_check_permissions" => return Ok(("這台電腦不需要另外的權限：可以截圖，也可以操作滑鼠和鍵盤。".into(), None)), "computer_use_screenshot" => return box_look(None, None, false), + "computer_use_zoom" => { + let out = dk_with_input(&["exec", "-i", "-u", "bot", "-e", "DISPLAY=:1", container(), "sh", "-c", "import -window root png:- 2>/dev/null"], Some(b""), 30_000).map_err(|e| format!("拍不到畫面：{e}"))?; + let img = image::load_from_memory(&out).map_err(|e| format!("截圖讀不出來：{e}"))?.to_rgb8(); + return crate::computer::zoom(&img, v).map(|(t, j)| (t, Some(j))); + } "computer_use_show" => return box_show(), _ => {} } @@ -2187,6 +2292,18 @@ pub fn shutdown() { mod tests { use super::*; + #[test] + fn a_command_asking_for_sudo_is_recognized() { + for c in ["sudo apt-get install -y tree", "cd x && sudo make install", "ls; sudo -n true", "(sudo id)", "echo ok | sudo tee /etc/x"] { + assert!(uses_sudo(c), "{c}"); + } + for c in ["echo pseudo code", "ls sudoers.d", "cat /etc/sudoers", "grep sudo_user log"] { + assert!(!uses_sudo(c), "{c}"); + } + assert!(exec_wrapper("/home/bot/bot-1", "sudo apt-get install -y tree", false, "j").contains("-u root")); + assert!(exec_wrapper("/home/bot/bot-1", "ls", false, "j").contains("-u bot")); + } + #[test] fn secret_names_follow_the_original_rule() { assert!(valid_secret_name("CRM_API_TOKEN")); @@ -2680,7 +2797,7 @@ mod tests { use chrono::TimeZone; let at = chrono::Utc.with_ymd_and_hms(2026, 11, 6, 19, 0, 0).unwrap().with_timezone(&chrono_tz::Asia::Taipei); assert!(in_schedule_window("03:00", at.time())); - assert_eq!(wanted_version(), "7"); + assert_eq!(wanted_version(), "8"); } #[test] diff --git a/src/computer.rs b/src/computer.rs index 4022e07..58fb026 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -12,7 +12,7 @@ use base64::Engine; use serde_json::{json, Value}; use std::sync::Mutex; -pub const NAMES: [&str; 11] = ["computer_use_screenshot", "computer_use_show", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app", "computer_use_batch"]; +pub const NAMES: [&str; 12] = ["computer_use_screenshot", "computer_use_zoom", "computer_use_show", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app", "computer_use_batch"]; /// Wait after an action before taking the picture that comes back with it (menus open, pages /// start to change; estimated). The box waits for the screen to stop changing instead, up to /// about this long. @@ -230,6 +230,7 @@ pub fn is_computer_tool(name: &str) -> bool { pub fn label(name: &str) -> &'static str { match name { "computer_use_screenshot" => "看你的螢幕", + "computer_use_zoom" => "放大看", "computer_use_show" => "把畫面傳給你看", "computer_use_click" => "點一下", "computer_use_typing" => "打字", @@ -255,6 +256,7 @@ pub fn describe(name: &str, args: &str) -> String { }; match name { "computer_use_screenshot" => "螢幕截圖".into(), + "computer_use_zoom" => format!("放大 ({}, {}) 起 {}×{}", v["x"].as_f64().unwrap_or(0.0), v["y"].as_f64().unwrap_or(0.0), v["width"].as_f64().unwrap_or(0.0), v["height"].as_f64().unwrap_or(0.0)), "computer_use_show" => "畫面傳進聊天".into(), "computer_use_click" => { let b = v["button"].as_str().unwrap_or("left"); @@ -282,30 +284,33 @@ pub fn describe(name: &str, args: &str) -> String { /// way to act (a real click or keystroke, best by number). pub fn specs_for(computers: &[String], default: &str, chosen: bool) -> Vec { let mods = json!({"type": "array", "items": {"type": "string", "enum": ["cmd", "shift", "option", "control"]}}); - let rf = json!({"type": "string", "description": "The number of a control on the latest picture (e.g. 12). Always use a number when the control has one."}); + let rf = json!({"type": "string", "description": "The number of a control on the latest picture (e.g. 12)."}); + let named = json!({"type": "string", "description": "Or the control's name exactly as the latest list shows it (e.g. 4, Close, 自動更新): the numbered control with that name. Easiest for keys and buttons with plain names; use ref when two have the same name."}); let mut v = vec![ json!({"name": "computer_use_screenshot", "description": "See a computer's screen: a picture with every button, link, field and menu item boxed and numbered, and the same numbers in a list with their names. Act on a control by its number (ref). Every action below already returns the next picture, so call this only to start or to look again.", "parameters": {"type": "object", "properties": {}}}), + json!({"name": "computer_use_zoom", "description": "Look closer at part of the latest picture, at the screen's full sharpness: small icons (the menu bar, a toolbar, a tray), small print, a tiny control. Give the area in the latest picture's pixels. You get a bigger, sharper picture of just that area; to act there, use the numbers on the normal picture, or turn a point in the zoomed picture back into the normal picture's pixels as the result explains. Use it instead of guessing, and never make your own zoom with commands.", + "parameters": {"type": "object", "properties": {"x": {"type": "number", "description": "Left edge, in the latest picture's pixels"}, "y": {"type": "number", "description": "Top edge"}, "width": {"type": "number"}, "height": {"type": "number"}}, "required": ["x", "y", "width", "height"]}}), json!({"name": "computer_use_show", "description": "Show the person a computer's screen: a picture of it as they would see it (no numbers) goes into the chat for them, and you see it too. Use it when they ask to see a screen (截圖給我看、讓我看看畫面). One call, no file needed.", "parameters": {"type": "object", "properties": {}}}), json!({"name": "computer_use_click", "description": "Click a control by its number (ref) from the latest picture. Only for something without a number (a spot in a picture or game): x, y in the picture's pixels. Returns the next picture.", - "parameters": {"type": "object", "properties": {"ref": rf, "x": {"type": "number"}, "y": {"type": "number"}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "count": {"type": "integer", "description": "2 for a double click"}, "modifier_keys": mods}}}), + "parameters": {"type": "object", "properties": {"ref": rf, "name": named, "x": {"type": "number"}, "y": {"type": "number"}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "count": {"type": "integer", "description": "2 for a double click"}, "modifier_keys": mods}}}), json!({"name": "computer_use_typing", "description": "Type text (any language) into a field by its number (ref; it is clicked first), or where the cursor is. submit=true presses return after it (send, search, run a command). Returns the next picture.", - "parameters": {"type": "object", "properties": {"value": {"type": "string"}, "ref": rf, "submit": {"type": "boolean"}}, "required": ["value"]}}), + "parameters": {"type": "object", "properties": {"value": {"type": "string"}, "ref": rf, "name": named, "submit": {"type": "boolean"}}, "required": ["value"]}}), json!({"name": "computer_use_press_key", "description": "Press a key or shortcut in the app in front: return, tab, escape, delete, up, slash, cmd+c, cmd+shift+t, cmd+space. Several in order: separate with spaces (\"cmd+a delete\"). On a Linux computer cmd means ctrl. Returns the next picture.", "parameters": {"type": "object", "properties": {"key": {"type": "string"}, "hold_duration_ms": {"type": "integer"}}, "required": ["key"]}}), json!({"name": "computer_use_scroll", "description": "Scroll a list or page: on a control by number (ref), or at x, y, or where the pointer is. Returns the next picture.", - "parameters": {"type": "object", "properties": {"ref": rf, "x": {"type": "number"}, "y": {"type": "number"}, "direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "amount": {"type": "integer", "description": "Scroll steps of 16 pixels, 1-20, default 3"}, "modifier_keys": mods}, "required": ["direction"]}}), + "parameters": {"type": "object", "properties": {"ref": rf, "name": named, "x": {"type": "number"}, "y": {"type": "number"}, "direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "amount": {"type": "integer", "description": "Scroll steps of 16 pixels, 1-20, default 3"}, "modifier_keys": mods}, "required": ["direction"]}}), json!({"name": "computer_use_open_app", "description": "Open an app, or bring it to the front when it is already open (its name as shown, e.g. Ghostty, Calculator, 計算機, Chromium, xterm). Returns the next picture.", "parameters": {"type": "object", "properties": {"app": {"type": "string"}}, "required": ["app"]}}), json!({"name": "computer_use_mouse_move", "description": "Move the mouse onto a control by number (ref) or to x, y (to open a hover menu). Returns the next picture.", - "parameters": {"type": "object", "properties": {"ref": rf, "x": {"type": "number"}, "y": {"type": "number"}}}}), + "parameters": {"type": "object", "properties": {"ref": rf, "name": named, "x": {"type": "number"}, "y": {"type": "number"}}}}), json!({"name": "computer_use_drag", "description": "Drag along a path of points in the picture's pixels (first = press, last = release). Returns the next picture.", "parameters": {"type": "object", "properties": {"path": {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "button": {"type": "string", "enum": ["left", "right"]}, "modifier_keys": mods}, "required": ["path"]}}), json!({"name": "computer_use_check_permissions", "description": "Check whether the screen can be seen and the mouse and keyboard used on that computer.", "parameters": {"type": "object", "properties": {}}}), json!({"name": "computer_use_batch", "description": format!("Do a short planned sequence of steps in one call (max {MAX_BATCH}): each item is one action with its own arguments, e.g. {{\"action\":\"click\",\"ref\":\"12\"}}, {{\"action\":\"typing\",\"ref\":\"15\",\"value\":\"cat\"}}, {{\"action\":\"press_key\",\"key\":\"tab\"}}. Use it when the current picture already tells you every step (filling a form, a known shortcut chain): they run in order, the first step that fails stops the rest, and you get one picture at the end. Do not use it to explore, or where step 2 depends on what step 1's screen shows — act one step at a time there."), - "parameters": {"type": "object", "properties": {"actions": {"type": "array", "maxItems": MAX_BATCH, "items": {"type": "object", "properties": {"action": {"type": "string", "enum": ["click", "typing", "press_key", "scroll", "mouse_move", "drag"]}, "ref": rf.clone(), "x": {"type": "number"}, "y": {"type": "number"}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "count": {"type": "integer"}, "value": {"type": "string"}, "submit": {"type": "boolean"}, "key": {"type": "string"}, "hold_duration_ms": {"type": "integer"}, "direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "amount": {"type": "integer"}, "path": {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "modifier_keys": mods.clone()}, "required": ["action"]}}}, "required": ["actions"]}}), + "parameters": {"type": "object", "properties": {"actions": {"type": "array", "maxItems": MAX_BATCH, "items": {"type": "object", "properties": {"action": {"type": "string", "enum": ["click", "typing", "press_key", "scroll", "mouse_move", "drag"]}, "ref": rf.clone(), "name": named.clone(), "x": {"type": "number"}, "y": {"type": "number"}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "count": {"type": "integer"}, "value": {"type": "string"}, "submit": {"type": "boolean"}, "key": {"type": "string"}, "hold_duration_ms": {"type": "integer"}, "direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "amount": {"type": "integer"}, "path": {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "modifier_keys": mods.clone()}, "required": ["action"]}}}, "required": ["actions"]}}), ]; // Always named: each computer goes by its own name (the instructions say which is which), // so the bot says whose screen it means and the person can tell where it worked. @@ -321,6 +326,49 @@ pub fn specs_for(computers: &[String], default: &str, chosen: bool) -> Vec Result<(String, String), String> { + let (w, h) = img.dimensions(); + let (fw, fh, _) = fit(w, h); + let scale = w as f64 / fw.max(1) as f64; + let num = |k: &str| v[k].as_f64().filter(|n| n.is_finite()); + let (Some(x), Some(y), Some(zw), Some(zh)) = (num("x"), num("y"), num("width"), num("height")) else { + return Err("放大要給 x、y、width、height（最新那張截圖上的像素）".into()); + }; + // In the bot's picture, clamped to it; at least 8 picture pixels each way. + let x0 = x.clamp(0.0, fw as f64 - 8.0); + let y0 = y.clamp(0.0, fh as f64 - 8.0); + let x1 = (x + zw).clamp(x0 + 8.0, fw as f64); + let y1 = (y + zh).clamp(y0 + 8.0, fh as f64); + // The same area in real screen pixels. + let (sx, sy) = ((x0 * scale).floor() as u32, (y0 * scale).floor() as u32); + let sw = (((x1 - x0) * scale).ceil() as u32).clamp(1, w - sx.min(w - 1)); + let sh = (((y1 - y0) * scale).ceil() as u32).clamp(1, h - sy.min(h - 1)); + let crop = image::imageops::crop_imm(img, sx, sy, sw, sh).to_image(); + let up = (ZOOM_LONG_EDGE as f64 / sw.max(sh) as f64).min(ZOOM_MAX_UPSCALE); + let pic = if (up - 1.0).abs() > 0.05 { + let (tw, th) = (((sw as f64 * up).round() as u32).max(1), ((sh as f64 * up).round() as u32).max(1)); + image::imageops::resize(&crop, tw, th, image::imageops::FilterType::CatmullRom) + } else { + crop + }; + let (pw, ph) = pic.dimensions(); + let jpeg = encode_jpeg(&pic).ok_or("放大的圖太大了，沒辦法送出")?; + // How a point in the zoomed picture maps back to the normal picture. + let k = (x1 - x0) / pw as f64; + let text = format!( + "放大圖（{pw}×{ph}）：最新截圖上 ({:.0}, {:.0}) 到 ({:.0}, {:.0}) 的區域，用螢幕原本的清晰度。這張圖上的點 (a, b) 在一般截圖上是 ({:.0} + a×{k:.3}, {:.0} + b×{k:.3})；有編號的東西還是照編號點。", + x0, y0, x1, y1, x0, y0 + ); + Ok((text, base64::engine::general_purpose::STANDARD.encode(&jpeg))) +} + /// zct: the size to send. Long edge down to `MAX_LONG_EDGE`, never an edge under `MIN_EDGE`. pub fn fit(w: u32, h: u32) -> (u32, u32, bool) { if w < 1 || h < 1 { @@ -612,6 +660,80 @@ fn bring_forward(app: &str) { /// The numbered controls of this computer's latest look, and the app they belong to. static SPOTS: Mutex> = Mutex::new(None); +/// How long the menu bar's status icons are trusted before they are read again (they rarely move). +#[cfg(target_os = "macos")] +const MENU_BAR_FRESH_SECS: u64 = 15; + +/// The menu bar's status icons as numbered controls, from every app's AXExtrasMenuBar. +#[cfg(target_os = "macos")] +fn menu_bar_controls() -> Vec { + static CACHE: Mutex)>> = Mutex::new(None); + static REFRESHING: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + let cached = CACHE.lock().unwrap_or_else(|e| e.into_inner()).clone(); + if let Some((t, v)) = cached { + // Stale: use what there is now, and read them again in the background for the next look. + if t.elapsed().as_secs() >= MENU_BAR_FRESH_SECS && !REFRESHING.swap(true, std::sync::atomic::Ordering::SeqCst) { + std::thread::spawn(|| { + let fresh = read_menu_bar(); + *CACHE.lock().unwrap_or_else(|e| e.into_inner()) = Some((std::time::Instant::now(), fresh)); + REFRESHING.store(false, std::sync::atomic::Ordering::SeqCst); + }); + } + return v; + } + let els = read_menu_bar(); + *CACHE.lock().unwrap_or_else(|e| e.into_inner()) = Some((std::time::Instant::now(), els.clone())); + els +} + +#[cfg(target_os = "macos")] +fn read_menu_bar() -> Vec { + crate::axread::status_items(&app_processes()) + .into_iter() + .map(|it| crate::marks::Element { + kind: "選單列圖示".into(), + name: icon_name(&it.app, &it.label), + value: String::new(), + x: it.rect[0], + y: it.rect[1], + w: it.rect[2], + h: it.rect[3], + active: true, + flags: vec![], + window: "選單列".into(), + }) + .collect() +} + +/// How a status icon is listed: the system's own (hosted by an agent process) by their words +/// alone (「Wi‑Fi，已連接」, not 「MenuBarAgent：Wi‑Fi…」), an app's as 「app：words」. +#[cfg_attr(not(target_os = "macos"), allow(dead_code))] +fn icon_name(app: &str, label: &str) -> String { + let system = app.ends_with("Agent") || matches!(app, "ControlCenter" | "Control Center" | "SystemUIServer" | "Siri AI" | "Spotlight"); + match (label.trim(), system) { + ("", _) => app.to_string(), + (l, true) => l.to_string(), + (l, false) if l == app => l.to_string(), + (l, false) => format!("{app}：{l}"), + } +} + +/// This user's running apps and app helpers (anything run from an .app bundle): pid and name. +#[cfg(target_os = "macos")] +fn app_processes() -> Vec<(i32, String)> { + let Ok(out) = std::process::Command::new("/bin/ps").args(["-x", "-o", "pid=,comm="]).output() else { return vec![] }; + String::from_utf8_lossy(&out.stdout) + .lines() + .filter_map(|l| { + let (pid, path) = l.trim_start().split_once(' ')?; + let path = path.trim(); + let i = path.find(".app/Contents/MacOS/")?; + let name = path[..i].rsplit('/').next()?.to_string(); + Some((pid.parse().ok()?, name)) + }) + .collect() +} + /// The latest look's fingerprint (to say when a step changed nothing). static LAST_LOOK: Mutex> = Mutex::new(None); @@ -806,7 +928,18 @@ fn look(after: Option, pointer: bool) -> Result<(String, Option) // A number is a promise that a click there lands in this app: a control another window // covers (or one on another desktop) gets none. let hidden = cover_check(window, &mut els); - let (numbered, lines, spots) = crate::marks::number(&els, picture.is_none()); + // The menu bar's status icons (Tailscale, Wi-Fi, a VPN…) belong to no window in front, so + // they had no numbers and a bot cropped and measured screenshots to find them. + #[cfg(target_os = "macos")] + if controls_allowed() { + els.extend(menu_bar_controls()); + } + // Numbers kept from the last look where the same control is still there (marks::Numbering). + static NUMBERS: Mutex> = Mutex::new(None); + let (numbered, lines, spots) = { + let mut m = NUMBERS.lock().unwrap_or_else(|e| e.into_inner()); + crate::marks::number_keeping(&els, picture.is_none(), m.get_or_insert_with(Default::default)) + }; *SPOTS.lock().unwrap_or_else(|e| e.into_inner()) = Some((app.clone(), spots)); let mut text = after.clone().map(|n| format!("{n}\n")).unwrap_or_default(); let mut fp = crate::marks::fingerprint(&els); @@ -879,6 +1012,13 @@ fn observe(note: String, pointer: bool) -> (String, Option) { } fn run_inner(name: &str, v: &Value) -> Result<(String, Option), String> { + // A control named instead of numbered (「按 4」): its number in the latest look. + let mut named = v.clone(); + if matches!(name, "computer_use_click" | "computer_use_typing" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_batch") { + let spots = SPOTS.lock().unwrap_or_else(|e| e.into_inner()).clone().map(|(_, s)| s); + crate::marks::names_to_numbers(&mut named, spots.as_ref())?; + } + let v = &named; if !cfg!(any(target_os = "macos", target_os = "linux")) { return Err("操作畫面只支援 macOS 和 Linux".into()); } @@ -888,6 +1028,10 @@ fn run_inner(name: &str, v: &Value) -> Result<(String, Option), String> } match name { "computer_use_screenshot" => return look(None, false), + "computer_use_zoom" => { + let img = capture_screen()?; + return zoom(&img, v).map(|(t, j)| (t, Some(j))); + } "computer_use_show" => { let (w, h, jpeg) = plain_picture()?; return Ok((format!("{SHOWN_NOTE}（{w}×{h}）"), Some(base64::engine::general_purpose::STANDARD.encode(&jpeg)))); @@ -1439,6 +1583,61 @@ mod mac { mod tests { use super::*; + #[test] + fn menu_bar_icons_are_named_plainly() { + assert_eq!(icon_name("MenuBarAgent", "Wi‑Fi，已連接，3格"), "Wi‑Fi，已連接，3格"); + assert_eq!(icon_name("Tailscale", ""), "Tailscale"); + assert_eq!(icon_name("Tailscale", "Tailscale"), "Tailscale"); + assert_eq!(icon_name("LINE", "好友名單"), "LINE：好友名單"); + assert_eq!(crate::axread::menu_extra_name("com.apple.menuextra.wifi"), "Wi-Fi"); + assert_eq!(crate::axread::menu_extra_name("com.apple.menuextra.clock"), "時鐘"); + assert_eq!(crate::axread::menu_extra_name("org.example.thing"), "org.example.thing"); + } + + /// The menu bar's icons get numbers on a Mac (needs 輔助使用 for the test runner): + /// `cargo test menu_bar_icons_get_numbers -- --ignored --nocapture`. + #[test] + #[ignore] + #[cfg(target_os = "macos")] + fn menu_bar_icons_get_numbers() { + let t = std::time::Instant::now(); + let els = menu_bar_controls(); + println!("{} icons in {} ms", els.len(), t.elapsed().as_millis()); + for e in &els { + println!(" {} 「{}」 ({:.0},{:.0} {:.0}x{:.0})", e.kind, e.name, e.x, e.y, e.w, e.h); + } + assert!(!els.is_empty() && els.iter().all(|e| e.active && e.y < 60.0 && !e.name.is_empty())); + // The second look within the freshness window is the cached list: instant. + let t = std::time::Instant::now(); + assert_eq!(menu_bar_controls().len(), els.len()); + assert!(t.elapsed().as_millis() < 20); + } + + /// A zoom cuts the asked area from the sharp screen, enlarges it, and says how to map back. + #[test] + fn a_zoom_is_the_asked_area_sharp_and_bigger() { + // A 2048×1152 screen (the bot sees it at 1024×576): a red square at screen (1800..1840, 10..50). + let mut img = image::RgbImage::from_pixel(2048, 1152, image::Rgb([255, 255, 255])); + for x in 1800..1840 { + for y in 10..50 { + img.put_pixel(x, y, image::Rgb([255, 0, 0])); + } + } + // In the bot's picture that square is at (900..920, 5..25): zoom the menu-bar strip around it. + let (text, b64) = zoom(&img, &json!({"x": 880, "y": 0, "width": 64, "height": 32})).unwrap(); + let pic = image::load_from_memory(&base64::engine::general_purpose::STANDARD.decode(b64).unwrap()).unwrap().to_rgb8(); + let (w, h) = pic.dimensions(); + // 64×32 picture pixels = 128×64 screen pixels, enlarged 4× (the cap) to 512×256. + assert_eq!((w, h), (512, 256), "{text}"); + // The square's middle (screen 1820, 30) → crop (60, 30) → zoomed (240, 120): red. + let p = pic.get_pixel(240, 120); + assert!(p[0] > 200 && p[1] < 80, "{p:?}"); + assert!(text.contains("(880, 0) 到 (944, 32)") && text.contains("0.125"), "{text}"); + // Out of the picture or missing numbers: clamped, or said plainly. + assert!(zoom(&img, &json!({"x": 5000, "y": 5000, "width": 10, "height": 10})).is_ok()); + assert!(zoom(&img, &json!({"x": 1})).unwrap_err().contains("width")); + } + #[test] fn screenshots_fit_like_the_original() { assert_eq!(fit(2940, 1912), (1024, 666, true)); diff --git a/src/desktop.rs b/src/desktop.rs index 1dc6c8b..1b7ac3f 100644 --- a/src/desktop.rs +++ b/src/desktop.rs @@ -57,7 +57,7 @@ pub fn run(cmd: &str, a: &Value) -> Result { include_bounds: a["bounds"].as_bool().unwrap_or(true), interactive_only: a["interactive_only"].as_bool().unwrap_or(true), compact: true, - surface: ad::SnapshotSurface::default(), + surface: a["surface"].as_str().and_then(|n| serde_json::from_value(Value::String(n.to_string())).ok()).unwrap_or_default(), skeleton: false, root_ref: opt(a, "root"), timeout_ms: None, @@ -240,3 +240,4 @@ mod tests { assert!(els[2].value.ends_with("END") && els[2].value.contains("中間省略") && els[2].active); } } + diff --git a/src/facts.rs b/src/facts.rs new file mode 100644 index 0000000..8f87651 --- /dev/null +++ b/src/facts.rs @@ -0,0 +1,261 @@ +// 電腦現況: what a person sitting at each computer would already know before starting — which +// system it is, whether it has a desktop and systemd, whether sudo works, whether the network +// is up, how to install things, which apps are there. Bots without this spent most of a job +// finding it out (or never did: they "set the timezone" through an environment variable because +// they believed sudo was broken, and tried timedatectl on a box without systemd). +// +// Read once per computer by one small read-only script, kept for half an hour, and put in the +// instructions at the start of a turn. The person's own computers are read only when the bot +// may already work on them. + +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use crate::tools::{Place, Target}; + +/// How long a computer's facts are trusted (they change when someone installs or reconfigures). +pub const FRESH_SECS: u64 = 30 * 60; +/// Longest a turn waits for facts it doesn't have yet; late answers still fill the cache. +pub const WAIT_MS: u64 = 6_000; +/// Apps listed at most (the rest is "…"). +const MAX_APPS: usize = 30; + +/// Read-only, POSIX sh, the same on Linux and macOS: one `key=value` per line. +pub const PROBE: &str = r#" +k() { printf '%s=%s\n' "$1" "$2"; } +k kernel "$(uname -s)"; k arch "$(uname -m)" +if [ -r /etc/os-release ]; then . /etc/os-release; k os "$PRETTY_NAME"; elif command -v sw_vers >/dev/null 2>&1; then k os "macOS $(sw_vers -productVersion)"; fi +[ "$(uname -s)" = Linux ] && { [ -d /run/systemd/system ] && k systemd yes || k systemd no; } +if [ "$(id -u)" = 0 ]; then k sudo root; elif command -v sudo >/dev/null 2>&1; then if sudo -n true >/dev/null 2>&1; then k sudo yes; else k sudo password; fi; else k sudo none; fi +if command -v curl >/dev/null 2>&1; then curl -s -m 4 -o /dev/null https://www.google.com/generate_204 && k net yes || k net no; fi +for p in apt-get dnf yum pacman apk zypper brew; do command -v $p >/dev/null 2>&1 && k pkg $p; done +k desktop "${XDG_CURRENT_DESKTOP:-}" +[ -n "${TZ:-}" ] && k tz "$TZ" +for w in fluxbox gnome-shell plasmashell xfce4-session mate-session cinnamon sway Hyprland; do pgrep -x $w >/dev/null 2>&1 && k wm $w; done +t=""; for x in python3 node npm git docker curl wget jq ffmpeg xdotool wmctrl osascript shortcuts brew; do command -v $x >/dev/null 2>&1 && t="$t $x"; done; k tools "$t" +if [ -d /usr/share/applications ]; then k apps "$(for f in /usr/share/applications/*.desktop; do sed -n 's/^Name=//p' "$f" | head -1; done 2>/dev/null | sort -u | head -60 | tr '\n' '|')"; fi +if [ -d /Applications ]; then k apps "$(ls /Applications 2>/dev/null | sed -n 's/\.app$//p' | head -80 | tr '\n' '|')"; fi +"#; + +#[derive(Default, Debug, Clone, PartialEq)] +pub struct Facts { + pub kernel: String, + pub arch: String, + pub os: String, + /// Some(true/false) on Linux; None elsewhere. + pub systemd: Option, + /// root, yes, password, none. + pub sudo: String, + pub net: Option, + pub pkg: Vec, + pub desktop: String, + /// A TZ variable every program is started with (it overrides /etc/localtime). + pub tz: String, + pub wm: Vec, + pub tools: Vec, + pub apps: Vec, +} + +pub fn parse(out: &str) -> Facts { + let mut f = Facts::default(); + for line in out.lines() { + let Some((k, v)) = line.split_once('=') else { continue }; + let v = v.trim(); + match k.trim() { + "kernel" => f.kernel = v.into(), + "arch" => f.arch = v.into(), + "os" => f.os = v.into(), + "systemd" => f.systemd = Some(v == "yes"), + "sudo" => f.sudo = v.into(), + "net" => f.net = Some(v == "yes"), + "pkg" => f.pkg.push(v.into()), + "desktop" => f.desktop = v.into(), + "tz" => f.tz = v.into(), + "wm" => f.wm.push(v.into()), + "tools" => f.tools = v.split_whitespace().map(String::from).collect(), + "apps" => f.apps.extend(v.split('|').map(str::trim).filter(|a| !a.is_empty()).map(String::from)), + _ => {} + } + } + f.apps.sort(); + f.apps.dedup(); + f +} + +/// One line about a computer, in the words the bot is instructed in. `own_box`: the bot's own +/// box, where AnyBot runs sudo commands as root (sudo itself can't rise under gVisor). +pub fn describe(name: &str, f: &Facts, own_box: bool, mac_sandbox: bool) -> String { + let mut parts: Vec = vec![]; + let os = if f.os.is_empty() { f.kernel.clone() } else { f.os.clone() }; + parts.push(if f.arch.is_empty() { os } else { format!("{os}（{}）", f.arch) }); + let desk = if !f.wm.is_empty() { f.wm.join("、") } else { f.desktop.clone() }; + if !desk.is_empty() { + parts.push(format!("桌面 {desk}")); + } + match f.systemd { + Some(false) => parts.push("沒有 systemd（systemctl、timedatectl、hostnamectl 都不能用：改系統設定就直接改設定檔，例如時區是 /etc/localtime 和 /etc/timezone）".into()), + Some(true) => parts.push("有 systemd".into()), + None => {} + } + let sudo = if own_box { + "可以 sudo（免密碼；AnyBot 會用 root 執行含 sudo 的指令）" + } else { + match f.sudo.as_str() { + "root" => "指令以 root 執行", + "yes" => "可以 sudo（免密碼）", + "password" if mac_sandbox => "sudo 在指令沙盒裡不能用", + "password" => "sudo 要密碼（不要嘗試，需要時請使用者自己執行）", + "none" => "沒有 sudo", + _ => "", + } + }; + if !sudo.is_empty() { + parts.push(sudo.into()); + } + if !f.tz.is_empty() && f.kernel != "Darwin" { + parts.push(format!("每個程式都帶著環境變數 TZ={}（容器設定的，會蓋過 /etc/localtime：改了系統時區，date 和桌面時鐘還是顯示 {}；確認系統時區用 env -u TZ date，或 readlink /etc/localtime）", f.tz, f.tz)); + } + match f.net { + Some(true) => parts.push("有網路".into()), + Some(false) => parts.push("連不到網路（不要嘗試下載或安裝，先告訴使用者）".into()), + None => {} + } + if let Some(p) = f.pkg.iter().find(|p| p.as_str() != "brew").or(f.pkg.first()) { + parts.push(format!("裝東西用 {p}")); + } + if !f.tools.is_empty() { + parts.push(format!("指令有 {}", f.tools.join("、"))); + } + if mac_sandbox { + parts.push("指令在 AnyBot 的沙盒裡跑：ps、top、App Store 版 App 自己的指令列、單獨一條 osascript -e 會在沙盒外跑，其他碰不到祕密資料夾".into()); + } + if !f.apps.is_empty() { + let more = if f.apps.len() > MAX_APPS { "…" } else { "" }; + parts.push(format!("裝了的 App：{}{more}", f.apps.iter().take(MAX_APPS).cloned().collect::>().join("、"))); + } + format!("「{name}」：{}。", parts.join("；")) +} + +fn cache() -> &'static Mutex> { + static C: std::sync::OnceLock>> = std::sync::OnceLock::new(); + C.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// Which computers to read, and whether they may be read: the bot's own always; the person's +/// only when the bot may already work there (no command runs on them for this otherwise). +fn readable(t: &Target, may_use_persons: bool) -> bool { + match &t.place { + Place::Box { .. } => crate::cloudbox::run_state().is_running(), + Place::Local { .. } => true, + Place::Device { .. } | Place::Ssh { .. } => may_use_persons, + Place::Sealed { .. } => false, + } +} + +/// Run the probe on one computer. The person's own Mac (this one) is read by AnyBot itself, not +/// as one of the bot's commands: nothing the bot asked runs there. +fn probe(t: &Target, via: &str) -> Result { + match &t.place { + Place::Box { .. } => crate::cloudbox::read_facts(PROBE), + Place::Local { .. } => { + let out = std::process::Command::new("/bin/sh").arg("-c").arg(PROBE).env("PATH", crate::cloudbox::tool_path()).output().map_err(|e| e.to_string())?; + Ok(String::from_utf8_lossy(&out.stdout).to_string()) + } + Place::Device { host, .. } => { + let args = serde_json::json!({"command": PROBE, "timeout_ms": 8000}).to_string(); + crate::remote::run_on_device(host, "shell", &args, via).map(|v| v["text"].as_str().unwrap_or("").to_string()) + } + Place::Ssh { .. } => { + let cancel = std::sync::atomic::AtomicBool::new(false); + let ids = crate::tools::RunIds { conversation: "facts".into(), request: "facts".into() }; + let args = serde_json::json!({"command": PROBE, "timeout_ms": 8000}).to_string(); + let (out, ok) = crate::tools::run_on("shell", &args, t, &cancel, &ids); + if ok { Ok(out) } else { Err(out) } + } + Place::Sealed { .. } => Err("no".into()), + } +} + +fn key(t: &Target) -> String { + format!("{}|{:?}", t.name, t.place) +} + +/// The facts section for a turn's instructions: cached facts, plus fresh reads of the rest (at +/// most WAIT_MS for all of them together; a slow one is used next turn). +pub fn for_turn(computers: &[Target], may_use_persons: bool, via: &str) -> String { + let wanted: Vec<&Target> = computers.iter().filter(|t| readable(t, may_use_persons)).collect(); + let fresh = |t: &Target| cache().lock().unwrap_or_else(|e| e.into_inner()).get(&key(t)).filter(|(at, _)| at.elapsed().as_secs() < FRESH_SECS).map(|(_, l)| l.clone()); + let missing: Vec = wanted.iter().filter(|t| fresh(t).is_none()).map(|t| (*t).clone()).collect(); + if !missing.is_empty() { + let (tx, rx) = std::sync::mpsc::channel(); + for t in missing { + let tx = tx.clone(); + let via = via.to_string(); + std::thread::spawn(move || { + if let Ok(out) = probe(&t, &via) { + let f = parse(&out); + if !f.kernel.is_empty() || !f.os.is_empty() { + let mac_sandbox = matches!(t.place, Place::Local { .. }) && cfg!(target_os = "macos") || f.kernel == "Darwin" && matches!(t.place, Place::Device { .. }); + let line = describe(&t.name, &f, matches!(t.place, Place::Box { .. }), mac_sandbox); + cache().lock().unwrap_or_else(|e| e.into_inner()).insert(key(&t), (Instant::now(), line)); + } + } + let _ = tx.send(()); + }); + } + drop(tx); + let end = Instant::now() + Duration::from_millis(WAIT_MS); + while let Some(left) = end.checked_duration_since(Instant::now()) { + if rx.recv_timeout(left).is_err() { + break; + } + } + } + let lines: Vec = wanted.iter().filter_map(|t| fresh(t)).collect(); + if lines.is_empty() { + return String::new(); + } + format!( + "\n\n電腦現況（AnyBot 剛剛自動查的，照這個選做法，不用再花步驟去試探）：\n{}", + lines.iter().map(|l| format!("- {l}")).collect::>().join("\n") + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + const BOX: &str = "kernel=Linux\narch=aarch64\nos=Debian GNU/Linux 13 (trixie)\nsystemd=no\nsudo=password\nnet=yes\npkg=apt-get\ndesktop=\ntz=UTC\nwm=fluxbox\ntools= python3 node npm git curl wget jq xdotool\napps=Chromium|XTerm|galculator|\n"; + const MAC: &str = "kernel=Darwin\narch=arm64\nos=macOS 26.1\nsudo=password\nnet=yes\npkg=brew\ndesktop=\ntools= python3 git curl osascript shortcuts brew\napps=Tailscale|LINE|Safari|\n"; + + #[test] + fn a_box_reads_like_a_person_would_know_it() { + let f = parse(BOX); + assert_eq!(f.systemd, Some(false)); + assert_eq!(f.net, Some(true)); + assert_eq!(f.apps, vec!["Chromium", "XTerm", "galculator"]); + let line = describe("Bot 的電腦", &f, true, false); + for want in ["Debian GNU/Linux 13", "fluxbox", "沒有 systemd", "/etc/localtime", "可以 sudo（免密碼", "有網路", "apt-get", "galculator", "TZ=UTC", "env -u TZ date"] { + assert!(line.contains(want), "{want}: {line}"); + } + } + + #[test] + fn a_mac_says_what_its_sandbox_lets_through() { + let line = describe("王小明的Mac mini", &parse(MAC), false, true); + assert!(line.contains("macOS 26.1") && line.contains("沙盒") && line.contains("osascript") && line.contains("裝東西用 brew") && line.contains("Tailscale"), "{line}"); + assert!(!line.contains("systemd"), "{line}"); + // Offline and no sudo, said plainly so the bot doesn't try. + let off = describe("x", &parse("kernel=Linux\nnet=no\nsudo=password\n"), false, false); + assert!(off.contains("連不到網路") && off.contains("sudo 要密碼"), "{off}"); + } + + /// The probe runs here and says something sensible (any Unix). + #[test] + fn the_probe_runs_on_this_computer() { + let out = std::process::Command::new("/bin/sh").arg("-c").arg(PROBE).output().unwrap(); + let f = parse(&String::from_utf8_lossy(&out.stdout)); + assert!(!f.kernel.is_empty() && !f.arch.is_empty(), "{f:?}"); + } +} diff --git a/src/main.rs b/src/main.rs index a7f8a67..20adb07 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,6 +24,7 @@ mod portal; #[cfg(target_os = "linux")] mod desk; mod egress; +mod facts; mod httpd; mod i18n; mod i18n_table; diff --git a/src/marks.rs b/src/marks.rs index bca214c..f31443a 100644 --- a/src/marks.rs +++ b/src/marks.rs @@ -49,14 +49,79 @@ impl Spots { } } +impl Spots { + /// The numbered control called `name` in the latest look (its name as listed: 「4」, 「Close」, + /// 「自動更新」), so a step can say which button the way a person would. Ok(number) or why not. + pub fn named(&self, name: &str) -> Result { + let want = name.trim().trim_matches(['「', '」']); + let name_of = |what: &str| what.split_once('「').and_then(|(_, r)| r.split_once('」')).map(|(n, _)| n.to_string()); + let mut hits: Vec = self.by_number.iter().filter(|(_, (_, what))| name_of(what).as_deref() == Some(want)).map(|(n, _)| *n).collect(); + hits.sort(); + match hits.as_slice() { + [n] => Ok(*n), + [] => Err(format!("最新的畫面裡沒有叫「{want}」的東西：先看一下畫面（computer_use_screenshot），用清單上的名字或編號。")), + many => Err(format!("最新的畫面裡有 {} 個叫「{want}」的東西（編號 {}）：請用編號指定是哪一個。", many.len(), many.iter().map(|n| n.to_string()).collect::>().join("、"))), + } + } +} + +/// A step that names its control (`name`, no number, no x/y) gets that control's number from +/// the latest look; so does every action of a batch. +pub fn names_to_numbers(v: &mut serde_json::Value, spots: Option<&Spots>) -> Result<(), String> { + let fix = |p: &mut serde_json::Value| -> Result<(), String> { + // ("name" in the screen tools; "text" too, the word the browser tools use.) + let named = ["name", "text"].iter().find_map(|k| p[*k].as_str().map(str::trim).filter(|t| !t.is_empty()).map(String::from)); + if let Some(t) = named { + if p["ref"].is_null() && p["x"].is_null() { + let s = spots.ok_or("先用 computer_use_screenshot 看一下畫面，再照清單上的名字或編號操作。")?; + p["ref"] = serde_json::Value::String(s.named(&t)?.to_string()); + } + } + Ok(()) + }; + if let Some(actions) = v["actions"].as_array_mut() { + for a in actions.iter_mut() { + fix(a)?; + } + return Ok(()); + } + fix(v) +} + pub fn number_of(r: &str) -> Option { let t = r.trim().trim_matches(['[', ']']).trim_start_matches(['e', 'E', '#']); t.parse().ok() } +/// What the last look's numbers meant, so the next look keeps them: a control that is still +/// there keeps its number (the 「7」 key stays #141 after 「1」 was pressed). Renumbering every +/// look broke steps planned from one picture: a bot sending five clicks at once had the second +/// to fifth land on whatever took those numbers after the first click. +#[derive(Default, Debug, Clone)] +pub struct Numbering { + /// (window, kind, name, position bucket) → number. + exact: HashMap<(String, String, String, i64, i64), usize>, + /// (window, kind, name) → numbers (moved a little, still the same control when only one). + loose: HashMap<(String, String, String), Vec>, + windows: Vec, + top: usize, +} + +/// Position buckets for "the same control" (screen units; a list scrolled a little stays put). +const SAME_SPOT: f64 = 24.0; +/// Numbers grow while windows stay; past this they start again from 1. +const MAX_NUMBER: usize = 600; + /// Order (reading order, by window) and number the controls; the list's lines and the spots. /// `all_text`: also list plain text (when there is no picture to read it from). +#[cfg(test)] pub fn number(elements: &[Element], all_text: bool) -> (Vec<(usize, Element)>, Vec, Spots) { + number_keeping(elements, all_text, &mut Numbering::default()) +} + +/// Same, keeping the numbers of controls that were there in the last look (`memory`, updated). +/// Starts again from 1 when none of the last look's windows is still there. +pub fn number_keeping(elements: &[Element], all_text: bool, memory: &mut Numbering) -> (Vec<(usize, Element)>, Vec, Spots) { let mut seen = std::collections::HashSet::new(); let mut els: Vec<&Element> = elements.iter().filter(|e| e.w >= 1.0 && e.h >= 1.0 && seen.insert((e.kind.clone(), e.name.clone(), e.x as i64, e.y as i64, e.w as i64, e.h as i64))).collect(); let mut windows: Vec = vec![]; @@ -70,10 +135,52 @@ pub fn number(elements: &[Element], all_text: bool) -> (Vec<(usize, Element)>, V let wb = windows.iter().position(|w| *w == b.window).unwrap_or(0); (wa, (a.y / 12.0) as i64, a.x as i64).cmp(&(wb, (b.y / 12.0) as i64, b.x as i64)) }); + // Which controls get numbers (reading order, at most MAX_MARKS), then which numbers: the + // last look's where the same control is still there, new ones above the highest. + let marked: Vec<&Element> = els.iter().copied().filter(|e| e.active).take(MAX_MARKS).collect(); + let fresh = memory.top > MAX_NUMBER || !windows.iter().any(|w| memory.windows.contains(w)); + if fresh { + *memory = Numbering::default(); + } + let exact_of = |e: &Element| (e.window.clone(), e.kind.clone(), e.name.clone(), (e.centre().0 / SAME_SPOT).round() as i64, (e.centre().1 / SAME_SPOT).round() as i64); + let loose_of = |e: &Element| (e.window.clone(), e.kind.clone(), e.name.clone()); + let mut taken: std::collections::HashSet = std::collections::HashSet::new(); + let mut assigned: Vec> = marked.iter().map(|e| memory.exact.get(&exact_of(e)).copied()).collect(); + for n in assigned.iter().flatten() { + taken.insert(*n); + } + // Moved a little: the one control of that kind and name in that window, both times. + let mut loose_now: HashMap<(String, String, String), usize> = HashMap::new(); + for e in &marked { + *loose_now.entry(loose_of(e)).or_default() += 1; + } + for (i, e) in marked.iter().enumerate() { + if assigned[i].is_none() && loose_now.get(&loose_of(e)) == Some(&1) { + if let Some([n]) = memory.loose.get(&loose_of(e)).map(Vec::as_slice) { + if !taken.contains(n) { + assigned[i] = Some(*n); + taken.insert(*n); + } + } + } + } + let mut top = memory.top.max(taken.iter().copied().max().unwrap_or(0)); + let mut next = Numbering { windows: windows.clone(), ..Default::default() }; + let mut number_of_el: HashMap<*const Element, usize> = HashMap::new(); + for (i, e) in marked.iter().enumerate() { + let n = assigned[i].unwrap_or_else(|| { + top += 1; + top + }); + number_of_el.insert(*e as *const Element, n); + next.exact.insert(exact_of(e), n); + next.loose.entry(loose_of(e)).or_default().push(n); + } + next.top = top; + *memory = next; let mut numbered = vec![]; let mut lines = vec![]; let mut spots = Spots::default(); - let mut n = 0; let mut window = None; for e in els { if !e.active && !all_text { @@ -84,8 +191,7 @@ pub fn number(elements: &[Element], all_text: bool) -> (Vec<(usize, Element)>, V lines.push(format!("## {}", e.window)); } let what = describe(e); - if e.active && n < MAX_MARKS { - n += 1; + if let Some(&n) = number_of_el.get(&(e as *const Element)) { spots.by_number.insert(n, (e.centre(), what.clone())); numbered.push((n, e.clone())); lines.push(format!("[{n}] {what}")); @@ -224,6 +330,61 @@ mod tests { Element { kind: kind.into(), name: name.into(), value: String::new(), x, y, w: 40.0, h: 20.0, active, flags: vec![], window: "W".into() } } + /// A control still there keeps its number from look to look, even when other controls come + /// and go before it in reading order; a new one gets a number above the highest; a look at a + /// different window starts from 1. + #[test] + fn a_control_can_be_named_instead_of_numbered() { + let els = vec![el("按鈕", "4", 10.0, 10.0, true), el("按鈕", "AC", 60.0, 10.0, true), el("按鈕", "C", 110.0, 10.0, true), el("勾選框", "自動更新", 10.0, 60.0, true)]; + let (_, _, spots) = number(&els, false); + assert_eq!(spots.named("4").unwrap(), 1); + assert_eq!(spots.named("「C」").unwrap(), 3, "C is not AC"); + assert!(spots.named("Save").unwrap_err().contains("沒有叫「Save」")); + let mut v = serde_json::json!({"actions": [{"action": "click", "name": "4"}, {"action": "click", "ref": "2"}, {"action": "typing", "value": "x"}]}); + names_to_numbers(&mut v, Some(&spots)).unwrap(); + assert_eq!(v["actions"][0]["ref"], "1"); + assert_eq!(v["actions"][1]["ref"], "2"); + assert!(v["actions"][2]["ref"].is_null()); + let mut one = serde_json::json!({"name": "自動更新"}); + names_to_numbers(&mut one, Some(&spots)).unwrap(); + assert_eq!(one["ref"], "4"); + } + + #[test] + fn numbers_stay_with_their_controls() { + let mut m = Numbering::default(); + let keys = |extra: bool| { + let mut v = vec![]; + if extra { + v.push(el("選單", "說明", 5.0, 5.0, true)); + } + for (i, k) in ["7", "8", "9", "*"].iter().enumerate() { + v.push(el("按鈕", k, 100.0 + 50.0 * i as f64, 200.0, true)); + } + v + }; + let (first, _, _) = number_keeping(&keys(false), false, &mut m); + let n_of = |list: &[(usize, Element)], name: &str| list.iter().find(|(_, e)| e.name == name).map(|(n, _)| *n).unwrap(); + assert_eq!(n_of(&first, "9"), 3); + // A menu appears first in reading order: the keys keep their numbers, the menu is new. + let (second, lines, spots) = number_keeping(&keys(true), false, &mut m); + assert_eq!(n_of(&second, "9"), 3); + assert_eq!(n_of(&second, "說明"), 5); + assert_eq!(spots.get("3").unwrap().1, "按鈕「9」"); + assert_eq!(lines[0], "[5] 選單「說明」"); + // Moved a little (a list scrolled): still the same control. + let mut moved = keys(true); + for e in moved.iter_mut() { + e.y += 30.0; + } + let (third, _, _) = number_keeping(&moved, false, &mut m); + assert_eq!(n_of(&third, "9"), 3); + // Another window altogether: from 1 again. + let other: Vec = keys(false).into_iter().map(|e| Element { window: "Other".into(), ..e }).collect(); + let (fourth, _, _) = number_keeping(&other, false, &mut m); + assert_eq!(n_of(&fourth, "7"), 1); + } + #[test] fn controls_are_numbered_in_reading_order() { let els = vec![el("按鈕", "下", 10.0, 300.0, true), el("連結", "Profile", 20.0, 100.0, true), el("文字", "標題", 0.0, 50.0, false), el("連結", "Profile", 20.0, 100.0, true)]; diff --git a/src/memory.rs b/src/memory.rs index 30707f3..1457e8a 100644 --- a/src/memory.rs +++ b/src/memory.rs @@ -15,6 +15,21 @@ pub struct Skill { pub name: String, pub description: String, pub instructions: String, + /// The steps that worked, to do again as they were (replay_skill): no model in between, + /// each control found again by what it is, the first mismatch hands back to the bot. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub steps: Vec, +} + +/// One step of a skill to do again: the tool and its arguments, and for a step that acted on a +/// numbered control, what that control was (「按鈕「Close」」), since numbers change from look to +/// look. `targets` has one entry per action of a computer_use_batch ("" where none). +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Default)] +pub struct ReplayStep { + pub tool: String, + pub args: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub targets: Vec, } pub struct Template { @@ -107,7 +122,9 @@ pub const LEARN_SYSTEM: &str = "你在幫一個會操作電腦的 Bot 整理經 只回一個 JSON，不要別的字： 不用存：{\"save\": false, \"why\": \"一句話說為什麼\"} -要存：{\"save\": true, \"name\": \"短名稱\", \"description\": \"一行：什麼時候用\", \"instructions\": \"做法\"}"; +要存：{\"save\": true, \"name\": \"短名稱\", \"description\": \"一行：什麼時候用\", \"instructions\": \"做法\", \"replay\": [步驟編號]} + +replay：下次做同一件事時可以原樣照做的步驟編號（上面清單的號碼，照順序）。只列這次真的成功、必要、會改變東西的步驟（不含截圖、讀取這類只是看的步驟，不含失敗或繞路的步驟）；內容每次會不一樣的工作（發文的文字、要回覆的留言、檔名）不要列，給空陣列 []。"; fn cut(s: &str, n: usize) -> String { let flat = s.split_whitespace().collect::>().join(" "); @@ -116,24 +133,31 @@ fn cut(s: &str, n: usize) -> String { /// What the model looks back on: the request, each step with its result, the answer and the /// skills there are. -pub fn learn_brief(turn: &[crate::model::Message], skills: &[Skill]) -> String { +/// The numbered steps the model looks back on, each with the call it was (None for words the +/// person typed in between, and for the "…省略…" line). +fn brief_steps(turn: &[crate::model::Message]) -> Vec<(String, Option)> { use crate::model::Role; - let mut steps: Vec = vec![]; + let mut steps: Vec<(String, Option)> = vec![]; for (i, m) in turn.iter().enumerate() { if m.role == Role::User && i > 0 { - steps.push(format!("（使用者中途說：{}）", cut(&m.text, 300))); + steps.push((format!("（使用者中途說：{}）", cut(&m.text, 300)), None)); } for c in &m.tool_calls { let result = turn.iter().find(|r| r.role == Role::Tool && r.tool_call_id == c.id).map(|r| r.text.as_str()).unwrap_or(""); let failed = if c.status == crate::model::ToolStatus::Failed { "（失敗）" } else { "" }; - steps.push(format!("{} {} → {}{failed}", c.name, cut(&c.args, 160), cut(result, 220))); + steps.push((format!("{} {} → {}{failed}", c.name, cut(&c.args, 160), cut(result, 220)), Some(c.clone()))); } } const KEEP: usize = 40; if steps.len() > KEEP * 2 { let gone = steps.len() - KEEP * 2; - steps.splice(KEEP..steps.len() - KEEP, [format!("…（中間省略 {gone} 步）…")]); + steps.splice(KEEP..steps.len() - KEEP, [(format!("…（中間省略 {gone} 步）…"), None)]); } + steps +} + +pub fn learn_brief(turn: &[crate::model::Message], skills: &[Skill]) -> String { + let steps: Vec = brief_steps(turn).into_iter().map(|(s, _)| s).collect(); let ask = turn.first().map(|m| cut(&m.text, 800)).unwrap_or_default(); let answer = turn.iter().rev().find(|m| m.role == crate::model::Role::Assistant && !m.text.trim().is_empty()).map(|m| cut(&m.text, 600)).unwrap_or_default(); let mut s = format!("使用者的要求：\n{ask}\n\n每一步（工具 參數 → 結果）：\n"); @@ -166,7 +190,56 @@ pub fn learned_skill(text: &str) -> Option { if name.is_empty() || instructions.is_empty() { return None; } - Some(Skill { name, description: v["description"].as_str().unwrap_or("").trim().chars().take(120).collect(), instructions }) + Some(Skill { name, description: v["description"].as_str().unwrap_or("").trim().chars().take(120).collect(), instructions, steps: vec![] }) +} + +/// The step numbers the model picked to do again as they were (its "replay"). +pub fn learned_replay(text: &str) -> Vec { + let t = text.trim(); + let Some(json) = t.find('{').and_then(|a| t.rfind('}').filter(|b| *b >= a).map(|b| &t[a..=b])) else { return vec![] }; + let v: serde_json::Value = serde_json::from_str(json).unwrap_or_default(); + v["replay"].as_array().map(|a| a.iter().filter_map(|n| n.as_u64().map(|n| n as usize)).collect()).unwrap_or_default() +} + +/// The control a numbered step acted on, in words, from the newest list before it in the turn +/// ("[12] 按鈕「Close」" → 按鈕「Close」). +fn control_named(turn: &[crate::model::Message], before_call: &str, r: &str) -> String { + use crate::model::Role; + let n = r.trim().trim_matches(['[', ']', '#', 'e']); + if n.is_empty() { + return String::new(); + } + let upto = turn.iter().position(|m| m.tool_calls.iter().any(|c| c.id == before_call)).unwrap_or(turn.len()); + let prefix = format!("[{n}] "); + turn[..upto].iter().rev().filter(|m| m.role == Role::Tool).find_map(|m| m.text.lines().find_map(|l| l.trim_start().strip_prefix(&prefix).map(|d| d.trim().to_string()))).unwrap_or_default() +} + +/// The skill's steps from the model's picks: each step's call, with the controls it acted on by +/// number written down as what they were. Steps that only look are left out whatever was picked. +pub fn replay_steps(turn: &[crate::model::Message], picks: &[usize]) -> Vec { + let steps = brief_steps(turn); + let mut out = vec![]; + for &k in picks { + let Some((_, Some(c))) = k.checked_sub(1).and_then(|i| steps.get(i)) else { continue }; + if c.status == crate::model::ToolStatus::Failed || matches!(c.name.as_str(), "computer_use_screenshot" | "computer_use_zoom" | "browser_read" | "browser_screenshot" | "browser_elements" | "use_skill" | "replay_skill") { + continue; + } + let v: serde_json::Value = serde_json::from_str(&c.args).unwrap_or_default(); + let targets: Vec = if c.name == "computer_use_batch" { + v["actions"].as_array().map(|a| a.iter().map(|x| x["ref"].as_str().map(|r| control_named(turn, &c.id, r)).unwrap_or_default()).collect()).unwrap_or_default() + } else { + match v["ref"].as_str().map(String::from).or_else(|| v["ref"].as_u64().map(|n| n.to_string())) { + Some(r) => vec![control_named(turn, &c.id, &r)], + None => vec![], + } + }; + // A numbered step whose control can't be named can't be found again: not a step to replay. + if targets.iter().zip(v["actions"].as_array().map(|a| a.iter().map(|x| !x["ref"].is_null()).collect::>()).unwrap_or_else(|| vec![!v["ref"].is_null()])).any(|(t, had)| had && t.is_empty()) { + return vec![]; + } + out.push(ReplayStep { tool: c.name.clone(), args: c.args.clone(), targets: if targets.iter().all(String::is_empty) { vec![] } else { targets } }); + } + out } /// The line under the bot's reply that says it learned (or improved) a skill. @@ -191,7 +264,11 @@ pub fn prompt_section(memories: &[String], skills: &[Skill]) -> String { if !skills.is_empty() { s.push_str("\n\n你會的技能（以前做成功的做法）。開始做一件事之前先看這裡：有符合的，先用 use_skill 讀出完整做法再照著做，不要重新摸索：\n"); for k in skills { - s.push_str(&format!("- {}：{}\n", k.name, k.description)); + let replay = if k.steps.is_empty() { String::new() } else { format!("（可重播，{} 步）", k.steps.len()) }; + s.push_str(&format!("- {}：{}{replay}\n", k.name, k.description)); + } + if skills.iter().any(|k| !k.steps.is_empty()) { + s.push_str("標著「可重播」的技能：要做的事和技能一模一樣（同一個網站或程式、同樣的設定，沒有每次不同的內容）時，直接用 replay_skill，AnyBot 會照存下的步驟一步一步做，每一步先在畫面上找到同一個東西，對不上就停下來交給你；有一點不同就照做法自己做。\n"); } } s.push_str("\n使用者教你一套可以重複用的做法時，可以用 save_skill 存成技能，所有 Bot 都能用。"); @@ -202,6 +279,33 @@ pub fn prompt_section(memories: &[String], skills: &[Skill]) -> String { mod tests { use super::*; + #[test] + fn the_steps_that_worked_are_kept_with_what_they_acted_on() { + use crate::model::{Message, Role, ToolCall, ToolStatus}; + let call = |id: &str, name: &str, args: &str| Message { role: Role::Assistant, tool_calls: vec![ToolCall { id: id.into(), name: name.into(), args: args.into(), status: ToolStatus::Done }], ..Default::default() }; + let turn = vec![ + Message::user("把自動更新打開"), + call("a", "computer_use_screenshot", "{}"), + Message::tool_result("a", "computer_use_screenshot", "螢幕截圖。\n[3] 按鈕「Close」\n[5] 勾選框「自動更新」：0"), + call("b", "computer_use_click", r#"{"ref":"5"}"#), + Message::tool_result("b", "computer_use_click", "已點 #5（勾選框「自動更新」：0）。"), + call("c", "shell", r#"{"command":"defaults read x"}"#), + Message::tool_result("c", "shell", "1"), + ]; + let brief = learn_brief(&turn, &[]); + assert!(brief.contains("1. computer_use_screenshot") && brief.contains("2. computer_use_click") && brief.contains("3. shell"), "{brief}"); + let steps = replay_steps(&turn, &[1, 2, 3]); + // The look is left out; the click remembers what it clicked. + assert_eq!(steps.len(), 2); + assert_eq!(steps[0].tool, "computer_use_click"); + assert_eq!(steps[0].targets, vec!["勾選框「自動更新」：0".to_string()]); + assert!(steps[1].targets.is_empty()); + assert_eq!(learned_replay(r#"{"save": true, "name": "x", "instructions": "y", "replay": [2, 3]}"#), vec![2, 3]); + assert!(learned_replay(r#"{"save": true, "name": "x", "instructions": "y"}"#).is_empty()); + // A numbered step whose control can't be named: nothing to replay safely. + assert!(replay_steps(&turn[..1].iter().cloned().chain([call("d", "computer_use_click", r#"{"ref":"77"}"#)]).collect::>(), &[1]).is_empty()); + } + #[test] fn remember_and_forget() { let mut m = vec![]; @@ -216,7 +320,7 @@ mod tests { } assert_eq!(m.len(), MAX_MEMORIES); assert_eq!(m[0], "事 5"); - let p = prompt_section(&m[..1], &[Skill { name: "週報".into(), description: "整理每週進度".into(), instructions: "…".into() }]); + let p = prompt_section(&m[..1], &[Skill { name: "週報".into(), description: "整理每週進度".into(), instructions: "…".into(), steps: vec![] }]); assert!(p.contains("1. 事 5") && p.contains("- 週報：整理每週進度") && p.contains("use_skill")); } @@ -265,7 +369,7 @@ mod tests { #[test] fn the_look_back_has_every_step_and_the_skills() { - let skills = [Skill { name: "Threads 發文".into(), description: "在 Threads 發文".into(), instructions: "1. browser_open https://www.threads.com/".into() }]; + let skills = [Skill { name: "Threads 發文".into(), description: "在 Threads 發文".into(), instructions: "1. browser_open https://www.threads.com/".into(), steps: vec![] }]; let b = learn_brief(&post_turn(), &skills); assert!(b.starts_with("使用者的要求：\n幫我發一篇 Threads，記得附圖"), "{b}"); assert!(b.contains("1. browser_open {\"url\":\"https://www.threads.com/\"} → 已打開 https://www.threads.com/ [13] 按鈕「Create」"), "{b}"); diff --git a/src/model.rs b/src/model.rs index 536b5cb..7cffa73 100644 --- a/src/model.rs +++ b/src/model.rs @@ -166,6 +166,11 @@ pub struct Bot { pub color: String, pub provider: ProviderKind, pub model: String, + /// 執行步驟用的模型: "" picks the provider's quick model (providers::step_model), "off" uses + /// `model` for every step, anything else is that model. The bot's own model plans, checks + /// and gets the hard steps; the quick one carries out the planned steps in between. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub step_model: String, #[serde(default)] pub pinned: bool, /// 資料夾 in the sidebar (empty = 未分區), like the original's groups. @@ -586,6 +591,7 @@ impl State { color: color.into(), provider, model: provider.suggested_model().into(), + step_model: String::new(), pinned, messages: vec![], unread: false, diff --git a/src/providers.rs b/src/providers.rs index f331df6..d9b49fe 100644 --- a/src/providers.rs +++ b/src/providers.rs @@ -109,6 +109,20 @@ impl ProviderKind { } } + /// A quick model of the same provider for carrying out planned steps (no long thinking), + /// when the bot's own model is a slower, thinking one. None: every step uses the bot's model. + pub fn quick_model(self, main: &str) -> Option<&'static str> { + let m = main.to_lowercase(); + let (quick, already) = match self { + ProviderKind::Xai => ("grok-4.20-0309-non-reasoning", m.contains("non-reasoning") || m.contains("fast") || m.contains("mini")), + ProviderKind::Anthropic => ("claude-haiku-5-5", m.contains("haiku")), + ProviderKind::Gemini => ("gemini-2.5-flash", m.contains("flash")), + ProviderKind::OpenAi => ("gpt-5-mini", m.contains("mini") || m.contains("nano")), + _ => return None, + }; + (!already && !m.is_empty()).then_some(quick) + } + /// A few names to pick from before the real list is fetched. pub fn starter_models(self) -> &'static [&'static str] { match self { diff --git a/src/tools.rs b/src/tools.rs index e327746..91fcdff 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -512,12 +512,237 @@ pub fn build_command(cmd: &str, workdir: &Path, protected: &[PathBuf], conversat build_command_sealed(cmd, workdir, protected, &[], None, conversation, request) } +/// Characters that join, redirect or substitute commands: a command with any of them is not one +/// plain program call. +const SHELL_JOINERS: [&str; 9] = [";", "&", "|", "<", ">", "`", "$(", "\n", "\\"]; + +/// The program a plain one-program command runs, quotes stripped ("" when it isn't one). +fn plain_program(cmd: &str) -> Option { + let c = cmd.trim(); + if c.is_empty() || SHELL_JOINERS.iter().any(|j| c.contains(j)) { + return None; + } + let first = if let Some(rest) = c.strip_prefix('"') { + rest.split('"').next()?.to_string() + } else if let Some(rest) = c.strip_prefix('\'') { + rest.split('\'').next()?.to_string() + } else { + c.split_whitespace().next()?.to_string() + }; + Some(first) +} + +/// An App Store app's own command line (`/Applications/X.app/Contents/MacOS/Y`): it carries +/// Apple's app sandbox, so outside AnyBot's sandbox it still can't read the secret folders — +/// and inside it, the two sandboxes clash and the program dies at once (Tailscale's CLI: +/// "Trace/BPT trap: 5"). Checked once per program with codesign. +fn app_sandboxed_cli(program: &str) -> bool { + let p = Path::new(program); + let in_app = p.is_absolute() + && !program.contains("..") + && p.components().count() >= 6 + && program.starts_with("/Applications/") + && program.split('/').nth(2).map(|a| a.ends_with(".app")).unwrap_or(false) + && program.contains(".app/Contents/MacOS/"); + if !in_app || !p.is_file() { + return false; + } + static SEEN: OnceLock>> = OnceLock::new(); + let seen = SEEN.get_or_init(|| Mutex::new(HashMap::new())); + if let Some(v) = seen.lock().unwrap_or_else(|e| e.into_inner()).get(program) { + return *v; + } + let ok = Command::new("/usr/bin/codesign") + .args(["-d", "--entitlements", "-", "--xml", program]) + .stderr(Stdio::null()) + .output() + .map(|o| { + let x = String::from_utf8_lossy(&o.stdout).replace(['\n', '\t', ' '], ""); + x.contains("com.apple.security.app-sandbox") + }) + .unwrap_or(false); + seen.lock().unwrap_or_else(|e| e.into_inner()).insert(program.to_string(), ok); + ok +} + +/// `ps` / `top`, alone or piped into plain text filters that take no file: they need system +/// rights (setuid), which no sandboxed process gets ("/bin/ps: Operation not permitted"), and +/// they read no files. +fn process_listing(cmd: &str) -> bool { + let c = cmd.trim(); + if [";", "&", "<", ">", "`", "$(", "\n", "\\", "~"].iter().any(|j| c.contains(j)) { + return false; + } + let mut parts = c.split('|').map(str::trim); + let Some(first) = parts.next() else { return false }; + let mut w = first.split_whitespace(); + let prog = w.next().unwrap_or(""); + if !matches!(prog, "ps" | "/bin/ps" | "top" | "/usr/bin/top") || w.any(|a| a.contains('/')) { + return false; + } + parts.all(|p| { + let mut w = p.split_whitespace(); + matches!(w.next().unwrap_or(""), "grep" | "egrep" | "head" | "tail" | "wc" | "sort" | "uniq" | "cut" | "awk") && !p.contains('/') + }) +} + +/// A command's words the way the shell reads them, or None when it is more than one plain +/// program call: anything outside quotes that joins, redirects, pipes, backgrounds or +/// substitutes (`;` `&` `|` `<` `>` `(` `)` a backtick, `$(`, a line break), and substitution +/// inside double quotes. Quotes, backslashes and `&` inside an AppleScript string are fine. +pub fn plain_words(cmd: &str) -> Option> { + let mut words = vec![]; + let mut cur = String::new(); + let mut started = false; + let mut it = cmd.trim().chars().peekable(); + while let Some(c) = it.next() { + match c { + '\'' => { + started = true; + loop { + match it.next()? { + '\'' => break, + ch => cur.push(ch), + } + } + } + '"' => { + started = true; + loop { + match it.next()? { + '"' => break, + '\\' => match it.next()? { + e @ ('"' | '\\' | '$' | '`') => cur.push(e), + '\n' => {} + e => { + cur.push('\\'); + cur.push(e); + } + }, + '`' => return None, + '$' if it.peek() == Some(&'(') => return None, + ch => cur.push(ch), + } + } + } + '\\' => { + started = true; + match it.next()? { + '\n' => {} + e => cur.push(e), + } + } + ';' | '&' | '|' | '<' | '>' | '(' | ')' | '`' | '\n' => return None, + '$' if it.peek() == Some(&'(') => return None, + c if c.is_whitespace() => { + if started { + words.push(std::mem::take(&mut cur)); + started = false; + } + } + c => { + started = true; + cur.push(c); + } + } + } + if started { + words.push(cur); + } + (!words.is_empty()).then_some(words) +} + +/// What an AppleScript run outside the sandbox may not do: run shell commands or other +/// scripts, drive a terminal or type keystrokes (into one), read files or reach the Objective-C +/// bridge, or name the secret folders. Outside the sandbox nothing else stops it, so this list +/// errs on the side of refusing; refused scripts still run, inside the sandbox. +const OSASCRIPT_OUTSIDE_REFUSED: [&str; 24] = [ + "do shell script", "run script", "load script", "store script", "do script", "terminal", "iterm", "warp", "ghostty", "keystroke", "key code", + "objc", "$.ns", "nsstring", "nsdata", "nsfilemanager", "read file", "read posix file", "open for access", ".ssh", "keychain", ".gnupg", ".aws", "gcloud", +]; + +/// A plain `osascript -e …` call that may run outside the sandbox (Apple events to other apps +/// only work there: inside, every app answers -10004), and why not when it may not. +pub fn osascript_outside(cmd: &str) -> Result<(), String> { + let words = plain_words(cmd).ok_or("不是單獨一條 osascript 指令")?; + if !matches!(words[0].as_str(), "osascript" | "/usr/bin/osascript") { + return Err("不是 osascript".into()); + } + // Only inline AppleScript (-e …): no script files (unseen code), no JavaScript (its ObjC + // bridge reads anything), no interactive mode. + let mut script = String::new(); + let mut i = 1; + while i < words.len() { + match words[i].as_str() { + "-e" if i + 1 < words.len() => { + script.push_str(&words[i + 1]); + script.push('\n'); + i += 2; + } + "-s" if i + 1 < words.len() => i += 2, + w if w.starts_with("-s") => i += 1, + _ => return Err("只能用 -e 寫 AppleScript（不能跑腳本檔、JavaScript 或互動模式）".into()), + } + } + if script.trim().is_empty() { + return Err("沒有 -e 的 AppleScript".into()); + } + let low = script.to_lowercase(); + let home = dirs::home_dir().map(|h| h.display().to_string().to_lowercase()).unwrap_or_default(); + let data = crate::model::State::path().parent().map(|d| d.display().to_string().to_lowercase()).unwrap_or_default(); + if let Some(bad) = OSASCRIPT_OUTSIDE_REFUSED.iter().find(|b| low.contains(*b)) { + return Err(format!("用到了「{bad}」")); + } + if (!data.is_empty() && low.contains(&data)) || low.contains("application support/anybot") || (!home.is_empty() && low.contains(&format!("{home}/library/keychains"))) { + return Err("碰到 AnyBot 的資料夾或鑰匙圈".into()); + } + Ok(()) +} + +/// Commands that run outside AnyBot's sandbox on the person's Mac, because the sandbox breaks +/// them and they can't reach the secret folders anyway (see app_sandboxed_cli, process_listing), +/// or because the person allowed AppleScript to drive their apps (osascript_outside; asked first). +pub fn runs_outside_sandbox(cmd: &str) -> bool { + if !cfg!(target_os = "macos") { + return false; + } + process_listing(cmd) || osascript_outside(cmd).is_ok() || plain_program(cmd).map(|p| app_sandboxed_cli(&p)).unwrap_or(false) +} + +/// The grant that lets AppleScript run outside the sandbox without asking again. +pub const OSASCRIPT_GRANT: &str = "osascript:outside"; + +/// A word for the bot when a command failed because of the sandbox, not because the program or +/// the computer is broken — so it changes course instead of trying the same thing again. +pub fn sandbox_note(cmd: &str, out: &str) -> Option<&'static str> { + sandbox_exec()?; + if runs_outside_sandbox(cmd) { + return None; + } + let setuid = ["/bin/ps: Operation not permitted", "/usr/bin/top: Operation not permitted", "sudo: Operation not permitted", "/usr/bin/sudo: Operation not permitted"]; + if setuid.iter().any(|s| out.contains(s)) { + return Some("（這是 AnyBot 的指令沙盒擋下的，不是電腦壞了：ps、top、sudo 這類要系統權限的程式在沙盒裡不能跑。查程序改用 pgrep -fl 名字，或把 ps／top 單獨執行（後面只接 grep、head 這類過濾），那樣會在沙盒外跑。）"); + } + if out.contains("(-10004)") || out.contains("越權取用") || out.to_lowercase().contains("privilege violation") { + return Some(match osascript_outside(cmd) { + // Plain osascript that still got -10004 can't be the sandbox: macOS itself said no. + Ok(()) => return None, + Err(_) if cmd.contains("osascript") => "（這是 AnyBot 的指令沙盒擋下的，不是那個 App 壞了：AppleScript 只有寫成單獨一條 osascript -e '…' 指令（不接其他指令、不用 heredoc，多行就多個 -e），而且不跑 shell 指令、不碰終端機、不讀檔案時，才會在沙盒外跑、能指揮別的 App。改寫成那樣，或改用畫面操作（computer_use_*）。）", + Err(_) => "（這是 AnyBot 的指令沙盒擋下的，不是那個 App 壞了：在沙盒裡不能指揮別的 App（-10004）。改用畫面操作（computer_use_*），或寫成單獨一條 osascript -e '…' 指令。）", + }); + } + if (out.contains("Trace/BPT trap") || out.contains("指令因為收到訊號而終止")) && cmd.contains(".app/Contents/MacOS/") { + return Some("（這個 App 的指令列在 AnyBot 的沙盒裡會當掉，不是 App 壞了。用它的完整路徑單獨執行（不要接 ;、|、&& 或其他指令），App Store 版的 App 就會在沙盒外跑；不行就改用畫面操作。）"); + } + None +} + /// `seal = Some(home)`: the bot's own computer. Nothing under the person's home folder is /// visible or writable except the bot's folder (`workdir`), like a separate machine. /// `readable`: folders inside `protected` that may still be read (never written). pub fn build_command_sealed(cmd: &str, workdir: &Path, protected: &[PathBuf], readable: &[PathBuf], seal: Option<&Path>, conversation: &str, request: &str) -> Command { let mut c = match sandbox_exec() { - Some(se) if !protected.is_empty() || seal.is_some() => { + Some(se) if (!protected.is_empty() || seal.is_some()) && !(seal.is_none() && runs_outside_sandbox(cmd)) => { let rules: String = (0..protected.len()).map(|i| format!(" (subpath (param \"P{i}\"))")).collect(); let mut profile = format!("(version 1)(allow default)(deny file-read* file-write* (with no-report){rules})"); if seal.is_some() { @@ -702,6 +927,31 @@ fn wrapper_for(place: &Place, cmd: &str, interactive: bool, job: &str) -> Option } } +/// On the bot's own box sudo is the bot's (the box gives it sudo without a password, and the +/// instructions say it may install packages): only what follows sudo is checked against the +/// always-denied list (`sudo rm -rf /` is still refused). It was refused outright, so the bot, +/// told it could sudo, worked around a block it couldn't see the reason for. +fn denied_on_box(cmd: &str) -> Option { + for part in split_commands(cmd)? { + let mut rest = part.trim(); + while let Some(after) = rest.strip_prefix("sudo").filter(|a| a.is_empty() || a.starts_with(char::is_whitespace)) { + rest = after.trim_start(); + // sudo's own options (-n, -E, -H, -u root…) before the command. + while rest.starts_with('-') { + let (opt, more) = rest.split_once(char::is_whitespace).unwrap_or((rest, "")); + rest = more.trim_start(); + if matches!(opt, "-u" | "-g" | "-C" | "-D" | "-h" | "-p" | "-r" | "-t" | "-U") { + rest = rest.split_once(char::is_whitespace).map(|(_, m)| m.trim_start()).unwrap_or(""); + } + } + } + if let Some(d) = DENIED_COMMANDS.iter().filter(|d| **d != "sudo").find(|d| starts_with_words(rest, d)) { + return Some(Decision::Deny(format!("「{d}」這類指令一律不允許。"))); + } + } + None +} + pub fn decide_on(name: &str, args: &str, target: &Target, grants: &[String]) -> Decision { let v = args_of(args); let denied = |cmd: &str| -> Option { @@ -734,7 +984,7 @@ pub fn decide_on(name: &str, args: &str, target: &Target, grants: &[String]) -> // Its own computer: everything is allowed (it can be rebuilt), like the original's box. Place::Box { .. } => { if name == "shell" { - if let Some(d) = denied(&arg_str(&v, "command")) { + if let Some(d) = denied_on_box(&arg_str(&v, "command")) { return d; } } @@ -1126,7 +1376,13 @@ fn edit_file(p: &Path, old: &str, new: &str) -> Result { } fn shell(cmd: &str, workdir: &Path, timeout_ms: u64, cancel: &AtomicBool, ids: &RunIds, seal: Option<&Path>) -> Result { - run_command(build_command_sealed(cmd, workdir, &protected_dirs(), &readable_dirs(), seal, &ids.conversation, &ids.request), cmd, timeout_ms, cancel, ids, None) + let r = run_command(build_command_sealed(cmd, workdir, &protected_dirs(), &readable_dirs(), seal, &ids.conversation, &ids.request), cmd, timeout_ms, cancel, ids, None); + // (The bot's own sealed computer keeps every command in the sandbox: no hints about outside.) + let note = |out: &str| sandbox_note(cmd, out).filter(|_| seal.is_none()).map(|n| format!("{out}\n{n}")).unwrap_or_else(|| out.to_string()); + match r { + Ok(out) => Ok(note(&out)), + Err(out) => Err(note(&out)), + } } /// Run a wrapper command (ssh / docker exec) the same way: background hand-off, stop, output. @@ -1332,6 +1588,115 @@ mod tests { assert!(ok && q.contains("quick")); } + #[test] + fn sandbox_breakers_are_recognized_and_nothing_else() { + assert!(process_listing("ps aux")); + assert!(process_listing("ps -ax | grep -i tailscale | grep -v grep")); + assert!(process_listing("/bin/ps -o pid,comm | awk '{print $1}' | head -5")); + assert!(process_listing("top -l 1 -n 0")); + assert!(!process_listing("ps aux > /tmp/x")); + assert!(!process_listing("ps aux | grep x ~/.ssh/id_rsa")); + assert!(!process_listing("ps aux | xargs cat")); + assert!(!process_listing("ps aux; cat ~/.ssh/id_rsa")); + assert!(!process_listing("ps $(cat /etc/passwd)")); + assert!(!process_listing("pss aux")); + assert_eq!(plain_program("/Applications/Tailscale.app/Contents/MacOS/Tailscale status").as_deref(), Some("/Applications/Tailscale.app/Contents/MacOS/Tailscale")); + assert_eq!(plain_program("\"/Applications/My App.app/Contents/MacOS/My App\" --help").as_deref(), Some("/Applications/My App.app/Contents/MacOS/My App")); + assert_eq!(plain_program("/Applications/T.app/Contents/MacOS/T status; cat ~/.ssh/id_rsa"), None); + assert_eq!(plain_program("/Applications/T.app/Contents/MacOS/T status | tee x"), None); + assert!(!app_sandboxed_cli("/usr/bin/osascript")); + assert!(!app_sandboxed_cli("/Applications/../usr/bin/osascript")); + assert!(!app_sandboxed_cli("/Applications/Nope.app/Contents/MacOS/Nope")); + assert!(!runs_outside_sandbox("osascript -e 'do shell script \"cat ~/.ssh/id_rsa\"'")); + assert!(!runs_outside_sandbox("cat ~/.ssh/id_rsa")); + } + + #[test] + fn sudo_is_the_bots_on_its_own_box_but_what_follows_is_still_checked() { + let b = Target { name: "Bot 的電腦".into(), place: Place::Box { dir: "/home/bot/bot-1".into() } }; + let d = |c: &str| decide_on("shell", &json!({"command": c}).to_string(), &b, &[]); + for ok in ["sudo apt-get install -y tree", "sudo ln -sf /usr/share/zoneinfo/Asia/Tokyo /etc/localtime && echo Asia/Tokyo | sudo tee /etc/timezone", "sudo -n -u root true", "ls; sudo -E make install"] { + assert!(matches!(d(ok), Decision::Allow), "{ok}"); + } + for bad in ["sudo rm -rf /", "sudo -u root mkfs /dev/sda", "rm -rf ~", "sudo sudo dd if=/dev/zero of=/dev/sda", "su -c id", "sudo su"] { + assert!(matches!(d(bad), Decision::Deny(_)), "{bad}"); + } + // The person's own computer is unchanged: sudo stays refused there. + let w = std::env::temp_dir(); + assert!(matches!(decide("shell", r#"{"command":"sudo ls"}"#, &w, &[]), Decision::Deny(_))); + } + + #[test] + fn applescript_runs_outside_only_when_plain_and_harmless() { + assert_eq!(plain_words(r#"osascript -e 'tell application "Tailscale" to quit' -e "set x to \"a\" & \"b\"""#).unwrap(), vec!["osascript", "-e", r#"tell application "Tailscale" to quit"#, "-e", r#"set x to "a" & "b""#]); + assert!(plain_words("osascript -e 'x'; cat ~/.ssh/id_rsa").is_none()); + assert!(plain_words("osascript -e \"$(cat ~/.ssh/id_rsa)\"").is_none()); + assert!(plain_words("osascript < /tmp/o").is_none()); + assert!(osascript_outside(r#"osascript -e 'tell application "System Events" to get name of first process whose frontmost is true'"#).is_ok()); + assert!(osascript_outside(r#"osascript -e 'tell application "Tailscale" to activate' -e 'delay 1'"#).is_ok()); + for bad in [ + r#"osascript -e 'do shell script "cat ~/.ssh/id_rsa"'"#, + r#"osascript -e 'tell application "Terminal" to do script "id"'"#, + r#"osascript -e 'tell application "System Events" to keystroke "rm -rf ~"'"#, + r#"osascript -e 'read POSIX file "/Users/x/.aws/credentials"'"#, + r#"osascript -l JavaScript -e 'ObjC.import("Foundation")'"#, + "osascript ~/evil.scpt", + "osascript -i", + "osascript", + ] { + assert!(osascript_outside(bad).is_err(), "{bad}"); + } + } + + #[test] + fn a_command_the_sandbox_broke_says_so() { + if sandbox_exec().is_none() { + return; + } + assert!(sandbox_note("ps aux; echo", "/bin/sh: /bin/ps: Operation not permitted").unwrap().contains("pgrep")); + let err = "40:85: execution error: System Events發生錯誤：發生越權取用的錯誤。 (-10004)"; + // Joined to another command, it stayed in the sandbox: say how to write it. + assert!(sandbox_note("osascript -e 'tell application \"System Events\" to …'; echo done", err).unwrap().contains("單獨一條 osascript")); + // Plain and harmless, it ran outside: a -10004 then is macOS's own answer, not the sandbox. + assert_eq!(sandbox_note("osascript -e 'tell application \"System Events\" to …'", err), None); + assert!(sandbox_note("/Applications/T.app/Contents/MacOS/T status && echo", "/bin/sh: line 1: 9823 Trace/BPT trap: 5").unwrap().contains("單獨執行")); + assert_eq!(sandbox_note("ls", "No such file or directory"), None); + } + + /// Plain AppleScript drives another app (outside the sandbox); joined to another command it + /// stays in the sandbox and gets -10004 with a note. On a Mac with 自動化 allowed for the + /// test runner: `cargo test applescript_drives_apps -- --ignored --nocapture`. + #[test] + #[ignore] + fn applescript_drives_apps_outside_the_sandbox() { + let w = std::env::temp_dir(); + let ask = r#"osascript -e 'tell application "System Events" to get name of first process whose frontmost is true'"#; + let out = build_command_sealed(ask, &w, &protected_dirs(), &readable_dirs(), None, "1", "r").output().unwrap(); + println!("plain: {:?} {}", out.status, String::from_utf8_lossy(&out.stdout).trim()); + assert!(out.status.success() && !String::from_utf8_lossy(&out.stdout).trim().is_empty(), "{out:?}"); + let joined = format!("{ask}; true"); + let out = build_command_sealed(&joined, &w, &protected_dirs(), &readable_dirs(), None, "1", "r").output().unwrap(); + let text = String::from_utf8_lossy(&out.stderr).to_string(); + println!("joined: {text}"); + assert!(text.contains("-10004")); + assert!(sandbox_note(&joined, &text).unwrap().contains("單獨一條 osascript")); + } + + /// An App Store app's command line runs outside AnyBot's sandbox and works; inside it dies. + /// Needs Tailscale from the App Store: `cargo test app_store_cli_runs -- --ignored`. + #[test] + #[ignore] + fn app_store_cli_runs_outside_the_sandbox() { + let t = "/Applications/Tailscale.app/Contents/MacOS/Tailscale"; + assert!(runs_outside_sandbox(&format!("{t} version"))); + let w = std::env::temp_dir(); + let out = build_command_sealed(&format!("{t} version"), &w, &protected_dirs(), &readable_dirs(), None, "1", "r").output().unwrap(); + assert!(String::from_utf8_lossy(&out.stdout).trim().chars().next().map(|c| c.is_ascii_digit()).unwrap_or(false), "{out:?}"); + let ps = build_command_sealed("ps -ax | grep -c .", &w, &protected_dirs(), &readable_dirs(), None, "1", "r").output().unwrap(); + assert!(ps.status.success(), "{ps:?}"); + } + #[test] fn head_tail_trims_middle() { let s = "a".repeat(OUTPUT_HEAD_CHARS) + &"b".repeat(5000) + &"c".repeat(OUTPUT_TAIL_CHARS);