From 5656d355af0ef20b4364cee9e945f1bf46d75b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=80=A7=E9=A9=8A?= Date: Fri, 6 Nov 2026 00:47:01 +0800 Subject: [PATCH] Bots click by number: every control on the screen is boxed and numbered, on every computer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates from a picture, and the web tools' description told it they didn't work on its own computer. - marks.rs: the picture a bot gets has every button, link, field and menu item boxed and numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click / typing / scroll / mouse_move take the number (ref) and land on the control's centre. - Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets no number (topmost window at its centre must be the app's; the Dock's invisible full-screen window doesn't count). open_app waits until the app is really in front. - No going round in circles: a step that changed nothing says so; the same step (clicks within 16 px) a third time in a turn comes with a warning, from the fourth time it isn't run. - The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP, renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look (list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a minute (files and logins stay); a new box is never made from an older image. - Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt says web work goes through browser_* by number, and never to read browser cookie or history files. Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28, 153 unit tests; Linux build + 157 tests + x11_live. Co-Authored-By: Claude Opus 5.5 --- FEATURES.md | 14 ++- box/Dockerfile | 16 ++- box/a11y | 154 +++++++++++++++++++++++++ box/browser | 35 ++++++ box/menu | 2 +- box/start.sh | 41 ++++--- src/agent.rs | 86 +++++++++++--- src/app.rs | 6 +- src/cloudbox.rs | 273 +++++++++++++++++++++++++++++++++++++-------- src/computer.rs | 290 +++++++++++++++++++++++++++--------------------- src/desk.rs | 11 -- src/desktop.rs | 195 ++++++++------------------------ src/main.rs | 9 +- src/marks.rs | 269 ++++++++++++++++++++++++++++++++++++++++++++ src/remote.rs | 12 +- src/web.rs | 2 +- src/x11.rs | 13 --- 17 files changed, 1025 insertions(+), 403 deletions(-) create mode 100644 box/a11y create mode 100644 box/browser create mode 100644 src/marks.rs diff --git a/FEATURES.md b/FEATURES.md index 84572a0..6056a1a 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -1045,19 +1045,21 @@ AnyBot 是一個桌面聊天 App，介面和運作方式照 Grok Bot 重做。 ## 24. 操作螢幕（滑鼠、鍵盤、截圖） - 有螢幕的電腦：Bot 自己的電腦（Linux 桌面，Docker）、這台電腦、帳號裡其他開了「裝置連結」的電腦。每個螢幕工具都有 `computer` 參數；沒指定時用 Bot 的預設電腦。 -- **動手只有一種方式：真的滑鼠和鍵盤。** Mac 用 CoreGraphics，Linux 用 X11／Wayland（desk.rs），Bot 的電腦用 xdotool。看畫面有兩種：截圖，或用「讀 App 畫面」拿到按鈕清單（e 編號）。用編號點或打字時，AnyBot 會換成那個按鈕在螢幕上的位置，用同一套滑鼠鍵盤去點（以前編號走 agent-desktop 自己的一套動作，按鍵名稱和失敗方式都不一樣，在終端機打不了字、`/` 和 `` ` `` 會出錯，已拆掉）。 -- **每個動作做完自動附上新畫面**（點、打字、按鍵、捲動、拖曳、移動滑鼠、打開 App；先等 0.5 秒讓畫面穩定），Bot 不用再花一步截圖確認。拍不到畫面時（Mac 還沒開「螢幕與系統錄音」），改附最前面 App 的按鈕清單。按住滑鼠鍵（拖到一半）不附。 -- 在 Bot 的電腦上：一個動作只開一次 `docker exec`（動作和之後的畫面在同一個指令裡），不用問你；英文一個鍵一個鍵打，中文和表情符號用剪貼簿貼上（終端機用 Shift+Insert，其他程式用 Ctrl+V），所以任何程式都打得進去；⌘ 當成 Ctrl。不用 `xdotool mousemove --sync`：滑鼠已經在那個位置時它會一直等（同一點點兩次會卡兩分鐘）。 +- **看：截圖上每個能點的東西都框起來、標了編號。** Bot 拿到的截圖上，每個按鈕、連結、輸入框、選單項目都有編號，另外附同樣編號的清單（例如「[21] 連結「Profile」」）。編號來自系統的無障礙資訊（Mac 的 AX、Linux 的 AT-SPI；Bot 的電腦裡網頁內容也有，Chromium 開了無障礙），所以網頁上的小圖示也有編號。只有被別的視窗擋住的按鈕不給編號（Mac 上逐一檢查最上層的視窗是不是同一個 App），免得點到別的 App。 +- **動手：照編號點，只有一種方式（真的滑鼠和鍵盤）。** 點編號就是點那個東西的正中央；圖上沒編號的東西才用 x、y。Mac 用 CoreGraphics，Linux 用 X11／Wayland（desk.rs），Bot 的電腦用 xdotool。（以前 Bot 看圖猜座標，曾把一塊空白當成選單項目連點十次。） +- **每個動作做完自動附上新的編號截圖**（點、打字、按鍵、捲動、拖曳、移動滑鼠、打開 App；先等 0.5 秒讓畫面穩定），Bot 不用再花一步截圖確認。拍不到畫面時（Mac 還沒開「螢幕與系統錄音」），改附最前面 App 的清單（含文字）。 +- **不鬼打牆**：動作後畫面和上一步完全一樣，結果會寫「畫面和上一步完全一樣：這一步沒有點到東西」，並叫 Bot 換編號、換方法或請人幫忙。同一步（點擊座標相差不到 16 像素也算）在一個回合裡做第 3 次會加警告，第 4 次起不執行，直接叫它換方法。 +- 在 Bot 的電腦上：一個動作只開一次 `docker exec`（動作、按鈕清單和之後的畫面在同一個指令裡，約 0.4 秒）。桌面是一個 TigerVNC（畫面和 VNC 合一，取代 Xvfb＋x11vnc），開機就啟動無障礙服務（AT-SPI），瀏覽器一律由 `anybot-browser` 開（同一個設定檔和 CDP、網頁開放無障礙、同時開兩個不會搶設定檔；做法取自 XBot 的 box）。這台電腦比 AnyBot 舊時（第 4 版起），大腦會在沒人用一分鐘後自己更新它（檔案和登入保留），不用問你；英文一個鍵一個鍵打，中文和表情符號用剪貼簿貼上（終端機用 Shift+Insert，其他程式用 Ctrl+V），所以任何程式都打得進去；⌘ 當成 Ctrl。不用 `xdotool mousemove --sync`：滑鼠已經在那個位置時它會一直等（同一點點兩次會卡兩分鐘）。 - 需要看得懂圖片的模型；看不懂圖片時用按鈕清單。 ### 原版的九個工具 computer_use_screenshot、click、mouse_move、mouse_button（按住／放開）、drag、scroll、typing、press_key、check_permissions。另外兩個： -- **「讀 App 畫面」（computer_use_read_app）**：App 視窗的按鈕、欄位、選單、清單和文字，每個控制項一個編號（e1、e2…），同時記下它在螢幕上的位置。很長的內容（終端機的整段捲動紀錄）留開頭 160 字和結尾 400 字，提示字和最新輸出在結尾。Bot 的電腦（Linux 桌面）沒有這個，會說請用截圖。 +- **（已併入截圖）「讀 App 畫面」**：按鈕清單現在就在每張截圖裡，不再是另一個工具。很長的內容（終端機的整段捲動紀錄）留開頭 160 字和結尾 400 字。 - **「打開 App」（computer_use_open_app）**：打開或叫到最前面，顯示的名字也可以（「計算機」→ Calculator）。Bot 的電腦上：chromium（瀏覽器，就是 browser_* 在用的那個，不會開第二個）、xterm（終端機）或其他程式名稱。 -- **「按住或放開滑鼠」（computer_use_mouse_button）**：action 是 press 或 release，可以先移到 x、y。用來長按或自己控制的拖曳。結果「已按住滑鼠鍵（記得之後放開）。」「已放開滑鼠鍵。」 +- **（已拿掉）「按住或放開滑鼠」**：拖曳已經涵蓋。 - **「檢查螢幕權限」（computer_use_check_permissions）**：回報「螢幕錄製：已允許／還沒允許」「輔助使用（滑鼠和鍵盤）：…」，不會跳出系統視窗。Bot 的電腦上一律可以。 ### 逐個 App 授權（原版：「Grok Bot 可以看到 {app}，並在其中點擊和輸入」） @@ -1106,7 +1108,7 @@ computer_use_screenshot、click、mouse_move、mouse_button（按住／放開） - 用截圖上的像素位置。截圖的大小只看螢幕大小（長邊 1,024），所以換算不用記「上一張截圖」：還沒截圖也能點，每個 Bot、手機看到的都是同一個尺寸。 - 例如寬 1,470 點的螢幕送出寬 1,024 的圖，圖上 (512, 333) 就是螢幕上的 (735, 478)。 -- 或用「讀 App 畫面」的編號（ref）：點那個控制項的中心。那個 App 不在最前面時先把它叫到前面，免得點到蓋在上面的視窗。 +- 或用截圖上的編號（ref）：點那個控制項的中心。那個 App 不在最前面時先把它叫到前面，免得點到蓋在上面的視窗。 - 手機網頁看螢幕時：Bot 的電腦送原尺寸畫面，這台電腦送和 Bot 一樣大小的截圖；手機點的位置照它拿到的那張畫面換算，是「你」在操作（不經過 Bot 的工具，你取得控制權時也照樣能點）。以前手機的點擊借用 Bot 上一張截圖的比例，點 Bot 的電腦會偏到右下（多 1.25 倍）或要求先截圖。 - 你自己的滑鼠和鍵盤不會被凍住。 diff --git a/box/Dockerfile b/box/Dockerfile index 562607c..af97a46 100644 --- a/box/Dockerfile +++ b/box/Dockerfile @@ -1,9 +1,12 @@ # AnyBot's own computer ("Bot 的電腦"): a Linux desktop the bots share, like the original's box -# (an X11 desktop driven by xdotool, watched through noVNC on port 6080). +# (an X11 desktop driven by xdotool, watched through noVNC on port 6080). One TigerVNC server is +# the screen and the VNC server; AT-SPI is on, so every app's controls (web pages included) can +# be read with their places and numbered on the bot's picture (box/a11y, after XBot's box). FROM debian:bookworm-slim ENV DEBIAN_FRONTEND=noninteractive LANG=C.UTF-8 TZ=UTC RUN apt-get update && apt-get install -y --no-install-recommends \ - xvfb x11vnc xdotool x11-utils x11-xserver-utils imagemagick fluxbox xterm \ + tigervnc-standalone-server tigervnc-tools xdotool x11-utils x11-xserver-utils imagemagick fluxbox xterm \ + at-spi2-core libatk-adaptor python3-gi gir1.2-atspi-2.0 libglib2.0-bin gsettings-desktop-schemas dconf-gsettings-backend \ novnc python3-websockify chromium fonts-noto-cjk fonts-noto-color-emoji \ ca-certificates curl wget git jq unzip zip less nano vim-tiny procps psmisc \ python3 python3-pip python3-venv nodejs npm sudo tini dbus-x11 xclip \ @@ -11,16 +14,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && useradd -m -s /bin/bash bot && echo 'bot ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/bot \ && ln -sf /usr/share/novnc/vnc.html /usr/share/novnc/index.html COPY start.sh /usr/local/bin/anybot-box-start +COPY a11y /usr/local/bin/anybot-a11y +COPY browser /usr/local/bin/anybot-browser COPY fluxbox-init /etc/anybot/fluxbox-init COPY overlay /etc/anybot/overlay COPY Xresources /etc/anybot/Xresources COPY menu /etc/anybot/menu -RUN chmod +x /usr/local/bin/anybot-box-start \ +RUN chmod +x /usr/local/bin/anybot-box-start /usr/local/bin/anybot-a11y /usr/local/bin/anybot-browser \ # Chromium has no sandbox inside a container (no user namespaces); the box itself is the # sandbox. Debian's chromium wrapper reads these flags for every launch (menu, bots, xdg-open). && printf '%s\n' 'export CHROMIUM_FLAGS="$CHROMIUM_FLAGS --no-sandbox --disable-dev-shm-usage --no-first-run --no-default-browser-check --disable-gpu --password-store=basic --test-type"' > /etc/chromium.d/anybot -ENV DISPLAY=:1 SCREEN_SIZE=1280x800x24 +# Accessibility on for every app, including those started over `docker exec`. +ENV DISPLAY=:1 SCREEN_SIZE=1280x800x24 GTK_MODULES=atk-bridge GTK_A11Y=atspi GNOME_ACCESSIBILITY=1 NO_AT_BRIDGE=0 EXPOSE 6080 -LABEL anybot.box.version="3" +LABEL anybot.box.version="4" ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/usr/local/bin/anybot-box-start"] diff --git a/box/a11y b/box/a11y new file mode 100644 index 0000000..7c63738 --- /dev/null +++ b/box/a11y @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""The controls on the box's desktop, from its accessibility tree (AT-SPI), as JSON: +{"elements": [{"role", "name", "value", "x", "y", "w", "h", "active", "flags", "window"}]} +in screen pixels. Web pages are in it too (Chromium runs with renderer accessibility on). +AnyBot numbers them on the bot's picture; a click by number lands on the centre. + +Adapted from XBot's box/box-a11y. Usage: a11y [--limit N] [--budget SECONDS] +Exit 2 when AT-SPI itself is unreachable. +""" +import json +import os +import sys +import time +import warnings + +warnings.filterwarnings("ignore", category=DeprecationWarning) + +# Clicked or typed into (they get a number). +ACTIVE = { + "push button", "toggle button", "radio button", "check box", "link", "menu item", "check menu item", + "radio menu item", "tab", "page tab", "list item", "tree item", "combo box", "spin button", "slider", + "entry", "password text", "menu", +} +# Listed when they carry words (headings, labels, cells...). +NAMED = {"heading", "label", "static", "text", "image", "table cell", "column header", "row header", "icon", + "tool tip", "alert", "dialog", "paragraph"} +# The desktop's own parts are visible in the picture; the list is for the windows worked in. +SKIP_APPS = {"fluxbox", "a11y", "python3", "xterm-wrapper"} + + +def main(): + limit, budget = 400, 3.0 + args = sys.argv[1:] + for i, a in enumerate(args): + if a == "--limit": + limit = int(args[i + 1]) + if a == "--budget": + budget = float(args[i + 1]) + if not os.environ.get("DBUS_SESSION_BUS_ADDRESS"): + try: + for line in open("/tmp/anybot-dbus.env", encoding="utf-8"): + k, _, v = line.strip().partition("=") + if k == "DBUS_SESSION_BUS_ADDRESS" and v: + os.environ[k] = v + except OSError: + pass + os.environ.setdefault("DISPLAY", ":1") + try: + import gi + gi.require_version("Atspi", "2.0") + from gi.repository import Atspi + Atspi.init() + desktop = Atspi.get_desktop(0) + apps = desktop.get_child_count() + except Exception as exc: # noqa: BLE001 + print(f"AT-SPI unreachable: {exc}", file=sys.stderr) + return 2 + + sw, sh = 0, 0 + try: + geo = os.popen("xdotool getdisplaygeometry 2>/dev/null").read().split() + sw, sh = int(geo[0]), int(geo[1]) + except Exception: # noqa: BLE001 + pass + deadline = time.monotonic() + budget + out = [] + + def state(node): + try: + return node.get_state_set() + except Exception: # noqa: BLE001 + return None + + def extents(node): + try: + r = node.get_component_iface().get_extents(Atspi.CoordType.SCREEN) + return int(r.x), int(r.y), int(r.width), int(r.height) + except Exception: # noqa: BLE001 + return None + + def onscreen(b): + x, y, w, h = b + if w <= 0 or h <= 0 or x + w <= 0 or y + h <= 0: + return False + return not (sw and x >= sw) and not (sh and y >= sh) + + def walk(node, depth, window): + if time.monotonic() > deadline or len(out) >= limit or depth > 45: + return + try: + role = node.get_role_name() + except Exception: # noqa: BLE001 + return + st = state(node) + if depth > 1 and (st is None or not (st.contains(Atspi.StateType.SHOWING) and st.contains(Atspi.StateType.VISIBLE))): + return + box = extents(node) + if depth > 1 and (box is None or not onscreen(box)): + return + try: + name = (node.get_name() or "").strip() + except Exception: # noqa: BLE001 + name = "" + if role in ("frame", "dialog", "window") and name: + window = name + editable = bool(st and st.contains(Atspi.StateType.EDITABLE)) + value = "" + if (role in ("entry", "password text", "text", "spin button", "slider") or editable) and role != "password text": + try: + t = node.get_text_iface() + if t is not None: + value = (t.get_text(0, min(t.get_character_count(), 200)) or "").strip() + except Exception: # noqa: BLE001 + pass + active = role in ACTIVE or (editable and role in ("text", "paragraph", "section", "document web")) + if depth > 1 and box is not None and (active or (role in NAMED and (name or value))): + flags = [] + if st is not None: + for s, word in ((Atspi.StateType.FOCUSED, "focused"), (Atspi.StateType.CHECKED, "checked"), + (Atspi.StateType.SELECTED, "selected")): + if st.contains(s): + flags.append(word) + if not st.contains(Atspi.StateType.ENABLED): + flags.append("disabled") + x, y, w, h = box + out.append({"role": role, "name": name[:120], "value": value[:200], "x": x, "y": y, "w": w, "h": h, + "active": active, "flags": flags, "window": window}) + try: + n = node.get_child_count() + except Exception: # noqa: BLE001 + return + for i in range(min(n, 2000)): + try: + c = node.get_child_at_index(i) + except Exception: # noqa: BLE001 + continue + if c is not None: + walk(c, depth + 1, window) + + for i in range(apps): + try: + app = desktop.get_child_at_index(i) + name = app.get_name() or "" + except Exception: # noqa: BLE001 + continue + if name in SKIP_APPS: + continue + walk(app, 0, name) + json.dump({"elements": out, "truncated": len(out) >= limit or time.monotonic() > deadline}, sys.stdout, ensure_ascii=False) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/box/browser b/box/browser new file mode 100644 index 0000000..b68077d --- /dev/null +++ b/box/browser @@ -0,0 +1,35 @@ +#!/bin/bash +# The one way to start the box's browser (adapted from XBot's box/box-chrome): the same profile +# and DevTools port every time, so AnyBot's browser tools and the person in the viewer use the +# same window and logins; web pages exposed to accessibility, so their links and buttons get +# numbers on the bot's picture. A second start only hands its address to the running one. +set -euo pipefail +export DISPLAY="${DISPLAY:-:1}" HOME=/home/bot +export GTK_MODULES=atk-bridge GTK_A11Y=atspi GNOME_ACCESSIBILITY=1 NO_AT_BRIDGE=0 +if [[ -z "${DBUS_SESSION_BUS_ADDRESS:-}" && -r /tmp/anybot-dbus.env ]]; then + # shellcheck disable=SC1091 + source /tmp/anybot-dbus.env + export DBUS_SESSION_BUS_ADDRESS +fi +PROFILE=/home/bot/.config/anybot-browser +mkdir -p "$PROFILE" +FLAGS=(--no-sandbox --test-type --user-data-dir="$PROFILE" --remote-debugging-address=127.0.0.1 --remote-debugging-port=9222 + --remote-allow-origins=* --force-renderer-accessibility --disable-blink-features=AutomationControlled + --disable-dev-shm-usage --disable-gpu --use-angle=swiftshader --enable-unsafe-swiftshader --password-store=basic + --no-first-run --no-default-browser-check --disable-session-crashed-bubble --hide-crash-restore-bubble + --disable-features=TranslateUI --start-maximized) +# Two starts in the same second (the bot and the person) would both find no browser and fight +# over the profile ("in use"): one at a time until the first is listening. +exec 9>/tmp/.anybot-browser.lock +flock -w 30 9 || true +if pgrep -u bot -f -- "--user-data-dir=$PROFILE" >/dev/null; then + flock -u 9 + exec chromium "${FLAGS[@]}" "$@" +fi +# A profile from before this box (a new container) still names the old one in its lock. +rm -f "$PROFILE/SingletonLock" "$PROFILE/SingletonSocket" "$PROFILE/SingletonCookie" +setsid chromium "${FLAGS[@]}" "${@:-about:blank}" >/tmp/anybot-browser.log 2>&1 < /dev/null & +for _ in $(seq 1 60); do + python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:9222/json/version', timeout=1).close()" 2>/dev/null && break + sleep 0.25 +done diff --git a/box/menu b/box/menu index b17d44e..67b915d 100644 --- a/box/menu +++ b/box/menu @@ -1,5 +1,5 @@ [begin] (Bot 的電腦) - [exec] (瀏覽器 Chromium) {chromium --no-first-run --no-default-browser-check --disable-gpu} + [exec] (瀏覽器 Chromium) {anybot-browser} [exec] (終端機) {xterm} [separator] [workspaces] (視窗) diff --git a/box/start.sh b/box/start.sh index 7e16c5a..c897f56 100644 --- a/box/start.sh +++ b/box/start.sh @@ -7,22 +7,38 @@ cp -n /etc/anybot/Xresources /home/bot/.Xresources 2>/dev/null cp /etc/anybot/menu /home/bot/.fluxbox/menu 2>/dev/null chown -R bot:bot /home/bot rm -f /tmp/.X1-lock /tmp/.X11-unix/X1 -Xvfb :1 -screen 0 "${SCREEN_SIZE}" -nolisten tcp & -XVFB_PID=$! -for i in $(seq 1 50); do xdpyinfo -display :1 >/dev/null 2>&1 && break; sleep 0.1; done -su bot -c "DISPLAY=:1 xrdb -nocpp -merge /home/bot/.Xresources" -su bot -c "DISPLAY=:1 fluxbox >/tmp/fluxbox.log 2>&1 &" -(sleep 1.5; xsetroot -display :1 -solid '#2b2f36') & -xsetroot -display :1 -solid '#2b2f36' -# VNC login key: the app starts the box with VNC_PASSWORD and opens the viewer with it. -# Without it (an older app) the desktop stays keyless as before. +# One TigerVNC server is the screen and the VNC server (was Xvfb + x11vnc, which polls the +# screen for changes). VNC only on 127.0.0.1 inside the box, with the box's VNC password when the +# app gave one (VNC_PASSWORD), as before. +GEOM="${SCREEN_SIZE%x*}" if [ -n "${VNC_PASSWORD:-}" ]; then mkdir -p /home/bot/.vnc - printf '%s' "$VNC_PASSWORD" | x11vnc -storepasswd /home/bot/.vnc/pass >/dev/null 2>&1 - x11vnc -display :1 -forever -shared -quiet -rfbauth /home/bot/.vnc/pass -rfbport 5900 -localhost >/tmp/x11vnc.log 2>&1 & + printf '%s\n' "$VNC_PASSWORD" | vncpasswd -f > /home/bot/.vnc/passwd && chmod 600 /home/bot/.vnc/passwd + VNC_SEC="-SecurityTypes VncAuth -rfbauth /home/bot/.vnc/passwd" else - x11vnc -display :1 -forever -shared -nopw -quiet -rfbport 5900 -localhost >/tmp/x11vnc.log 2>&1 & + VNC_SEC="-SecurityTypes None" fi +Xtigervnc :1 -geometry "$GEOM" -depth 24 -rfbport 5900 -localhost $VNC_SEC -AlwaysShared -AcceptCutText -SendCutText \ + -UseBlacklist 0 -IdleTimeout 0 -MaxIdleTime 0 -MaxDisconnectionTime 0 -MaxConnectionTime 0 -s 0 -dpms -desktop "Bot 的電腦" -ac -noreset \ + >/tmp/xvnc.log 2>&1 & +XVFB_PID=$! +for i in $(seq 1 50); do xdpyinfo -display :1 >/dev/null 2>&1 && break; sleep 0.1; done +# The session bus and the accessibility registry; tools started later over `docker exec` +# (anybot-a11y, anybot-browser) join this bus through /tmp/anybot-dbus.env. +su bot -c 'cd /home/bot && dbus-launch --sh-syntax > /tmp/anybot-dbus.sh' && . /tmp/anybot-dbus.sh +printf 'DBUS_SESSION_BUS_ADDRESS=%s\n' "$DBUS_SESSION_BUS_ADDRESS" > /tmp/anybot-dbus.env && chmod 644 /tmp/anybot-dbus.env +export DBUS_SESSION_BUS_ADDRESS +for c in /usr/libexec/at-spi-bus-launcher /usr/lib/at-spi2-core/at-spi-bus-launcher; do + [ -x "$c" ] && { su bot -c "DISPLAY=:1 DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS $c --launch-immediately >/tmp/atspi-bus.log 2>&1 &"; break; } +done +for c in /usr/libexec/at-spi2-registryd /usr/lib/at-spi2-core/at-spi2-registryd; do + [ -x "$c" ] && { su bot -c "DISPLAY=:1 DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS $c >/tmp/atspi-registry.log 2>&1 &"; break; } +done +su bot -c "DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS gsettings set org.gnome.desktop.interface toolkit-accessibility true" 2>/dev/null || true +su bot -c "DISPLAY=:1 DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS xrdb -nocpp -merge /home/bot/.Xresources" +su bot -c "DISPLAY=:1 DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS fluxbox >/tmp/fluxbox.log 2>&1 &" +(sleep 1.5; xsetroot -display :1 -solid '#2b2f36') & +xsetroot -display :1 -solid '#2b2f36' websockify --web /usr/share/novnc 6080 localhost:5900 >/tmp/novnc.log 2>&1 & WEBSOCKIFY_PID=$! # Say ready only once the viewer's port listens: the app opens noVNC on this flag, and a too @@ -38,6 +54,5 @@ WEBSOCKIFY_PID=$! # with a dead X server, and every screen action failed forever). wait $XVFB_PID kill $WEBSOCKIFY_PID 2>/dev/null -pkill -f x11vnc 2>/dev/null pkill -u bot fluxbox 2>/dev/null exit 1 diff --git a/src/agent.rs b/src/agent.rs index 122d8b5..d34d826 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -236,12 +236,30 @@ fn keep_to_chosen(job: &Job, history: &[Message], args: &str) -> (String, Option /// What element `e7` is, from the latest app list in the chat (「按鈕「7」」). fn element_named(history: &[Message], r: &str) -> Option { - let tag = format!("{r} "); + let tag = format!("[{}] ", crate::marks::number_of(r)?); history.iter().rev().filter(|m| m.role == Role::Tool).take(12).find_map(|m| { m.text.lines().find_map(|l| l.trim_start().strip_prefix(&tag).map(|rest| rest.trim().to_string())).filter(|e| !e.is_empty()) }) } +/// What makes two screen or browser steps "the same" (None for other tools): the tool, the +/// computer and the target, a click's spot on a 16-pixel grid (an inch beside an empty strip is +/// the same miss). +fn step_key(name: &str, args: &str) -> Option { + if !(computer::is_computer_tool(name) || web::is_web_tool(name)) || matches!(name, "computer_use_screenshot" | "browser_read" | "browser_screenshot" | "browser_wait") { + return None; + } + let mut v = tools::args_of(args); + if let Some(o) = v.as_object_mut() { + for k in ["x", "y"] { + if let Some(n) = o.get(k).and_then(serde_json::Value::as_f64) { + o.insert(k.into(), serde_json::Value::from((n / 16.0).round() as i64)); + } + } + } + Some(format!("{name} {v}")) +} + /// A step's result, with a word first when the step was moved back to the chosen computer. fn noted(moved: &std::collections::HashMap, id: &str, text: &str) -> String { match moved.get(id) { @@ -417,6 +435,8 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< let mut grants = job.grants.clone(); // Turned on by an "always" answer mid-turn (the window saves the setting too). let mut tools_on = job.tools_enabled; + // The screen and browser steps of this turn (to stop the same step being tried over and over). + let mut tried: Vec = Vec::new(); let mut local_on = job.local_execution; maybe_compact(&job, &mut history, COMPACT_THRESHOLD_PERCENT, &cancel, &send); for step in 0..MAX_STEPS { @@ -469,6 +489,25 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< } // (Each call's computer was already kept to the chosen one, see keep_to_chosen.) for call in calls { + // 鬼打牆: the same screen or browser step again and again (a click on the same + // empty spot ten times). The third time it runs with a warning; after that it isn't + // run at all, and the bot is told to change course. + let mut repeat_note = String::new(); + if let Some(key) = step_key(&call.name, &call.args) { + let times = tried.iter().rev().take(10).filter(|k| **k == key).count() + 1; + tried.push(key); + if times >= 4 { + let text = format!("沒有執行：這一步你已經試了 {times} 次，畫面都沒有變成你要的樣子。不要再試同一步。改用別的方法：照截圖上的編號點、網頁改用 browser_read 拿連結直接 browser_open，或用 hand_over_computer 請使用者幫忙。"); + send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Failed)); + let result = Message::tool_result(&call.id, &call.name, &text); + send(StreamEvent::ToolResult(result.clone())); + history.push(result); + continue; + } + if times == 3 { + repeat_note = "⚠️ 這一步你已經做了 3 次。如果畫面還不是你要的，下一步一定要換方法，不要再做同一步。\n".into(); + } + } if appctl::is_app_tool(&call.name) { // App actions run on the window's side, which also asks for deletions. send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running)); @@ -485,7 +524,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< } }; send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed })); - let result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &output)); + let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); send(StreamEvent::ToolResult(result.clone())); history.push(result); continue; @@ -495,7 +534,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running)); let (output, ok, img) = crate::teach::view_step(&job.demos, &call.args); send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed })); - let mut result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &output)); + let mut result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); result.images.extend(img); send(StreamEvent::ToolResult(result.clone())); history.push(result); @@ -518,7 +557,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< // Which computer's browser it was, like every other step. let output = if on.is_empty() { output } else { format!("[{on}] {output}") }; send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed })); - let mut result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &output)); + let mut result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); result.images.extend(img); send(StreamEvent::ToolResult(result.clone())); history.push(result); @@ -580,7 +619,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< } }; send(StreamEvent::ToolStatus(call.id.clone(), status)); - let result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &output)); + let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); send(StreamEvent::ToolResult(result.clone())); history.push(result); continue; @@ -630,7 +669,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< Err(e) => (e, false), }; send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed })); - let result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &output)); + let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); send(StreamEvent::ToolResult(result.clone())); history.push(result); continue; @@ -665,7 +704,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< (_, false) => ("AnyBot 在伺服器上，檔案沒辦法直接在使用者的螢幕打開。改用 send_file 把檔案傳到聊天裡，使用者點一下就能看。".to_string(), false), }; send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed })); - let result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &output)); + let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); send(StreamEvent::ToolResult(result.clone())); history.push(result); continue; @@ -688,7 +727,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< Err(e) => (e, false), }; send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed })); - let result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &output)); + let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); send(StreamEvent::ToolResult(result.clone())); history.push(result); continue; @@ -711,7 +750,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< } }; send(StreamEvent::ToolStatus(call.id.clone(), status)); - let result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &output)); + let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); send(StreamEvent::ToolResult(result.clone())); history.push(result); continue; @@ -755,7 +794,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< } }; send(StreamEvent::ToolStatus(call.id.clone(), status)); - let result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &output)); + let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); send(StreamEvent::ToolResult(result.clone())); history.push(result); continue; @@ -766,7 +805,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< if call.name == crate::vault::LIST_TOOL { let output = crate::vault::list_for_bot(&call.args); send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Done)); - let result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &output)); + let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); send(StreamEvent::ToolResult(result.clone())); history.push(result); continue; @@ -824,7 +863,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< } }; send(StreamEvent::ToolStatus(call.id.clone(), status)); - let result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &output)); + let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); send(StreamEvent::ToolResult(result.clone())); history.push(result); continue; @@ -859,7 +898,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< ("使用者拒絕了這個操作。請不要再試同樣的操作，改問使用者想怎麼做。".to_string(), ToolStatus::Denied, None) }; send(StreamEvent::ToolStatus(call.id.clone(), status)); - let mut result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &output)); + let mut result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); result.images.extend(img); send(StreamEvent::ToolResult(result.clone())); history.push(result); @@ -1095,7 +1134,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< (Err(x), _) => x, }; send(StreamEvent::ToolStatus(call.id.clone(), status)); - let mut result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &output)); + let mut result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output))); result.images.extend(image); send(StreamEvent::ToolResult(result.clone())); history.push(result); @@ -1156,12 +1195,25 @@ mod tests { assert!(!mentions("在 macOS 的登入機制下，幫 Bot 的電腦截圖", &mac, &roles)); } + #[test] + fn the_same_miss_counts_as_the_same_step() { + // Clicks a few pixels apart on the same strip are one step; another computer or target isn't. + assert_eq!(step_key("computer_use_click", r#"{"x":70,"y":355,"computer":"M"}"#), step_key("computer_use_click", r#"{"x":66,"y":358,"computer":"M"}"#)); + assert_ne!(step_key("computer_use_click", r#"{"x":70,"y":355}"#), step_key("computer_use_click", r#"{"x":70,"y":290}"#)); + assert_ne!(step_key("computer_use_click", r#"{"ref":"3"}"#), step_key("computer_use_click", r#"{"ref":"4"}"#)); + // Looking is never a repeat; other tools aren't counted. + assert_eq!(step_key("computer_use_screenshot", "{}"), None); + assert_eq!(step_key("shell", r#"{"command":"ls"}"#), None); + assert!(step_key("browser_click", r#"{"ref":"5"}"#).is_some()); + } + #[test] fn a_click_by_number_is_named_in_the_question() { - let list = Message::tool_result("c1", "computer_use_read_app", "[王小明的Mac mini] 「計算機」的畫面：\n 文字「0」\n e4 按鈕「全部清除」\n e7 按鈕「7」"); + let list = Message::tool_result("c1", "computer_use_screenshot", "[王小明的Mac mini] 螢幕截圖（1024×576）。\n「計算機」上可以操作的東西（圖上同樣的編號，用 ref 點）：\n 文字「0」\n[4] 按鈕「全部清除」\n[7] 按鈕「7」"); + assert_eq!(element_named(&[list.clone()], "7").as_deref(), Some("按鈕「7」")); assert_eq!(element_named(&[list.clone()], "e7").as_deref(), Some("按鈕「7」")); - assert_eq!(element_named(&[list], "e9"), None); - assert_eq!(computer::describe("computer_use_click", r#"{"ref":"e7"}"#), "點 e7"); + assert_eq!(element_named(&[list], "9"), None); + assert_eq!(computer::describe("computer_use_click", r#"{"ref":"7"}"#), "點 #7"); assert_eq!(computer::describe("computer_use_click", r#"{"x":10,"y":20}"#), "點 (10, 20)"); } diff --git a/src/app.rs b/src/app.rs index ad52057..009b211 100644 --- a/src/app.rs +++ b/src/app.rs @@ -4026,12 +4026,12 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu 你也可以上網和用使用者的瀏覽器：\n\ - 公開網頁：web_fetch。\n\ - 要登入才看得到的（Threads、Instagram、Facebook、Gmail、銀行…）：用 browser_open 在 AnyBot 自己的瀏覽器視窗打開，再用 browser_read 讀分頁內容。例如看 Threads 帳號：browser_open(\"https://www.threads.net\") 然後 browser_read。\n\ - - 在網頁上做事（後台、報表、表單）要像人一樣操作：先 browser_screenshot 看一眼（圖上每個可點的東西都有編號），再用 browser_click／browser_type／browser_select／browser_press 照編號操作；表格用 browser_table 讀，找字用 browser_find，要等載入用 browser_wait，匯出報表用 browser_download。不要猜網址；找不到想要的功能（例如排序、篩選），就看欄位標題、篩選器、下拉選單或分頁標籤，還是沒有就照頁面現有的資料自己整理，並說明網站沒有這個功能。 - - 絕對不要直接呼叫網站的內部 API、下載網站的程式碼來研究，或用程式大量抓資料：網站會把帳號當成機器人（例如跳出拼圖驗證、封鎖帳號）。 + - 網頁上的事（任何電腦、任何網站）一律用 browser_* 工具：先 browser_read（或 browser_screenshot）拿到每個連結、按鈕、欄位的編號和網址，再用 browser_click／browser_type／browser_select／browser_press 照編號操作；要去的頁面有網址就直接 browser_open。網頁上絕對不要看截圖猜座標點。表格用 browser_table 讀，找字用 browser_find，要等載入用 browser_wait，匯出報表用 browser_download。不要猜網址；找不到想要的功能（例如排序、篩選），就看欄位標題、篩選器、下拉選單或分頁標籤，還是沒有就照頁面現有的資料自己整理，並說明網站沒有這個功能。 + - 絕對不要直接呼叫網站的內部 API、下載網站的程式碼來研究，或用程式大量抓資料：網站會把帳號當成機器人（例如跳出拼圖驗證、封鎖帳號）。也絕對不要用指令去讀瀏覽器的 Cookie、瀏覽紀錄、密碼或設定檔資料庫。 - 需要使用者親手做的步驟（登入、輸入密碼、驗證碼、拼圖、簡訊或 email 驗證、確認付款）：先在電腦上把那一頁打開、點到要輸入的地方，再呼叫 hand_over_computer，instruction 寫清楚要他做什麼。聊天裡會出現那台電腦的即時畫面，他點「接管」做完後按「我完成了」，你再截圖確認、接著做。不要只用文字叫他自己去找視窗；也絕對不要替他打密碼。 - 如果讀到的是登入頁：把登入頁開在那台電腦上，用 hand_over_computer 請使用者登入一次（之後會記住），他按「我完成了」再讀。絕對不要叫使用者去改瀏覽器的設定或選單（例如「允許來自 Apple 事件的 JavaScript」），那不需要。也不要說要從 Chrome 複製登入或 Cookie，沒有這個功能。\n\ - AnyBot 瀏覽器有設定檔（profile），每個有自己的登入。使用者說要換 profile／設定檔／另一個帳號時，用 browser_profiles：list 看有哪些，use 切換，add 新增（新的要登入一次）。也可以在 browser_open / browser_read 帶 profile 參數。使用者想用自己平常的 Chrome（自己的 Chrome 設定檔、已經登入的帳號）時，用 profile「你的瀏覽器」：請使用者在 Chrome 切到想要的設定檔，再 browser_read。\n\ - - 你也能像人一樣操作螢幕（{screens}），用在檔案、指令和網頁工具做不到的事（例如操作其他 App）。動手只有一種方式：真的滑鼠和鍵盤。開始時用 computer_use_screenshot 看一次；之後每個動作（click、typing、press_key、scroll、drag、open_app）做完都會直接附上新的畫面，不要再另外截圖確認。位置用最新畫面上的像素 x、y；或用 computer_use_read_app 讀出 App 的按鈕清單，用 e 編號（ref）點和打字，拍不到畫面時也能用。要在某個 App 裡打字：先 computer_use_open_app 把它放到最前面，再 computer_use_typing（submit=true 會順便按 return）。網頁上的事用 browser_* 工具，比點螢幕準。\n\ + - 你也能像人一樣操作螢幕（{screens}），用在檔案、指令和網頁工具做不到的事（例如操作其他 App）。先 computer_use_screenshot：你會拿到一張每個按鈕、連結、欄位、選單項目都框起來、標了編號的截圖，和同樣編號的清單。一律照編號操作（click／typing／scroll 的 ref，例如 ref 填 12），點下去就是那個東西的正中央；只有圖上沒有編號的東西（圖片裡的一個點、遊戲）才用 x、y。每個動作做完都會附上新的編號截圖，不要另外截圖。結果說「畫面沒有變化」就是沒點到：換一個編號或換方法，絕對不要重複同一步。要在某個 App 裡打字：先 computer_use_open_app，再 computer_use_typing（ref 是輸入框的編號；submit=true 順便按 return）\n\ {}\ - 不要說「我沒有瀏覽器或工具」：先試這些工具。需要權限時 App 會自己跳出視窗問使用者；工具回報要開什麼設定時，照實轉告。", now.format("%Y-%m-%d %H:%M"), diff --git a/src/cloudbox.rs b/src/cloudbox.rs index 8d02ce1..3164f0f 100644 --- a/src/cloudbox.rs +++ b/src/cloudbox.rs @@ -96,6 +96,8 @@ const FLUXBOX_INIT: &str = include_str!("../box/fluxbox-init"); const OVERLAY: &str = include_str!("../box/overlay"); const XRESOURCES: &str = include_str!("../box/Xresources"); const MENU: &str = include_str!("../box/menu"); +const A11Y: &str = include_str!("../box/a11y"); +const BROWSER: &str = include_str!("../box/browser"); // ---------- state ---------- @@ -555,8 +557,10 @@ pub fn available() -> bool { // ---------- image ---------- +/// The box image is here and is the version this AnyBot carries (an old one would make a new +/// box without the newer parts: the numbered controls, the browser launcher). fn image_exists() -> bool { - dk(&["image", "inspect", IMAGE]).is_ok() + dk(&["image", "inspect", "-f", "{{index .Config.Labels \"anybot.box.version\"}}", IMAGE]).map(|v| v.trim() == wanted_version()).unwrap_or(false) } /// Build the box image from the files baked into AnyBot. @@ -564,7 +568,7 @@ fn build_image(pull: bool) -> Result<(), String> { let dir = std::env::temp_dir().join(format!("anybot-box-build-{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; - for (name, body) in [("Dockerfile", DOCKERFILE), ("start.sh", START_SH), ("fluxbox-init", FLUXBOX_INIT), ("overlay", OVERLAY), ("Xresources", XRESOURCES), ("menu", MENU)] { + for (name, body) in [("Dockerfile", DOCKERFILE), ("start.sh", START_SH), ("fluxbox-init", FLUXBOX_INIT), ("overlay", OVERLAY), ("Xresources", XRESOURCES), ("menu", MENU), ("a11y", A11Y), ("browser", BROWSER)] { std::fs::write(dir.join(name), body).map_err(|e| e.to_string())?; } log("正在建立電腦的映像檔（第一次要幾分鐘）…"); @@ -680,8 +684,9 @@ pub fn browser_port() -> Result { if box_cdp_alive(port) { return Ok(port); } - let dbg = format!("--remote-debugging-port={BOX_CDP_PORT}"); - dk(&["exec", "-d", "-u", "bot", "-e", "DISPLAY=:1", "-e", "HOME=/home/bot", container(), "chromium", &dbg, "--remote-allow-origins=*", "--user-data-dir=/home/bot/.config/anybot-browser", "about:blank"]).map_err(|e| format!("Bot 的電腦打不開瀏覽器：{e}"))?; + // The box's one browser launcher (same profile and port as the person's menu entry, web + // pages exposed to accessibility). + dk_timeout(&["exec", "-u", "bot", container(), "anybot-browser"], 30_000).map_err(|e| format!("Bot 的電腦打不開瀏覽器：{e}"))?; let start = Instant::now(); while start.elapsed() < Duration::from_secs(20) { if box_cdp_alive(port) { @@ -777,11 +782,34 @@ pub fn start_background() { } refresh(); maybe_scheduled_update(); + update_when_idle(idle); } } }); } +/// The brain's box is older than this AnyBot (it lacks parts the bots now use: the numbered +/// controls, the browser launcher): update it by itself once nobody has used it for a minute, +/// once per version. Files and logins stay (they live in the home volume). +fn update_when_idle(idle_secs: u64) { + static TRIED: Mutex = Mutex::new(String::new()); + if !IS_BRAIN.load(Ordering::SeqCst) || idle_secs < 60 || is_busy() || HELD_BY_PERSON.load(Ordering::SeqCst) || viewer_open() || !update_available() { + return; + } + let want = wanted_version(); + { + let mut t = TRIED.lock().unwrap_or_else(|e| e.into_inner()); + if *t == want { + return; + } + *t = want.clone(); + } + log(&format!("Bot 的電腦是舊版，自動更新到第 {want} 版（檔案和登入都會保留）。")); + if let Err(e) = rebuild(Rebuild::Update) { + log(&format!("自動更新沒有成功：{e}")); + } +} + /// This AnyBot is the brain (its box is the bots' computer). A computer showing another brain /// doesn't use its own box, so it isn't kept awake there. static IS_BRAIN: AtomicBool = AtomicBool::new(true); @@ -1198,14 +1226,6 @@ pub fn copy_out(path_in_box: &str) -> Result { // Docker host elsewhere; a click with a modifier used to be four of them, plus one more for the // screenshot the bot then had to ask for. -/// The desktop as an image (full size). -pub fn grab() -> Result { - let png = dk_timeout(&["exec", container(), "import", "-window", "root", "-display", DISPLAY, "png:-"], 10_000)?; - let img = image::load_from_memory(&png).map(|i| i.to_rgb8()).map_err(|e| format!("截圖讀不出來：{e}"))?; - shared().lock().unwrap_or_else(|e| e.into_inner()).size = Some(img.dimensions()); - Ok(img) -} - /// The latest picture of the box's screen, small (for the 電腦 card). While the box runs it is /// taken now (and kept on disk); while it sleeps the last one taken is shown. pub fn thumbnail(width: u32) -> Option { @@ -1258,14 +1278,27 @@ fn box_scale() -> Result<(f64, f64), String> { /// times `scale` (desktop pixels per picture pixel; 1 for a person's viewer, which shows the /// desktop at full size). pub fn box_script(name: &str, v: &Value, scale: (f64, f64)) -> Result<(String, Option, String), String> { + // A control's number from the latest picture: its centre on the desktop. + let by_number = |p: &Value| -> Option> { + let r = match &p["ref"] { + Value::String(r) if !r.trim().is_empty() => r.trim().to_string(), + Value::Number(n) => n.to_string(), + _ => return None, + }; + let spots = BOX_SPOTS.lock().unwrap_or_else(|e| e.into_inner()).clone(); + Some(spots.ok_or_else(|| "先用 computer_use_screenshot 看一下畫面，再照圖上的編號操作。".to_string()).and_then(|s| s.get(&r)).map(|((x, y), what)| ((x.round() as i64, y.round() as i64), format!("#{}（{what}）", crate::marks::number_of(&r).unwrap_or(0))))) + }; let pt = |p: &Value| -> Result<(i64, i64), String> { - let (Some(x), Some(y)) = (p["x"].as_f64(), p["y"].as_f64()) else { return Err("請給 x 和 y（截圖上的位置）".into()) }; + if let Some(r) = by_number(p) { + return r.map(|(xy, _)| xy); + } + let (Some(x), Some(y)) = (p["x"].as_f64(), p["y"].as_f64()) else { return Err("請給編號 ref（截圖上的數字），或 x 和 y（截圖上的位置）".into()) }; Ok(((x * scale.0).round() as i64, (y * scale.1).round() as i64)) }; - let at = |p: &Value| format!("({}, {})", p["x"].as_f64().unwrap_or(0.0), p["y"].as_f64().unwrap_or(0.0)); - if v["ref"].as_str().map(|r| !r.trim().is_empty()).unwrap_or(false) { - return Err("這台是 Linux 桌面，沒有 App 的編號（ref）：請用截圖上的 x、y。".into()); - } + let at = |p: &Value| match by_number(p) { + Some(Ok((_, what))) => what, + _ => format!("({}, {})", p["x"].as_f64().unwrap_or(0.0), p["y"].as_f64().unwrap_or(0.0)), + }; let mods = mods_of(v); let down: String = mods.iter().map(|m| format!(" keydown {m}")).collect(); let up: String = mods.iter().rev().map(|m| format!(" keyup {m}")).collect(); @@ -1288,11 +1321,6 @@ pub fn box_script(name: &str, v: &Value, scale: (f64, f64)) -> Result<(String, O (wrap(format!("{} click --repeat {n} --delay {CLICK_REPEAT_MS} {button}", mv(pt(v)?))), None, format!("已{}{} {}。", if button == "3" { "右鍵" } else { "" }, if n >= 2 { format!("連點 {n} 下") } else { "點".into() }, at(v))) } "computer_use_mouse_move" => (format!("xdotool {}", mv(pt(v)?)), None, format!("已把滑鼠移到 {}。", at(v))), - "computer_use_mouse_button" => { - let go = if v["x"].is_null() { String::new() } else { format!("{} ", mv(pt(v)?)) }; - let release = v["action"].as_str() == Some("release"); - (format!("xdotool {go}{} {button}", if release { "mouseup" } else { "mousedown" }), None, (if release { "已放開滑鼠鍵。" } else { "已按住滑鼠鍵（記得之後放開）。" }).to_string()) - } "computer_use_scroll" => { let amount = crate::computer::scroll_amount(v["amount"].as_i64()); let dir = v["direction"].as_str().unwrap_or("").trim().to_lowercase(); @@ -1303,7 +1331,7 @@ pub fn box_script(name: &str, v: &Value, scale: (f64, f64)) -> Result<(String, O "right" => "7", _ => return Err(format!("方向「{dir}」要是 up、down、left 或 right")), }; - let go = if v["x"].is_null() { String::new() } else { format!("{} ", mv(pt(v)?)) }; + let go = if v["x"].is_null() && v["ref"].is_null() { String::new() } else { format!("{} ", mv(pt(v)?)) }; (wrap(format!("{go}click --repeat {amount} --delay 20 {wheel}")), None, format!("已捲動 {dir} ×{amount}。")) } "computer_use_drag" => { @@ -1328,14 +1356,23 @@ pub fn box_script(name: &str, v: &Value, scale: (f64, f64)) -> Result<(String, O return Err("請給要打的字".into()); } let mut script = String::new(); + // Into a field by its number: click it first, like a person would. + let field = match by_number(v) { + Some(r) => { + let ((x, y), what) = r?; + script.push_str(&format!("xdotool mousemove {x} {y} sleep 0.03 click 1 sleep 0.15")); + format!("在 {what}") + } + None => String::new(), + }; if !text.is_empty() { - script.push_str(&type_script(&text)); + script.push_str(&format!(" && {}", type_script(&text))); } if submit { script.push_str(" && sleep 0.06 && xdotool key --clearmodifiers Return"); } let script = script.trim_start_matches(" && ").to_string(); - (script, Some(text.clone()).filter(|t| !t.is_empty()), format!("已打字（{} 個字）{}。", text.chars().count(), if submit { "，再按了 return" } else { "" })) + (script, Some(text.clone()).filter(|t| !t.is_empty()), format!("已打字（{} 個字）{field}{}。", text.chars().count(), if submit { "，再按了 return" } else { "" })) } "computer_use_press_key" => { let keys = v["key"].as_str().unwrap_or(""); @@ -1392,13 +1429,121 @@ fn box_exec(script: &str, stdin: Option<&str>, picture: bool) -> Result Result<(String, String), String> { +/// The numbered controls of the box's latest look (centres in desktop pixels). +static BOX_SPOTS: Mutex> = Mutex::new(None); +/// The latest look's fingerprint (to say when a step changed nothing). +static BOX_LAST: Mutex> = Mutex::new(None); + +/// AT-SPI roles as words. +fn role_words(role: &str) -> String { + match role { + "push button" | "toggle button" => "按鈕", + "link" => "連結", + "entry" => "輸入框", + "password text" => "密碼欄", + "check box" => "勾選框", + "radio button" => "選項", + "menu item" | "check menu item" | "radio menu item" => "選單項目", + "menu" => "選單", + "page tab" | "tab" => "分頁", + "list item" | "tree item" => "項目", + "combo box" => "下拉選單", + "spin button" => "數字欄", + "slider" => "滑桿", + "heading" => "標題", + "image" | "icon" => "圖片", + "table cell" => "格子", + "column header" | "row header" => "欄名", + "dialog" | "alert" => "對話框", + "text" | "paragraph" | "section" | "document web" => "文字", + _ => "文字", + } + .to_string() +} + +/// The box's controls from its accessibility tree (box/a11y's JSON). +pub fn elements_of(v: &Value) -> Vec { + v["elements"] + .as_array() + .into_iter() + .flatten() + .map(|e| { + let role = e["role"].as_str().unwrap_or(""); + let active = e["active"].as_bool().unwrap_or(false); + let kind = if active && matches!(role, "text" | "paragraph" | "section" | "document web") { "輸入框".to_string() } else { role_words(role) }; + let flags = e["flags"].as_array().into_iter().flatten().filter_map(|f| match f.as_str()? { + "focused" => Some("游標在這"), + "checked" => Some("已勾"), + "selected" => Some("已選"), + "disabled" => Some("停用"), + _ => None, + }).collect(); + crate::marks::Element { + kind, + name: e["name"].as_str().unwrap_or("").to_string(), + value: e["value"].as_str().unwrap_or("").to_string(), + x: e["x"].as_f64().unwrap_or(0.0), + y: e["y"].as_f64().unwrap_or(0.0), + w: e["w"].as_f64().unwrap_or(0.0), + h: e["h"].as_f64().unwrap_or(0.0), + active, + flags, + window: e["window"].as_str().unwrap_or("").to_string(), + } + }) + .collect() +} + +/// Do a step's script (if any), wait for the screen to settle, and look: the picture with the +/// controls numbered and the list, in one `docker exec`. After a step it also says when nothing +/// changed, so the bot doesn't click the same empty place again and again. +fn box_look(script: Option<(&str, Option<&str>)>, after: Option) -> Result<(String, Option), String> { + let look = "anybot-a11y --budget 3 > /tmp/anybot-look.json 2>/dev/null || echo '{\"elements\":[]}' > /tmp/anybot-look.json; \ + wc -c < /tmp/anybot-look.json; cat /tmp/anybot-look.json; import -window root png:- 2>/dev/null"; + let full = match script { + Some((s, _)) => format!("{{ {s}; }} || exit $?; sleep {:.2}; {look}", crate::computer::SETTLE_MS as f64 / 1000.0), + None => look.to_string(), + }; + let input = script.and_then(|(_, i)| i).unwrap_or(""); + let out = match dk_with_input(&["exec", "-i", "-u", "bot", "-e", "DISPLAY=:1", container(), "sh", "-c", &full], Some(input.as_bytes()), 30_000) { + Ok(o) => o, + Err(e) if e == "逾時" => return Err("這台電腦太久沒有回應".into()), + Err(e) if e.is_empty() => return Err("這一步沒有成功".into()), + Err(e) => return Err(format!("xdotool：{e}")), + }; + // "\n" + let nl = out.iter().position(|b| *b == b'\n').ok_or("這台電腦沒有回傳畫面")?; + let n: usize = String::from_utf8_lossy(&out[..nl]).trim().parse().map_err(|_| "這台電腦回傳的畫面讀不懂")?; + let json_end = (nl + 1 + n).min(out.len()); + let tree: Value = serde_json::from_slice(&out[nl + 1..json_end]).unwrap_or(Value::Null); + let img = image::load_from_memory(&out[json_end..]).map_err(|e| format!("截圖讀不出來：{e}"))?.to_rgb8(); let (w, h) = img.dimensions(); + shared().lock().unwrap_or_else(|e| e.into_inner()).size = Some((w, h)); + let els = elements_of(&tree); + let (numbered, lines, spots) = crate::marks::number(&els, false); + *BOX_SPOTS.lock().unwrap_or_else(|e| e.into_inner()) = Some(spots); let (fw, fh, resize) = crate::computer::fit(w, h); - let img = if resize { image::imageops::resize(&img, fw, fh, image::imageops::FilterType::Triangle) } else { img }; - let jpeg = crate::computer::encode_jpeg(&img).ok_or("截圖太大了，沒辦法送出")?; - Ok((format!("螢幕截圖（{fw}×{fh}，Linux 桌面）。座標用這張圖上的像素。"), base64::engine::general_purpose::STANDARD.encode(jpeg))) + let mut pic = if resize { image::imageops::resize(&img, fw, fh, image::imageops::FilterType::Triangle) } else { img }; + let fp = crate::marks::fingerprint(&els) ^ crate::marks::picture_fingerprint(&pic).rotate_left(1); + crate::marks::draw(&mut pic, &numbered, (w as f64 / fw.max(1) as f64, h as f64 / fh.max(1) as f64)); + let jpeg = crate::computer::encode_jpeg(&pic).ok_or("截圖太大了，沒辦法送出")?; + let mut text = after.clone().map(|n| format!("{n}\n")).unwrap_or_default(); + text.push_str(&format!("螢幕截圖（{fw}×{fh}，Linux 桌面）。")); + let unchanged = { + let mut last = BOX_LAST.lock().unwrap_or_else(|e| e.into_inner()); + let same = after.is_some() && *last == Some(fp); + *last = Some(fp); + same + }; + if unchanged { + text.push_str("\n⚠️ 畫面和上一步完全一樣：這一步沒有點到東西。不要再做同一步；換一個編號、換一個方法（網頁用 browser_* 工具，或直接開網址），或用 hand_over_computer 請使用者幫忙。"); + } + if lines.is_empty() { + text.push_str("\n畫面上沒有可以照編號點的東西：只好照圖上的位置點 x、y。"); + } else { + text.push_str(&format!("\n可以操作的東西（圖上同樣的編號，用 ref 點）：\n{}", lines.join("\n"))); + } + Ok((text, Some(base64::engine::general_purpose::STANDARD.encode(jpeg)))) } /// Programs people name, as the box knows them. @@ -1446,28 +1591,18 @@ pub fn computer_run(name: &str, v: &Value) -> Result<(String, Option), S let _busy = Busy::new(true); match name { "computer_use_check_permissions" => return Ok(("這台電腦不需要另外的權限：可以截圖，也可以操作滑鼠和鍵盤。".into(), None)), - "computer_use_screenshot" => return picture_of(grab()?).map(|(t, img)| (t, Some(img))), - "computer_use_read_app" => { - return Err("這台是 Linux 桌面，沒有讀 App 文字的方式：用 computer_use_screenshot 看畫面、用座標操作。網頁請用 browser_* 工具（有編號，比座標準）。".into()); - } + "computer_use_screenshot" => return box_look(None, None), _ => {} } if HELD_BY_PERSON.load(Ordering::SeqCst) || crate::vnc::person_active() { return Err("使用者正在自己操作這台電腦。請等他操作完（停下幾秒後）再繼續，或先截圖看看畫面。".into()); } - let (note, picture) = if name == "computer_use_open_app" { + if name == "computer_use_open_app" { let note = open_on_box(v["app"].as_str().unwrap_or(""))?; - std::thread::sleep(Duration::from_millis(crate::computer::SETTLE_MS)); - (note, grab().ok()) - } else { - let (script, stdin, note) = box_script(name, v, box_scale()?)?; - let picture = box_exec(&script, stdin.as_deref(), name != "computer_use_mouse_button")?; - (note, picture) - }; - match picture.map(picture_of).transpose()? { - Some((t, img)) => Ok((format!("{note}\n{t}"), Some(img))), - None => Ok((note, None)), + return box_look(Some(("true", None)), Some(note)); } + let (script, stdin, note) = box_script(name, v, box_scale()?)?; + box_look(Some((&script, stdin.as_deref())), Some(note)) } // ---------- text a person types in a viewer ---------- @@ -1651,6 +1786,52 @@ mod tests { assert!(box_script("computer_use_scroll", &serde_json::json!({"direction": "sideways"}), s).is_err()); } + /// A box made from this image (`ANYBOT_BOX_NAME= cargo test --release -- --ignored + /// box_numbers_are_exact --nocapture`, with a scratch ANYBOT_DATA_DIR). A page like Threads' + /// (small icon links down the side): every link, field and button gets a number on the + /// picture, a click by number lands on it, typing goes into the numbered field, and clicking + /// an empty spot twice is called out. + #[test] + #[ignore] + fn box_numbers_are_exact() { + ensure_running().expect("box starts"); + let cancel = AtomicBool::new(false); + let page = r##"start +

測試頁

+
"##; + dk_with_input(&["exec", "-i", "-u", "bot", container(), "sh", "-c", "mkdir -p /tmp/anybot-page && cat > /tmp/anybot-page/index.html && (cd /tmp/anybot-page && setsid python3 -m http.server 8765 --bind 127.0.0.1 >/dev/null 2>&1 &) ; sleep 0.5"], Some(page.as_bytes()), 10_000).expect("page served"); + let opened = crate::botbrowser::in_box(|| crate::web::run_full("browser_open", r#"{"url":"http://127.0.0.1:8765/"}"#, &cancel)); + assert!(opened.1, "{}", opened.0); + std::thread::sleep(Duration::from_millis(1500)); + let title = || crate::botbrowser::in_box(|| crate::botbrowser::eval_current(None, "document.title")).map(|(_, v)| v.as_str().unwrap_or("").to_string()).unwrap_or_default(); + let t0 = Instant::now(); + let (txt, img) = computer_run("computer_use_screenshot", &serde_json::json!({})).unwrap(); + println!("look: {} ms\n{txt}", t0.elapsed().as_millis()); + if let Some(b) = &img { + let _ = std::fs::write(std::env::temp_dir().join("anybot-numbers.jpg"), base64::engine::general_purpose::STANDARD.decode(b).unwrap_or_default()); + } + let number = |what: &str| -> String { + txt.lines().find(|l| l.contains(what)).and_then(|l| l.trim().strip_prefix('[')).and_then(|l| l.split(']').next()).unwrap_or_else(|| panic!("no number for {what} in:\n{txt}")).to_string() + }; + // The small icon link down the side, by number. + let profile = number("連結「Profile」"); + computer_run("computer_use_click", &serde_json::json!({"ref": profile})).unwrap(); + assert_eq!(title(), "Profile"); + // Typing into the numbered field (Chinese), then the numbered button. + let field = number("輸入框「搜尋」"); + computer_run("computer_use_typing", &serde_json::json!({"ref": field, "value": "貓咪 cats"})).unwrap(); + let (after, _) = computer_run("computer_use_click", &serde_json::json!({"ref": number("按鈕「搜尋」")})).unwrap(); + assert_eq!(title(), "Go:貓咪 cats"); + assert!(!after.contains("完全一樣"), "{after}"); + // An empty spot, twice: the second time it says nothing changed. + computer_run("computer_use_click", &serde_json::json!({"x": 700, "y": 520})).unwrap(); + let (again, _) = computer_run("computer_use_click", &serde_json::json!({"x": 700, "y": 520})).unwrap(); + assert!(again.contains("畫面和上一步完全一樣"), "{again}"); + } + /// Uses the real Docker: `cargo test --release -- --ignored box_screen_is_exact --nocapture`. /// The bot's clicks land where its picture says, the same spot twice doesn't hang, typing (ASCII, Chinese, a key by name) reaches the app, /// and every step comes back with the screen after it. @@ -1822,7 +2003,7 @@ mod tests { assert!(!in_schedule_window("03:00", t("02:59"))); assert!(in_schedule_window("23:30", t("00:10"))); assert!(!in_schedule_window("bad", t("00:10"))); - assert_eq!(wanted_version(), "3"); + assert_eq!(wanted_version(), "4"); } #[test] diff --git a/src/computer.rs b/src/computer.rs index 278228a..493b012 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -12,7 +12,7 @@ use base64::Engine; use serde_json::{json, Value}; use std::sync::Mutex; -pub const NAMES: [&str; 11] = ["computer_use_screenshot", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_mouse_button", "computer_use_check_permissions", "computer_use_read_app", "computer_use_open_app"]; +pub const NAMES: [&str; 9] = ["computer_use_screenshot", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app"]; /// Wait after an action before taking the picture that comes back with it (menus open, pages /// start to change; estimated). pub const SETTLE_MS: u64 = 500; @@ -207,9 +207,7 @@ pub fn label(name: &str) -> &'static str { "computer_use_scroll" => "捲動", "computer_use_mouse_move" => "移動滑鼠", "computer_use_drag" => "拖曳", - "computer_use_mouse_button" => "按住或放開滑鼠", "computer_use_check_permissions" => "檢查螢幕權限", - "computer_use_read_app" => "讀 App 畫面", "computer_use_open_app" => "打開 App", _ => "操作畫面", } @@ -221,7 +219,7 @@ pub fn describe(name: &str, args: &str) -> String { // By number from the app's list (e7), else where on the screen (it said "(0, 0)" for every // click by number). let xy = || match v["ref"].as_str().map(str::trim).filter(|r| !r.is_empty()) { - Some(r) => r.to_string(), + Some(r) => format!("#{}", r.trim_start_matches(['e', '#'])), None => format!("({}, {})", v["x"].as_f64().unwrap_or(0.0), v["y"].as_f64().unwrap_or(0.0)), }; match name { @@ -241,39 +239,34 @@ pub fn describe(name: &str, args: &str) -> String { "computer_use_scroll" => format!("{} {} 格，在 {}", v["direction"].as_str().unwrap_or("down"), v["amount"].as_i64().unwrap_or(3), xy()), "computer_use_mouse_move" => xy(), "computer_use_drag" => format!("{} 個點", v["path"].as_array().map(|a| a.len()).unwrap_or(0)), - "computer_use_mouse_button" => format!("{}{}", if v["action"].as_str() == Some("release") { "放開" } else { "按住" }, if v["x"].is_null() { String::new() } else { format!(" {}", xy()) }), "computer_use_check_permissions" => "螢幕錄製、輔助使用".into(), - "computer_use_read_app" | "computer_use_open_app" => v["app"].as_str().filter(|a| !a.is_empty()).unwrap_or("最前面的 App").to_string(), + "computer_use_open_app" => v["app"].as_str().filter(|a| !a.is_empty()).unwrap_or("").to_string(), _ => String::new(), } } -/// The screen tools, each with a `computer` choice. One way to act (a real click or keystroke), -/// two ways to see: the picture, or an app's controls as numbered text. +/// The screen tools, each with a `computer` choice. One way to see (the numbered picture), one +/// way to act (a real click or keystroke, best by number). pub fn specs_for(computers: &[String], default: &str) -> Vec { let mods = json!({"type": "array", "items": {"type": "string", "enum": ["cmd", "shift", "option", "control"]}}); - let rf = json!({"type": "string", "description": "A control's number from computer_use_read_app (e.g. e12), instead of x, y"}); + let rf = json!({"type": "string", "description": "The number of a control on the latest picture (e.g. 12). Always use a number when the control has one."}); let mut v = vec![ - json!({"name": "computer_use_screenshot", "description": "See a computer's screen. x, y in the other computer_use tools are pixels of this picture. Every action below already returns a fresh picture of the result, so call this only to start or to look again.", + json!({"name": "computer_use_screenshot", "description": "See a computer's screen: a picture with every button, link, field and menu item boxed and numbered, and the same numbers in a list with their names. Act on a control by its number (ref). Every action below already returns the next picture, so call this only to start or to look again.", "parameters": {"type": "object", "properties": {}}}), - json!({"name": "computer_use_click", "description": "Click at x, y (pixels of the latest picture), or on a control by its number (ref) from computer_use_read_app. Returns the screen after the click.", - "parameters": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}, "ref": rf, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "count": {"type": "integer", "description": "2 for a double click"}, "modifier_keys": mods}}}), - json!({"name": "computer_use_typing", "description": "Type text (any language) where the cursor is, or into a field by its number (ref; it is clicked first). submit=true presses return after the text (send a message, run a command). Returns the screen after.", + json!({"name": "computer_use_click", "description": "Click a control by its number (ref) from the latest picture. Only for something without a number (a spot in a picture or game): x, y in the picture's pixels. Returns the next picture.", + "parameters": {"type": "object", "properties": {"ref": rf, "x": {"type": "number"}, "y": {"type": "number"}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "count": {"type": "integer", "description": "2 for a double click"}, "modifier_keys": mods}}}), + json!({"name": "computer_use_typing", "description": "Type text (any language) into a field by its number (ref; it is clicked first), or where the cursor is. submit=true presses return after it (send, search, run a command). Returns the next picture.", "parameters": {"type": "object", "properties": {"value": {"type": "string"}, "ref": rf, "submit": {"type": "boolean"}}, "required": ["value"]}}), - json!({"name": "computer_use_press_key", "description": "Press a key or shortcut in the app in front: return, tab, escape, delete, up, slash, cmd+c, cmd+shift+t, cmd+space. Several in order: separate with spaces (\"cmd+a delete\"). On a Linux computer cmd means ctrl. Returns the screen after.", + json!({"name": "computer_use_press_key", "description": "Press a key or shortcut in the app in front: return, tab, escape, delete, up, slash, cmd+c, cmd+shift+t, cmd+space. Several in order: separate with spaces (\"cmd+a delete\"). On a Linux computer cmd means ctrl. Returns the next picture.", "parameters": {"type": "object", "properties": {"key": {"type": "string"}, "hold_duration_ms": {"type": "integer"}}, "required": ["key"]}}), - json!({"name": "computer_use_scroll", "description": "Scroll at x, y or on a control by number (ref); neither = where the pointer is. Returns the screen after.", - "parameters": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}, "ref": rf, "direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "amount": {"type": "integer", "description": "Scroll steps of 16 pixels, 1-20, default 3"}, "modifier_keys": mods}, "required": ["direction"]}}), - json!({"name": "computer_use_open_app", "description": "Open an app, or bring it to the front when it is already open (its name as shown, e.g. Ghostty, Calculator, 計算機, Chromium, xterm). Returns the screen after.", + json!({"name": "computer_use_scroll", "description": "Scroll a list or page: on a control by number (ref), or at x, y, or where the pointer is. Returns the next picture.", + "parameters": {"type": "object", "properties": {"ref": rf, "x": {"type": "number"}, "y": {"type": "number"}, "direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "amount": {"type": "integer", "description": "Scroll steps of 16 pixels, 1-20, default 3"}, "modifier_keys": mods}, "required": ["direction"]}}), + json!({"name": "computer_use_open_app", "description": "Open an app, or bring it to the front when it is already open (its name as shown, e.g. Ghostty, Calculator, 計算機, Chromium, xterm). Returns the next picture.", "parameters": {"type": "object", "properties": {"app": {"type": "string"}}, "required": ["app"]}}), - json!({"name": "computer_use_read_app", "description": "Read an app's window as text: buttons, fields, menus, list items and visible text, each control numbered (e1, e2…) for click/typing/scroll's ref. For when a picture can't be taken or isn't enough (exact text, long lists). app: its name; leave out for the app in front. Only on computers that offer it (not the bots' own Linux computer).", - "parameters": {"type": "object", "properties": {"app": {"type": "string"}}}}), - json!({"name": "computer_use_mouse_move", "description": "Move the mouse to x, y (e.g. to show a hover menu). Returns the screen after.", - "parameters": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}), - json!({"name": "computer_use_drag", "description": "Drag with the mouse along a path of points (first = press, last = release). Returns the screen after.", + json!({"name": "computer_use_mouse_move", "description": "Move the mouse onto a control by number (ref) or to x, y (to open a hover menu). Returns the next picture.", + "parameters": {"type": "object", "properties": {"ref": rf, "x": {"type": "number"}, "y": {"type": "number"}}}}), + json!({"name": "computer_use_drag", "description": "Drag along a path of points in the picture's pixels (first = press, last = release). Returns the next picture.", "parameters": {"type": "object", "properties": {"path": {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "button": {"type": "string", "enum": ["left", "right"]}, "modifier_keys": mods}, "required": ["path"]}}), - json!({"name": "computer_use_mouse_button", "description": "Press and hold, or release, a mouse button (optionally moving to x, y first). For long presses and custom drags.", - "parameters": {"type": "object", "properties": {"action": {"type": "string", "enum": ["press", "release"]}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "x": {"type": "number"}, "y": {"type": "number"}}, "required": ["action"]}}), json!({"name": "computer_use_check_permissions", "description": "Check whether the screen can be seen and the mouse and keyboard used on that computer.", "parameters": {"type": "object", "properties": {}}}), ]; @@ -454,7 +447,7 @@ pub fn run_by(name: &str, args: &str, who: &str) -> (String, bool, Option bool { - matches!(name, "computer_use_click" | "computer_use_typing" | "computer_use_press_key" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_drag" | "computer_use_mouse_button" | "computer_use_open_app") + matches!(name, "computer_use_click" | "computer_use_typing" | "computer_use_press_key" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_drag" | "computer_use_open_app") } /// Ask macOS for a permission (its own prompt, then 系統設定 at that pane) at most once every ten @@ -527,10 +520,20 @@ fn open_app(app: &str) -> Result { if app.is_empty() { return Err("請給要打開的 App 名稱".into()); } - launch(app).or_else(|e| match app_named(app) { - Some(real) if real != app => launch(&real), + let real = app_named(app); + launch(app).or_else(|e| match &real { + Some(r) if r != app => launch(r), _ => Err(e), })?; + // Wait for its window to be in front (switching desktops takes a moment) before looking. + let start = std::time::Instant::now(); + while start.elapsed() < std::time::Duration::from_millis(2500) { + let front = mac::front_app().map(|f| f.0.to_lowercase()).unwrap_or_default(); + if front == app.to_lowercase() || real.as_ref().map(|r| front == r.to_lowercase()).unwrap_or(false) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } Ok(format!("已打開「{app}」，放在最前面。")) } @@ -567,13 +570,19 @@ fn bring_forward(app: &str) { } } -/// Where a step acts: a control's number (from computer_use_read_app), or x, y on the picture. -/// Returns the screen point and how to say it (「e7（按鈕「7」）」, 「(512, 300)」). +/// The numbered controls of this computer's latest look, and the app they belong to. +static SPOTS: Mutex> = Mutex::new(None); +/// The latest look's fingerprint (to say when a step changed nothing). +static LAST_LOOK: Mutex> = Mutex::new(None); + +/// Where a step acts: a control's number on the latest picture, or x, y on the picture. +/// Returns the screen point and how to say it (「#7（按鈕「7」）」, 「(512, 300)」). fn place_of(v: &Value, scale: (f64, f64)) -> Result, String> { - if let Some(r) = v["ref"].as_str().map(str::trim).filter(|r| !r.is_empty()) { - let s = crate::desktop::spot(r)?; - bring_forward(&s.app); - return Ok(Some((s.x, s.y, format!("{}（{}）", crate::desktop::short_ref(r)?, s.what)))); + if let Some(r) = v["ref"].as_str().map(str::trim).filter(|r| !r.is_empty()).or_else(|| v["ref"].as_u64().map(|_| "")).map(|r| if r.is_empty() { v["ref"].to_string() } else { r.to_string() }) { + let (app, spots) = SPOTS.lock().unwrap_or_else(|e| e.into_inner()).clone().ok_or("先用 computer_use_screenshot 看一下畫面，再照圖上的編號操作。")?; + let ((x, y), what) = spots.get(&r)?; + bring_forward(&app); + return Ok(Some((x, y, format!("#{}（{what}）", crate::marks::number_of(&r).unwrap_or(0))))); } match (v["x"].as_f64(), v["y"].as_f64()) { (Some(x), Some(y)) => { @@ -651,12 +660,6 @@ pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result { let (a, b) = (pts[0], pts[pts.len() - 1]); Ok(format!("已從 ({:.0}, {:.0}) 拖曳到 ({:.0}, {:.0})（{side}）。", a.0, a.1, b.0, b.1)) } - "computer_use_mouse_button" => { - let at = place_of(v, scale)?.map(|(x, y, _)| (x, y)); - let release = v["action"].as_str() == Some("release"); - mac::mouse_button(at, button, !release); - Ok((if release { "已放開滑鼠鍵。" } else { "已按住滑鼠鍵（記得之後放開）。" }).into()) - } "computer_use_typing" => { let text = v["value"].as_str().unwrap_or(""); let submit = v["submit"].as_bool().unwrap_or(false); @@ -688,24 +691,87 @@ pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result { } } -/// What the screen looks like now, after a step: the picture when this computer can take one, -/// else the app in front as numbered text. The bot sees the result of each step in the same -/// answer instead of spending another step (another trip to the model) on a screenshot. +/// What the screen looks like now: the picture with the controls of the app in front boxed and +/// numbered, and the list of them. Without a picture (no 螢幕錄製), the list with the app's text. +/// After a step (`after`), it also says when nothing changed, so the bot doesn't click the same +/// empty place again and again. +fn look(after: Option) -> Result<(String, Option), String> { + let picture = if mac::screen_capture_preflight() { Some(capture_screen().map_err(|e| format!("看不到畫面：{e}"))?) } else { None }; + let found = if controls_allowed() { crate::desktop::elements("").ok() } else { None }; + if picture.is_none() && found.is_none() { + ask_permission("Privacy_ScreenCapture"); + return Err(NO_SCREEN_RECORDING.into()); + } + let (app, window, mut els) = found.unwrap_or_default(); + // A number is a promise that a click there lands in this app: a control another window + // covers (or one on another desktop) gets none. + let hidden = cover_check(window, &mut els); + let (numbered, lines, spots) = crate::marks::number(&els, picture.is_none()); + *SPOTS.lock().unwrap_or_else(|e| e.into_inner()) = Some((app.clone(), spots)); + let mut text = after.clone().map(|n| format!("{n}\n")).unwrap_or_default(); + let mut fp = crate::marks::fingerprint(&els); + let mut img_b64 = None; + if let Some(img) = picture { + let (fw, fh) = shot_size(); + let mut img = if img.dimensions() != (fw, fh) { image::imageops::resize(&img, fw, fh, image::imageops::FilterType::Triangle) } else { img }; + fp ^= crate::marks::picture_fingerprint(&img).rotate_left(1); + crate::marks::draw(&mut img, &numbered, scale_here()); + let jpeg = encode_jpeg(&img).ok_or("截圖太大了，沒辦法送出")?; + img_b64 = Some(base64::engine::general_purpose::STANDARD.encode(&jpeg)); + text.push_str(&format!("螢幕截圖（{fw}×{fh}）。")); + } else { + text.push_str("（這台現在拍不到畫面：macOS 還沒允許「螢幕與系統錄音」。下面是最前面 App 的內容。）"); + } + let unchanged = { + let mut last = LAST_LOOK.lock().unwrap_or_else(|e| e.into_inner()); + let same = after.is_some() && *last == Some(fp); + *last = Some(fp); + same + }; + if unchanged { + text.push_str("\n⚠️ 畫面和上一步完全一樣：這一步沒有點到東西。不要再做同一步；換一個編號、換一個方法（網頁用 browser_* 工具，或直接開網址），或用 hand_over_computer 請使用者幫忙。"); + } + if hidden > 0 { + text.push_str(&format!("\n（「{app}」有 {hidden} 個按鈕被別的視窗擋住或不在這個桌面，沒有編號：先 computer_use_open_app 把它叫到前面。）")); + } + if lines.is_empty() { + text.push_str(&format!("\n「{app}」沒有可以照編號點的東西（有些 App 不提供）：只好照圖上的位置點 x、y。")); + } else { + text.push_str(&format!("\n「{app}」上可以操作的東西（圖上同樣的編號，用 ref 點）：\n{}", lines.join("\n"))); + } + Ok((text, img_b64)) +} + +/// Take the number away from controls whose spot another app's window covers (macOS; elsewhere +/// the window system doesn't say). Returns how many. +fn cover_check(window: Option, els: &mut [crate::marks::Element]) -> usize { + #[cfg(target_os = "macos")] + { + let Some(pid) = window.and_then(mac::pid_of_window) else { return 0 }; + let mut n = 0; + for e in els.iter_mut().filter(|e| e.active) { + let (x, y) = e.centre(); + if mac::pid_on_top_at(x, y).map(|p| p != pid).unwrap_or(false) { + e.active = false; + n += 1; + } + } + n + } + #[cfg(not(target_os = "macos"))] + { + let _ = (window, els); + 0 + } +} + +/// After a step: wait for the screen to settle, then look. fn observe(note: String) -> (String, Option) { std::thread::sleep(std::time::Duration::from_millis(SETTLE_MS)); - if mac::screen_capture_preflight() { - return match screenshot() { - Ok((t, img)) => (format!("{note}\n{t}"), Some(img)), - Err(e) => (format!("{note}\n（看不到結果：{e}）"), None), - }; + match look(Some(note.clone())) { + Ok(r) => r, + Err(e) => (format!("{note}（看不到結果：{e}）"), None), } - if controls_allowed() { - if let Ok((app, lines)) = crate::desktop::read_app("") { - let shown: Vec<&String> = lines.iter().take(120).collect(); - return (format!("{note}\n（這台現在拍不到畫面，改讀最前面的「{app}」；e 編號可以用在 ref）\n{}", shown.iter().map(|l| l.as_str()).collect::>().join("\n")), None); - } - } - (format!("{note}（這台現在看不到畫面，無法確認結果）"), None) } fn run_inner(name: &str, v: &Value) -> Result<(String, Option), String> { @@ -717,7 +783,7 @@ fn run_inner(name: &str, v: &Value) -> Result<(String, Option), String> return Err(crate::desk::describe().into()); } match name { - "computer_use_screenshot" => return screenshot().map(|(t, img)| (t, Some(img))), + "computer_use_screenshot" => return look(None), "computer_use_check_permissions" => { #[cfg(target_os = "linux")] return Ok((crate::desk::describe().into(), None)); @@ -728,15 +794,6 @@ fn run_inner(name: &str, v: &Value) -> Result<(String, Option), String> return Ok((format!("螢幕錄製（看畫面）：{}\n輔助使用（滑鼠、鍵盤、讀 App）：{}{}", if see { "已允許" } else { "還沒允許" }, if control { "已允許" } else { "還沒允許" }, if see && control { "" } else { "\n請使用者到「系統設定 → 隱私權與安全性」打開 AnyBot。" }), None)); } } - "computer_use_read_app" => { - need_controls()?; - let (app, lines) = crate::desktop::read_app(v["app"].as_str().unwrap_or("")).map_err(plain_words)?; - return Ok((if lines.is_empty() { - format!("「{app}」的畫面讀不到內容（有些 App 不提供輔助使用資訊）。可以改用 computer_use_screenshot 看。") - } else { - format!("「{app}」的畫面（e 編號可以用在 click / typing / scroll 的 ref）：\n{}", lines.join("\n")) - }, None)); - } _ => {} } if !moves_things(name) { @@ -749,10 +806,6 @@ fn run_inner(name: &str, v: &Value) -> Result<(String, Option), String> need_controls()?; } let note = act(name, v, scale_here()).map_err(plain_words)?; - // Holding a button down is the middle of something (a long press, a drag): no picture yet. - if name == "computer_use_mouse_button" { - return Ok((note, None)); - } Ok(observe(note)) } @@ -781,18 +834,15 @@ pub fn person_act(name: &str, v: &Value) -> Result<(), String> { /// frontmost app. Used for the per-app permission (「AnyBot 可以看到 {app}，並在其中點擊和輸入」). pub fn target_app(name: &str, args: &str) -> Option { let v = crate::tools::args_of(args); - if let Some(app) = v["app"].as_str().filter(|a| !a.trim().is_empty() && matches!(name, "computer_use_read_app" | "computer_use_open_app")) { + if let Some(app) = v["app"].as_str().filter(|a| !a.trim().is_empty() && name == "computer_use_open_app") { return Some(app.trim().to_string()); } - if name == "computer_use_read_app" { - return frontmost_app(); - } - if let Some(r) = v["ref"].as_str().filter(|r| !r.trim().is_empty()) { - return crate::desktop::spot(r).ok().map(|s| s.app).or_else(crate::desktop::last_app); + if !v["ref"].is_null() { + return SPOTS.lock().unwrap_or_else(|e| e.into_inner()).as_ref().map(|(app, _)| app.clone()); } let s = scale_here(); let point = match name { - "computer_use_click" | "computer_use_mouse_move" | "computer_use_scroll" | "computer_use_mouse_button" if !v["x"].is_null() => Some(to_screen(v["x"].as_f64()?, v["y"].as_f64()?, s)), + "computer_use_click" | "computer_use_mouse_move" | "computer_use_scroll" if !v["x"].is_null() => Some(to_screen(v["x"].as_f64()?, v["y"].as_f64()?, s)), "computer_use_drag" => v["path"].as_array().and_then(|a| a.first()).and_then(|p| Some(to_screen(p["x"].as_f64()?, p["y"].as_f64()?, s))), _ => None, }; @@ -849,18 +899,16 @@ pub fn app_at_point(x: f64, y: f64) -> Option { mac::window_at(x, y).map(|w| w.0) } -fn screenshot() -> Result<(String, String), String> { +/// The screen as the person sees it (no numbers), at the bot's picture size: for the phone. +pub fn plain_picture() -> Result<(u32, u32, Vec), String> { if !mac::screen_capture_preflight() { ask_permission("Privacy_ScreenCapture"); return Err(NO_SCREEN_RECORDING.into()); } let img = capture_screen()?; - // Always the same size for this screen (shot_size), so coordinates map without state. let (fw, fh) = shot_size(); let img = if img.dimensions() != (fw, fh) { image::imageops::resize(&img, fw, fh, image::imageops::FilterType::Triangle) } else { img }; - let jpeg = encode_jpeg(&img).ok_or("截圖太大了，沒辦法送出")?; - let b64 = base64::engine::general_purpose::STANDARD.encode(&jpeg); - Ok((format!("螢幕截圖（{fw}×{fh}）。座標用這張圖上的像素。"), b64)) + Ok((fw, fh, encode_jpeg(&img).ok_or("截圖太大了，沒辦法送出")?)) } /// The whole screen as RGB (backing pixels). @@ -936,6 +984,7 @@ mod mac { static kCGWindowOwnerPID: *const c_void; static kCGWindowBounds: *const c_void; static kCGWindowLayer: *const c_void; + static kCGWindowNumber: *const c_void; } #[link(name = "CoreFoundation", kind = "framework")] extern "C" { @@ -946,8 +995,8 @@ mod mac { fn CFStringGetCString(s: *const c_void, buf: *mut std::ffi::c_char, len: isize, enc: u32) -> bool; } - /// On-screen windows front to back: (owner app, pid, bounds), normal layer only. - fn windows() -> Vec<(String, i32, CGRect)> { + /// On-screen windows front to back: (owner app, pid, bounds, layer, window number). + fn all_windows() -> Vec<(String, i32, CGRect, i32, u32)> { let mut out = vec![]; unsafe { // kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements @@ -955,18 +1004,17 @@ mod mac { if list.is_null() { return out; } + let num = |d: *const c_void, k: *const c_void| -> i64 { + let mut v: i64 = -1; + let p = CFDictionaryGetValue(d, k); + if !p.is_null() { + CFNumberGetValue(p, 4, &mut v as *mut i64 as *mut c_void); + } + v + }; for i in 0..CFArrayGetCount(list) { let d = CFArrayGetValueAtIndex(list, i); - let mut layer: i32 = -1; - let l = CFDictionaryGetValue(d, kCGWindowLayer); - if l.is_null() || !CFNumberGetValue(l, 3, &mut layer as *mut i32 as *mut c_void) || layer != 0 { - continue; - } - let mut pid: i32 = 0; - let p = CFDictionaryGetValue(d, kCGWindowOwnerPID); - if !p.is_null() { - CFNumberGetValue(p, 3, &mut pid as *mut i32 as *mut c_void); - } + let (layer, pid, number) = (num(d, kCGWindowLayer) as i32, num(d, kCGWindowOwnerPID) as i32, num(d, kCGWindowNumber) as u32); let mut name = String::new(); let n = CFDictionaryGetValue(d, kCGWindowOwnerName); if !n.is_null() { @@ -981,13 +1029,34 @@ mod mac { if !b.is_null() { CGRectMakeWithDictionaryRepresentation(b, &mut r); } - out.push((name, pid, r)); + out.push((name, pid, r, layer, number)); } CFRelease(list); } out } + /// On-screen windows front to back: (owner app, pid, bounds), normal layer only. + fn windows() -> Vec<(String, i32, CGRect)> { + all_windows().into_iter().filter(|w| w.3 == 0).map(|(n, p, r, _, _)| (n, p, r)).collect() + } + + /// The app (pid) that owns a window, by its number. + pub fn pid_of_window(number: u32) -> Option { + all_windows().into_iter().find(|w| w.4 == number).map(|w| w.1) + } + + /// Whose window is on top at a screen point: app windows, floating and modal panels, pop-up + /// menus (window levels 0, 3, 8, 101). Not the menu bar, overlays, or the Dock's invisible + /// full-screen window (level 20), which lies over everything. + pub fn pid_on_top_at(x: f64, y: f64) -> Option { + all_windows() + .into_iter() + .filter(|w| matches!(w.3, 0 | 3 | 8 | 101) && w.2.size.width > 1.0) + .find(|(_, _, r, _, _)| x >= r.origin.x && y >= r.origin.y && x < r.origin.x + r.size.width && y < r.origin.y + r.size.height) + .map(|w| w.1) + } + /// The app whose window is under a screen point. pub fn window_at(x: f64, y: f64) -> Option<(String, i32)> { windows().into_iter().find(|(_, _, r)| x >= r.origin.x && y >= r.origin.y && x < r.origin.x + r.size.width && y < r.origin.y + r.size.height).map(|(n, p, _)| (n, p)) @@ -1023,40 +1092,6 @@ mod mac { }) } - /// Press or release a mouse button where the pointer is (or at a point). - pub fn mouse_button(at: Option<(f64, f64)>, button: &str, press: bool) { - let (down, up, _, b) = kinds(button); - let p = match at { - Some((x, y)) => { - move_to(x, y); - gap(); - CGPoint { x, y } - } - None => current_point(), - }; - let src = Source::new(); - let e = unsafe { CGEventCreateMouseEvent(src.0, if press { down } else { up }, p, b) }; - if !e.is_null() { - unsafe { CGEventSetIntegerValueField(e, CLICK_STATE, 1) }; - } - post(e, 0); - } - - fn current_point() -> CGPoint { - extern "C" { - fn CGEventCreate(src: Src) -> Ev; - fn CGEventGetLocation(e: Ev) -> CGPoint; - } - unsafe { - let e = CGEventCreate(std::ptr::null_mut()); - if e.is_null() { - return CGPoint { x: 0.0, y: 0.0 }; - } - let p = CGEventGetLocation(e); - CFRelease(e); - p - } - } #[link(name = "CoreFoundation", kind = "framework")] extern "C" { fn CFRelease(x: *const c_void); @@ -1291,7 +1326,6 @@ mod mac { pub fn screen_pixels() -> (u32, u32) { (0, 0) } - pub fn mouse_button(_: Option<(f64, f64)>, _: &str, _: bool) {} } #[cfg(test)] diff --git a/src/desk.rs b/src/desk.rs index a65dca4..be4fafe 100644 --- a/src/desk.rs +++ b/src/desk.rs @@ -180,17 +180,6 @@ pub fn click(px: f64, py: f64, which: &str, count: i64, flags: u64) { }); } -pub fn mouse_button(at: Option<(f64, f64)>, which: &str, press: bool) { - if kind() == Kind::X11 { - return x11::mouse_button(at, which, press); - } - if let Some((x, y)) = at { - motion(x, y); - gap(); - } - button(button_no(which), press); -} - pub fn drag(path: &[(f64, f64)], which: &str, flags: u64) { if kind() == Kind::X11 { return x11::drag(path, which, flags); diff --git a/src/desktop.rs b/src/desktop.rs index b5f3b35..9be5dd8 100644 --- a/src/desktop.rs +++ b/src/desktop.rs @@ -78,52 +78,6 @@ pub fn run(cmd: &str, a: &Value) -> Result { // ---------- what the bot uses ---------- -/// Where a numbered control is: its centre in screen coordinates (the space the mouse uses), -/// and what it is (「按鈕「7」」), for the step's description. -#[derive(Clone, Debug, PartialEq)] -pub struct Spot { - pub app: String, - pub x: f64, - pub y: f64, - pub what: String, -} - -/// Each number of a reading and its spot (None when the app didn't say where). -pub type Spots = std::collections::HashMap>; - -/// The latest reading: the app and its numbers. -static LAST: std::sync::Mutex> = std::sync::Mutex::new(None); - -/// "e6", "[e6]", "6" or a full "@snap:e6" → "e6". -pub fn short_ref(r: &str) -> Result { - let r = r.trim().trim_matches(['[', ']']); - let r = r.rsplit(':').next().unwrap_or(r); - let n = r.trim_start_matches('e'); - if n.is_empty() || !n.chars().all(|c| c.is_ascii_digit()) { - return Err(format!("「{r}」不是元素編號，請用 computer_use_read_app 列出的 e 編號")); - } - Ok(format!("e{n}")) -} - -/// Where to click for a number from the latest reading. -pub fn spot(r: &str) -> Result { - let r = short_ref(r)?; - let last = LAST.lock().unwrap_or_else(|e| e.into_inner()); - let (_, spots) = last.as_ref().ok_or("請先用 computer_use_read_app 讀一下那個 App 的畫面")?; - match spots.get(&r) { - Some(Some(s)) => Ok(s.clone()), - // Wayland doesn't tell apps where other windows' controls are: none has a place. - Some(None) if spots.values().all(Option::is_none) => Err(format!("這台電腦讀得到 {r}，但系統不告訴 App 它在螢幕上的位置：請用 computer_use_screenshot 看，用截圖上的 x、y 點。")), - Some(None) => Err(format!("{r} 現在不在畫面上（被捲走或藏起來了）。先捲動或改用截圖上的座標。")), - None => Err(format!("最近一次讀到的畫面裡沒有 {r}：先用 computer_use_read_app 再讀一次。")), - } -} - -/// The app the latest reading was of (for the per-app permission). -pub fn last_app() -> Option { - LAST.lock().unwrap_or_else(|e| e.into_inner()).as_ref().map(|l| l.0.clone()) -} - fn role_name(role: &str) -> &str { match role { "button" => "按鈕", @@ -138,17 +92,19 @@ fn role_name(role: &str) -> &str { "slider" => "滑桿", "statictext" | "text" => "文字", "image" => "圖片", - "scrollarea" => "捲動區", - "group" => "群組", - "outline" | "list" | "table" | "tree" | "grid" => "清單", - "toolbar" => "工具列", - "menubar" | "menu" => "選單", "heading" => "標題", - "window" => "視窗", other => other, } } +/// Roles a person clicks or types into (the rest is text to read). +fn active(role: &str) -> bool { + matches!( + role, + "button" | "link" | "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield" | "checkbox" | "switch" | "toggle" | "radiobutton" | "popupbutton" | "menubutton" | "menuitem" | "tab" | "row" | "cell" | "listitem" | "treeitem" | "outlinerow" | "slider" | "incrementor" + ) +} + /// Longest value shown whole; longer ones (a terminal's scrollback, a document) keep their /// start and their end, where a terminal's prompt and newest output are. const VALUE_HEAD: usize = 160; @@ -164,99 +120,51 @@ fn shorten(v: &str) -> String { format!("{head} …（中間省略 {} 字）… {tail}", n - VALUE_HEAD - VALUE_TAIL) } -/// The centre of a node's box, when it has a real one on the screen. -fn centre(n: &Value) -> Option<(f64, f64)> { - let b = n.get("bounds")?; - let (x, y, w, h) = (b["x"].as_f64()?, b["y"].as_f64()?, b["width"].as_f64()?, b["height"].as_f64()?); - if w < 1.0 || h < 1.0 || x + w <= 0.0 || y + h <= 0.0 { - return None; - } - Some((x + w / 2.0, y + h / 2.0)) -} - -/// A snapshot as short lines, and each number's spot. -pub fn lines_of(tree: &Value, app: &str) -> (Vec, Spots) { - let mut lines = vec![]; - let mut spots = std::collections::HashMap::new(); - fn walk(n: &Value, depth: usize, lines: &mut Vec, spots: &mut Spots, parent: &str, app: &str) { - if lines.len() >= 400 { +/// The controls and text of a snapshot, with where each is on the screen (nodes the app gave +/// no place are left out: they can't be shown or clicked). +pub fn elements_of(tree: &Value, app: &str) -> Vec { + fn walk(n: &Value, out: &mut Vec, app: &str) { + if out.len() >= 600 { return; } - let name = n["name"].as_str().unwrap_or("").trim(); - let value = n["value"].as_str().map(str::to_string).or_else(|| n["value"].as_f64().map(|f| f.to_string())).unwrap_or_default(); - let value = shorten(value.trim()); let role = n["role"].as_str().unwrap_or(""); - let text = if !name.is_empty() && !value.is_empty() && value != name { format!("「{name}」：{value}") } else if !name.is_empty() { format!("「{name}」") } else if !value.is_empty() { format!("：{value}") } else { String::new() }; - let mut flags = vec![]; - let is_field = matches!(role, "textfield" | "textarea" | "searchfield" | "combobox" | "securetextfield"); - for (k, word) in [("checked", "已勾"), ("selected", "已選"), ("disabled", "停用"), ("focused", "游標在這")] { - // Only text fields report focus usefully (Finder marks every sidebar cell focused). - if k == "focused" && !is_field { - continue; - } - if n["states"].as_array().map(|a| a.iter().any(|x| x == k)).unwrap_or(false) || n[k] == true { - flags.push(word); + let name = n["name"].as_str().unwrap_or("").trim().to_string(); + let value = n["value"].as_str().map(str::to_string).or_else(|| n["value"].as_f64().map(|f| f.to_string())).unwrap_or_default(); + let b = &n["bounds"]; + if let (Some(x), Some(y), Some(w), Some(h)) = (b["x"].as_f64(), b["y"].as_f64(), b["width"].as_f64(), b["height"].as_f64()) { + let is_active = n["ref_id"].is_string() && active(role); + if w >= 1.0 && h >= 1.0 && x + w > 0.0 && y + h > 0.0 && (is_active || !name.is_empty() || !value.trim().is_empty()) && role != "window" && role != "group" { + let has = |k: &str| n["states"].as_array().map(|a| a.iter().any(|x| x == k)).unwrap_or(false) || n[k] == true; + let mut flags = vec![]; + for (k, word) in [("checked", "已勾"), ("selected", "已選"), ("disabled", "停用"), ("focused", "游標在這")] { + if has(k) && (k != "focused" || role.contains("text") || role.contains("field")) { + flags.push(word); + } + } + out.push(crate::marks::Element { kind: role_name(role).to_string(), name, value: shorten(value.trim()), x, y, w, h, active: is_active, flags, window: app.to_string() }); } } - let flags = if flags.is_empty() { String::new() } else { format!("（{}）", flags.join("、")) }; - let indent = " ".repeat(depth.min(6)); - // A cell or label that only repeats its parent's name adds nothing. - let repeats = !name.is_empty() && name == parent; - if repeats && n["ref_id"].is_null() { - // plain text under a control with the same name - } else if repeats && lines.last().map(|l| l.contains(&format!("「{name}」"))).unwrap_or(false) { - // e.g. Finder's row → cell with the same name: keep the row only - } else if let Some(r) = n["ref_id"].as_str() { - let short = r.rsplit(':').next().unwrap_or(r).to_string(); - if !text.is_empty() || matches!(role, "button" | "textfield" | "textarea" | "checkbox" | "searchfield" | "popupbutton") { - let what = format!("{}{}", role_name(role), if name.is_empty() { String::new() } else { format!("「{}」", name.chars().take(30).collect::()) }); - spots.insert(short.clone(), centre(n).map(|(x, y)| Spot { app: app.to_string(), x, y, what })); - lines.push(format!("{indent}{short} {}{text}{flags}", role_name(role))); - } - } else if !text.is_empty() && matches!(role, "statictext" | "text" | "heading" | "cell" | "value" | "") { - lines.push(format!("{indent}{}{text}", role_name(role))); - } - let next_parent = if name.is_empty() { parent } else { name }; for c in n["children"].as_array().into_iter().flatten() { - walk(c, depth + 1, lines, spots, next_parent, app); + walk(c, out, app); } } - walk(tree, 0, &mut lines, &mut spots, "", app); - let pad = lines.iter().map(|l| l.len() - l.trim_start().len()).min().unwrap_or(0); - (lines.into_iter().map(|l| l[pad..].to_string()).collect(), spots) + let mut out = vec![]; + walk(tree, &mut out, app); + out } -/// An app's controls (numbered) and visible text, as short lines. `app` empty = the app in front. -/// The numbers and where they are on the screen are remembered for the next click or typing. -pub fn read_app(app: &str) -> Result<(String, Vec), String> { +/// The app in front (or `app`): its name, its window's number (macOS: the CGWindowID) and its +/// controls and text, in screen coordinates. +pub fn elements(app: &str) -> Result<(String, Option, Vec), String> { let mut args = serde_json::json!({"interactive_only": false, "max_depth": 40, "bounds": true}); if !app.trim().is_empty() { args["app"] = Value::String(app.trim().to_string()); } - let v = match run("snapshot", &args) { - // Running without a window (closed, or minimised to the Dock): bring it forward once. - Err(e) if e.contains("WINDOW_NOT_FOUND") && !app.trim().is_empty() => { - let _ = run("launch", &serde_json::json!({"app": app.trim()})); - std::thread::sleep(std::time::Duration::from_millis(900)); - match run("snapshot", &args) { - Ok(v) => v, - Err(e) => { - // macOS: "open" sends the reopen event, like clicking the app in the Dock. - if !cfg!(target_os = "macos") { - return Err(e); - } - let _ = std::process::Command::new("open").args(["-a", app.trim()]).status(); - std::thread::sleep(std::time::Duration::from_millis(1200)); - run("snapshot", &args)? - } - } - } - r => r?, - }; - let app_name = v["app"].as_str().unwrap_or(app).to_string(); - let (lines, spots) = lines_of(&v["tree"], &app_name); - *LAST.lock().unwrap_or_else(|e| e.into_inner()) = Some((app_name.clone(), spots)); - Ok((app_name, lines)) + let v = run("snapshot", &args)?; + let name = v["app"].as_str().unwrap_or(app).to_string(); + let els = elements_of(&v["tree"], &name); + let window = v["window"]["id"].as_str().and_then(|w| w.strip_prefix("w-")).and_then(|n| n.parse().ok()); + Ok((name, window, els)) } /// Open an app, or bring it to the front when it runs (macOS uses LaunchServices instead). @@ -270,24 +178,19 @@ mod tests { use super::*; #[test] - fn numbers_keep_where_they_are() { - let tree = serde_json::json!({"role": "window", "name": "計算機", "children": [ - {"role": "statictext", "value": "54"}, + fn controls_and_text_come_with_their_place() { + let tree = serde_json::json!({"role": "window", "name": "計算機", "bounds": {"x": 0.0, "y": 0.0, "width": 300.0, "height": 400.0}, "children": [ + {"role": "statictext", "value": "54", "bounds": {"x": 10.0, "y": 10.0, "width": 100.0, "height": 30.0}}, {"ref_id": "@s1:e7", "role": "button", "name": "7", "bounds": {"x": 100.0, "y": 200.0, "width": 40.0, "height": 30.0}}, {"ref_id": "@s1:e8", "role": "button", "name": "8", "bounds": {"x": 0.0, "y": 0.0, "width": 0.0, "height": 0.0}}, - {"ref_id": "@s1:e9", "role": "textarea", "value": format!("{}END", "x".repeat(2000))} + {"ref_id": "@s1:e9", "role": "textarea", "value": format!("{}END", "x".repeat(2000)), "bounds": {"x": 0.0, "y": 300.0, "width": 300.0, "height": 90.0}} ]}); - let (lines, spots) = lines_of(&tree, "Calculator"); - assert!(lines.iter().any(|l| l == "e7 按鈕「7」")); - assert_eq!(spots["e7"], Some(Spot { app: "Calculator".into(), x: 120.0, y: 215.0, what: "按鈕「7」".into() })); - // No real box (hidden, scrolled away): known, but not clickable. - assert_eq!(spots["e8"], None); + let els = elements_of(&tree, "Calculator"); + // The window itself and a control with no place are left out. + assert_eq!(els.len(), 3); + assert_eq!((els[0].kind.as_str(), els[0].value.as_str(), els[0].active), ("文字", "54", false)); + assert_eq!((els[1].kind.as_str(), els[1].name.as_str(), els[1].active, els[1].centre()), ("按鈕", "7", true, (120.0, 215.0))); // A terminal's whole scrollback: the end (prompt, newest output) stays. - let e9 = lines.iter().find(|l| l.starts_with("e9")).unwrap(); - assert!(e9.ends_with("END") && e9.contains("中間省略") && e9.chars().count() < 700); - assert_eq!(short_ref("[e7]").unwrap(), "e7"); - assert_eq!(short_ref("@s1:e7").unwrap(), "e7"); - assert_eq!(short_ref("12").unwrap(), "e12"); - assert!(short_ref("ok").is_err()); + assert!(els[2].value.ends_with("END") && els[2].value.contains("中間省略") && els[2].active); } } diff --git a/src/main.rs b/src/main.rs index fdcab69..c843bf3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,6 +28,7 @@ mod i18n; mod i18n_table; mod icons; mod macapps; +mod marks; mod mcp; mod memory; mod model; @@ -204,10 +205,10 @@ fn main() -> eframe::Result<()> { } return Ok(()); } - // For testing on a new system: `anybot --tool computer_use_read_app '{"app":"Calculator"}' - // computer_use_click '{"ref":"e7"}' …` runs computer tools in order (one process, so a - // reading's numbers carry over), prints what the bot would hear, and saves the pictures that - // come back as anybot-tool-N.jpg in the temp folder. + // For testing on a new system: `anybot --tool computer_use_screenshot '{}' computer_use_click + // '{"ref":"7"}' …` runs computer tools in order (one process, so a picture's numbers carry + // over), prints what the bot would hear, and saves the numbered pictures that come back as + // anybot-tool-N.jpg in the temp folder. if args.get(1).map(String::as_str) == Some("--tool") { let mut all_ok = true; for (i, pair) in args[2..].chunks(2).enumerate() { diff --git a/src/marks.rs b/src/marks.rs new file mode 100644 index 0000000..bca214c --- /dev/null +++ b/src/marks.rs @@ -0,0 +1,269 @@ +// What a bot sees of a screen: the picture with every control it can use boxed and numbered, +// and the same numbers in a list ("[12] 連結「Profile」"). It acts by number; the click lands on +// that control's centre. Guessing coordinates from a picture is where clicks went wrong (a +// model took an empty strip for a menu item and clicked it ten times); a number can't miss. +// +// The controls come from the system's accessibility tree: macOS AX and Linux AT-SPI through +// agent-desktop on a person's computer (desktop.rs), AT-SPI in the bots' own computer +// (box/a11y). Web pages are in those trees too (Chromium with renderer accessibility on). + +use std::collections::HashMap; + +/// One control on the screen, in screen units (the mouse's: points on a Mac, pixels elsewhere). +#[derive(Clone, Debug, PartialEq)] +pub struct Element { + /// 按鈕, 連結, 輸入框… (already in words). + pub kind: String, + pub name: String, + pub value: String, + pub x: f64, + pub y: f64, + pub w: f64, + pub h: f64, + /// Something to click or type into (gets a number), not just text. + pub active: bool, + pub flags: Vec<&'static str>, + pub window: String, +} + +impl Element { + pub fn centre(&self) -> (f64, f64) { + (self.x + self.w / 2.0, self.y + self.h / 2.0) + } +} + +/// Most controls numbered on one picture (more is clutter the model can't use). +pub const MAX_MARKS: usize = 150; + +/// The numbered controls of the latest look at a screen: number → (centre, words). +#[derive(Clone, Debug, Default)] +pub struct Spots { + pub by_number: HashMap, +} + +impl Spots { + /// "12", "e12", "[12]" → the control's centre and what it is. + pub fn get(&self, r: &str) -> Result<((f64, f64), String), String> { + let n = number_of(r).ok_or_else(|| format!("「{}」不是編號：請用截圖上的編號（例如 12）", r.trim()))?; + self.by_number.get(&n).cloned().ok_or_else(|| format!("最新的畫面裡沒有編號 {n}：先 computer_use_screenshot 再看一次。")) + } +} + +pub fn number_of(r: &str) -> Option { + let t = r.trim().trim_matches(['[', ']']).trim_start_matches(['e', 'E', '#']); + t.parse().ok() +} + +/// Order (reading order, by window) and number the controls; the list's lines and the spots. +/// `all_text`: also list plain text (when there is no picture to read it from). +pub fn number(elements: &[Element], all_text: bool) -> (Vec<(usize, Element)>, Vec, Spots) { + let mut seen = std::collections::HashSet::new(); + let mut els: Vec<&Element> = elements.iter().filter(|e| e.w >= 1.0 && e.h >= 1.0 && seen.insert((e.kind.clone(), e.name.clone(), e.x as i64, e.y as i64, e.w as i64, e.h as i64))).collect(); + let mut windows: Vec = vec![]; + for e in &els { + if !windows.contains(&e.window) { + windows.push(e.window.clone()); + } + } + els.sort_by(|a, b| { + let wa = windows.iter().position(|w| *w == a.window).unwrap_or(0); + let wb = windows.iter().position(|w| *w == b.window).unwrap_or(0); + (wa, (a.y / 12.0) as i64, a.x as i64).cmp(&(wb, (b.y / 12.0) as i64, b.x as i64)) + }); + let mut numbered = vec![]; + let mut lines = vec![]; + let mut spots = Spots::default(); + let mut n = 0; + let mut window = None; + for e in els { + if !e.active && !all_text { + continue; + } + if windows.len() > 1 && window.as_ref() != Some(&e.window) && !e.window.is_empty() { + window = Some(e.window.clone()); + lines.push(format!("## {}", e.window)); + } + let what = describe(e); + if e.active && n < MAX_MARKS { + n += 1; + spots.by_number.insert(n, (e.centre(), what.clone())); + numbered.push((n, e.clone())); + lines.push(format!("[{n}] {what}")); + } else if !e.active { + lines.push(format!(" {what}")); + } + } + (numbered, lines, spots) +} + +fn describe(e: &Element) -> String { + let short = |s: &str, n: usize| -> String { + let c: Vec = s.chars().collect(); + if c.len() > n { format!("{}…", c[..n].iter().collect::()) } else { s.to_string() } + }; + let mut t = e.kind.clone(); + if !e.name.is_empty() { + t.push_str(&format!("「{}」", short(&e.name, 60))); + } + if !e.value.is_empty() && e.value != e.name { + t.push_str(&format!("：{}", short(&e.value, 80))); + } + if !e.flags.is_empty() { + t.push_str(&format!("（{}）", e.flags.join("、"))); + } + t +} + +/// A fingerprint of what the controls are and where (to say "nothing changed"). +pub fn fingerprint(elements: &[Element]) -> u64 { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + for e in elements { + (e.kind.as_str(), e.name.as_str(), e.value.as_str(), e.x as i64, e.y as i64, e.w as i64, e.h as i64, e.flags.len()).hash(&mut h); + } + h.finish() +} + +/// A coarse fingerprint of a picture (16×10 grey cells), for "nothing changed" when there is no +/// tree to compare. +pub fn picture_fingerprint(img: &image::RgbImage) -> u64 { + use std::hash::{Hash, Hasher}; + let small = image::imageops::resize(img, 16, 10, image::imageops::FilterType::Triangle); + let mut h = std::collections::hash_map::DefaultHasher::new(); + for p in small.pixels() { + // Quantised, so a blinking caret or a clock's second doesn't count as a change. + ((p[0] as u16 + p[1] as u16 + p[2] as u16) / 24).hash(&mut h); + } + h.finish() +} + +const COLOURS: [[u8; 3]; 6] = [[220, 38, 38], [37, 99, 235], [22, 163, 74], [192, 38, 211], [234, 88, 12], [8, 145, 178]]; + +/// Box and number each control on the picture. `to_pic` maps screen units to picture pixels. +pub fn draw(img: &mut image::RgbImage, numbered: &[(usize, Element)], to_pic: (f64, f64)) { + let (iw, ih) = (img.width() as i64, img.height() as i64); + for (n, e) in numbered { + let c = COLOURS[n % COLOURS.len()]; + let (x0, y0) = ((e.x / to_pic.0).round() as i64, (e.y / to_pic.1).round() as i64); + let (x1, y1) = (((e.x + e.w) / to_pic.0).round() as i64, ((e.y + e.h) / to_pic.1).round() as i64); + rect_outline(img, x0, y0, x1, y1, c); + // The number: white digits on the control's colour, at its top-left corner (inside the + // picture). + let label = n.to_string(); + let (lw, lh) = (label.len() as i64 * (DIGIT_W + 1) * SCALE + 3, DIGIT_H * SCALE + 4); + let lx = x0.clamp(0, (iw - lw).max(0)); + let ly = (y0 - lh).clamp(0, (ih - lh).max(0)); + let ly = if ly + lh > y0 && y0 >= lh { y0 - lh } else { ly }; + fill(img, lx, ly, lx + lw, ly + lh, c); + let mut cx = lx + 2; + for ch in label.bytes() { + digit(img, cx, ly + 2, ch - b'0', [255, 255, 255]); + cx += (DIGIT_W + 1) * SCALE; + } + } +} + +fn put(img: &mut image::RgbImage, x: i64, y: i64, c: [u8; 3]) { + if x >= 0 && y >= 0 && (x as u32) < img.width() && (y as u32) < img.height() { + img.put_pixel(x as u32, y as u32, image::Rgb(c)); + } +} + +fn fill(img: &mut image::RgbImage, x0: i64, y0: i64, x1: i64, y1: i64, c: [u8; 3]) { + for y in y0..y1 { + for x in x0..x1 { + put(img, x, y, c); + } + } +} + +fn rect_outline(img: &mut image::RgbImage, x0: i64, y0: i64, x1: i64, y1: i64, c: [u8; 3]) { + for x in x0..=x1 { + put(img, x, y0, c); + put(img, x, y1, c); + } + for y in y0..=y1 { + put(img, x0, y, c); + put(img, x1, y, c); + } +} + +/// A 3×5 digit font, each pixel drawn SCALE×SCALE. +const DIGIT_W: i64 = 3; +const DIGIT_H: i64 = 5; +const SCALE: i64 = 2; +const DIGITS: [[u8; 5]; 10] = [ + [0b111, 0b101, 0b101, 0b101, 0b111], + [0b010, 0b110, 0b010, 0b010, 0b111], + [0b111, 0b001, 0b111, 0b100, 0b111], + [0b111, 0b001, 0b111, 0b001, 0b111], + [0b101, 0b101, 0b111, 0b001, 0b001], + [0b111, 0b100, 0b111, 0b001, 0b111], + [0b111, 0b100, 0b111, 0b101, 0b111], + [0b111, 0b001, 0b010, 0b010, 0b010], + [0b111, 0b101, 0b111, 0b101, 0b111], + [0b111, 0b101, 0b111, 0b001, 0b111], +]; + +fn digit(img: &mut image::RgbImage, x: i64, y: i64, d: u8, c: [u8; 3]) { + let rows = DIGITS[(d as usize).min(9)]; + for (r, bits) in rows.iter().enumerate() { + for col in 0..DIGIT_W { + if bits >> (DIGIT_W - 1 - col) & 1 == 1 { + fill(img, x + col * SCALE, y + r as i64 * SCALE, x + (col + 1) * SCALE, y + (r as i64 + 1) * SCALE, c); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn el(kind: &str, name: &str, x: f64, y: f64, active: bool) -> Element { + Element { kind: kind.into(), name: name.into(), value: String::new(), x, y, w: 40.0, h: 20.0, active, flags: vec![], window: "W".into() } + } + + #[test] + fn controls_are_numbered_in_reading_order() { + let els = vec![el("按鈕", "下", 10.0, 300.0, true), el("連結", "Profile", 20.0, 100.0, true), el("文字", "標題", 0.0, 50.0, false), el("連結", "Profile", 20.0, 100.0, true)]; + let (numbered, lines, spots) = number(&els, false); + // Reading order, duplicates once, plain text left out when there is a picture. + assert_eq!(lines, vec!["[1] 連結「Profile」", "[2] 按鈕「下」"]); + assert_eq!(numbered.len(), 2); + assert_eq!(spots.get("1").unwrap(), ((40.0, 110.0), "連結「Profile」".to_string())); + assert_eq!(spots.get("e2").unwrap().0, (30.0, 310.0)); + assert!(spots.get("3").is_err() && spots.get("ok").is_err()); + // Without a picture the text is listed too (not numbered). + let (_, lines, _) = number(&els, true); + assert_eq!(lines, vec![" 文字「標題」", "[1] 連結「Profile」", "[2] 按鈕「下」"]); + } + + #[test] + fn numbers_are_drawn_on_the_picture() { + let mut img = image::RgbImage::from_pixel(200, 100, image::Rgb([255, 255, 255])); + let els = vec![el("按鈕", "OK", 100.0, 60.0, true)]; + let (numbered, _, _) = number(&els, false); + // Screen twice the picture's size: the box lands at (50, 30)–(70, 40). + draw(&mut img, &numbered, (2.0, 2.0)); + assert_eq!(img.get_pixel(50, 35).0, COLOURS[1]); + assert_eq!(img.get_pixel(60, 30).0, COLOURS[1]); + // The label sits above the box with white digits in it. + let label_has_white = (16..30).any(|y| (50..62).any(|x| img.get_pixel(x, y).0 == [255, 255, 255])); + let label_has_colour = (16..30).any(|y| (50..62).any(|x| img.get_pixel(x, y).0 == COLOURS[1])); + assert!(label_has_white && label_has_colour); + } + + #[test] + fn a_still_screen_has_the_same_fingerprint() { + let a = vec![el("按鈕", "OK", 1.0, 2.0, true)]; + let mut b = a.clone(); + assert_eq!(fingerprint(&a), fingerprint(&b)); + b[0].name = "Cancel".into(); + assert_ne!(fingerprint(&a), fingerprint(&b)); + let img = image::RgbImage::from_pixel(64, 40, image::Rgb([10, 20, 30])); + let mut img2 = img.clone(); + img2.put_pixel(0, 0, image::Rgb([12, 20, 30])); + assert_eq!(picture_fingerprint(&img), picture_fingerprint(&img2)); + } +} diff --git a/src/remote.rs b/src/remote.rs index b06644c..5878c5f 100644 --- a/src/remote.rs +++ b/src/remote.rs @@ -591,15 +591,9 @@ fn upload(bot: u64, req: &Req) -> Resp { fn screen(which: &str, s: &Settings) -> Resp { match which { "mac" if s.mac_from_phone => { - let (text, ok, img) = crate::computer::run_by("computer_use_screenshot", "{}", "手機"); - match (ok, img) { - (true, Some(b)) => { - use base64::Engine; - let bytes = base64::engine::general_purpose::STANDARD.decode(b).unwrap_or_default(); - let size = image::load_from_memory(&bytes).map(|i| format!("{}x{}", i.width(), i.height())).unwrap_or_default(); - Resp::bytes("image/jpeg", bytes).with_header("X-Size", &size) - } - _ => Resp::error(503, &text), + match crate::computer::plain_picture() { + Ok((w, h, jpeg)) => Resp::bytes("image/jpeg", jpeg).with_header("X-Size", &format!("{w}x{h}")), + Err(e) => Resp::error(503, &e), } } "mac" => Resp::error(403, "這台 Mac 沒有開放從手機操作（AnyBot 設定 › 手機與帳號）"), diff --git a/src/web.rs b/src/web.rs index b860586..e564069 100644 --- a/src/web.rs +++ b/src/web.rs @@ -84,7 +84,7 @@ fn specs_inner(browser: bool) -> Vec { "parameters": {"type": "object", "properties": {"url": {"type": "string"}}, "required": ["url"]}}), json!({"name": "browser_open", "description": "Open a link in AnyBot's own browser window (a separate browser profile the person logs in to once, for sites like Threads, Instagram, Facebook, Gmail). Waits a few seconds for it to load; then call browser_read. Never ask the person to change browser settings.", "parameters": {"type": "object", "properties": {"url": {"type": "string"}, "profile": profile}, "required": ["url"]}}), - json!({"name": "browser_read", "description": "Read the title, address and visible text of the current tab in AnyBot's browser (logged-in pages included), plus a numbered list of its links (with their addresses), buttons, fields and dropdowns. Use those numbers with browser_click / browser_type / browser_select instead of guessing URLs. AnyBot's browser runs on the person's computer, not on 「Bot 的電腦」: screenshots of 「Bot 的電腦」 never show it. If it shows a login page, ask the person to log in once in that window.", + json!({"name": "browser_read", "description": "Read the title, address and visible text of the current tab in AnyBot's browser (logged-in pages included), plus a numbered list of its links (with their addresses), buttons, fields and dropdowns. Use those numbers with browser_click / browser_type / browser_select instead of guessing URLs. Each computer has its own AnyBot browser with its own logins (on a bot's own computer it is the Chromium on its desktop). For anything on a web page, read it here and act by these numbers; never guess screen coordinates on a web page. If it shows a login page, hand the computer to the person (hand_over_computer) to log in once.", "parameters": {"type": "object", "properties": {"max_chars": {"type": "integer", "description": "Longest text to return (default 20000)."}, "profile": profile}}}), json!({"name": "browser_profiles", "description": "List, add or switch AnyBot browser profiles (設定檔/profile). Use when the person wants another profile or account: 'list' shows them, 'add' makes a new empty one (they log in once there) and switches to it, 'use' switches. Switching changes which profile browser_open/browser_read use.", "parameters": {"type": "object", "properties": {"action": {"type": "string", "enum": ["list", "add", "use"]}, "name": {"type": "string", "description": "Profile name for add/use."}}, "required": ["action"]}}), diff --git a/src/x11.rs b/src/x11.rs index 4cfd078..2252b22 100644 --- a/src/x11.rs +++ b/src/x11.rs @@ -153,19 +153,6 @@ pub fn click(px: f64, py: f64, button: &str, count: i64, flags: u64) { }); } -pub fn mouse_button(at: Option<(f64, f64)>, button: &str, press: bool) { - if let Some((px, py)) = at { - move_to(px, py); - gap(); - } - let b = button_no(button); - with(|x| { - fake(x, if press { xproto::BUTTON_PRESS_EVENT } else { xproto::BUTTON_RELEASE_EVENT }, b, 0, 0); - flush(x); - Some(()) - }); -} - pub fn drag(path: &[(f64, f64)], button: &str, flags: u64) { let (x0, y0) = path[0]; move_to(x0, y0);