From c87d598e64102e99b11e45100a79756f3db4d73f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=80=A7=E9=A9=8A?= Date: Thu, 5 Nov 2026 20:28:52 +0800 Subject: [PATCH] E2E: remote screenshot, click and typing; a Linux Wayland run with a full runtime folder remote_e2e.py: with E2E_SCREENSHOT=1 a screenshot of the hands comes back to the model as a JPEG; with E2E_TYPE=1 also a click, typing (Chinese too) and Enter in a terminal there. The fake model no longer takes a screenshot's picture message for the person speaking. E2E_APP picks the app to open. deploy/test/wayland: the same test on a headless sway in Docker whose XDG_RUNTIME_DIR is full (31/31). Co-Authored-By: Claude Opus 5.5 --- deploy/test/remote_e2e.py | 44 ++++++++++++++++++++++++++++++---- deploy/test/wayland/Dockerfile | 16 +++++++++++++ deploy/test/wayland/run.sh | 24 +++++++++++++++++++ deploy/test/wayland/sway.conf | 2 ++ 4 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 deploy/test/wayland/Dockerfile create mode 100755 deploy/test/wayland/run.sh create mode 100644 deploy/test/wayland/sway.conf diff --git a/deploy/test/remote_e2e.py b/deploy/test/remote_e2e.py index a367f70..c356351 100644 --- a/deploy/test/remote_e2e.py +++ b/deploy/test/remote_e2e.py @@ -5,7 +5,7 @@ # Build first (cargo build --release; and in relay/), then, from app/: # python3 -I deploy/test/remote_e2e.py . # On a Mac the terminal needs 輔助使用 (the hands open Calculator through agent-desktop). -import json, os, shutil, socket, subprocess, sys, tempfile, threading, time, urllib.request, urllib.error +import base64, json, os, re, shutil, socket, subprocess, sys, tempfile, threading, time, urllib.request, urllib.error from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer APP = os.path.abspath(sys.argv[1]) @@ -14,7 +14,14 @@ RELAY = os.path.join(APP, 'relay/target/release/anybot-relay') WORK = tempfile.mkdtemp(prefix='anybot-e2e-') USER, PW = 'e2euser', 'e2e-pass-12345' HANDS = 'E2E手腳' +APP_NAME = os.environ.get('E2E_APP', 'Calculator') +# E2E_SCREENSHOT=1: also a screenshot of the hands' screen, sent back to the brain (needs a +# screen: on Linux a Wayland or X11 session, on a Mac 螢幕錄製 for the terminal). +SHOT = os.environ.get('E2E_SCREENSHOT') == '1' +# E2E_TYPE=1 (with E2E_SCREENSHOT): also click, type and press Enter in a terminal on the right +# half of the hands' screen (the Linux Wayland test opens one there). results = [] +images = [] def free_port(): s = socket.socket(); s.bind(('127.0.0.1', 0)); p = s.getsockname()[1]; s.close(); return p @@ -29,7 +36,14 @@ def tool_call(name, args): SCRIPTS = { 'SC_LIST': [tool_call('list_computers', {})], - 'SC_OPEN': [tool_call('computer_use_open_app', {'app': 'Calculator', 'computer': HANDS})], + 'SC_OPEN': [tool_call('computer_use_open_app', {'app': APP_NAME, 'computer': HANDS})], + 'SC_SHOT': [tool_call('computer_use_screenshot', {'computer': HANDS})], + # Mouse and keyboard there: click the terminal on the right, type a command (Chinese too), Enter. + 'SC_TYPE': [tool_call('computer_use_screenshot', {'computer': HANDS}), + tool_call('computer_use_click', {'x': 768, 'y': 300, 'computer': HANDS}), + tool_call('computer_use_typing', {'value': 'echo 打字-$((3*4)) > /tmp/e2e-typed.txt', 'computer': HANDS}), + tool_call('computer_use_press_key', {'key': 'return', 'computer': HANDS}), + tool_call('shell', {'command': 'sleep 1; cat /tmp/e2e-typed.txt', 'computer': HANDS})], 'SC_SHELL': [tool_call('shell', {'command': 'echo e2e-$((6*7))', 'computer': HANDS})], 'SC_REDIRECT': [tool_call('shell', {'command': 'echo moved-here', 'computer': HANDS})], 'SC_HANDOFF': [tool_call('hand_over_computer', {'instruction': 'E2E：請按「我完成了」', 'computer': HANDS})], @@ -45,9 +59,15 @@ class Model(BaseHTTPRequestHandler): self.send_response(200); self.send_header('Content-Type', 'application/json'); self.send_header('Content-Length', str(len(body))); self.end_headers(); self.wfile.write(body) def do_POST(self): n = int(self.headers.get('Content-Length', '0') or 0) - req = json.loads(self.rfile.read(n) or b'{}') + raw = self.rfile.read(n) or b'{}' + req = json.loads(raw) + # Pictures the model was shown (a screenshot from another computer comes back as one). + for m in re.finditer(rb'data:image/jpeg;base64,([A-Za-z0-9+/=]+)', raw): + images.append(base64.b64decode(m.group(1))) msgs = req.get('messages', []) - last_user = max((i for i, m in enumerate(msgs) if m.get('role') == 'user'), default=-1) + # The person's last message (a screenshot comes back as a user message of pictures: not them). + said = lambda m: m.get('role') == 'user' and not (isinstance(m.get('content'), list) and any(isinstance(p, dict) and p.get('type') == 'image_url' for p in m['content'])) + last_user = max((i for i, m in enumerate(msgs) if said(m)), default=-1) text = msgs[last_user].get('content', '') if last_user >= 0 else '' if isinstance(text, list): text = ' '.join(p.get('text', '') for p in text if isinstance(p, dict)) @@ -250,6 +270,22 @@ def run(): wait_turn('once') t = ' '.join(tool_text_after('SC_SHELL')) check('shell runs on the hands', 'e2e-42' in t and f'[{HANDS}]' in t, t[:300]) + # 3b. A screenshot there, back to the brain and on to the model as a picture. + if SHOT: + before = len(images) + sync('POST', '/act', {'bot': 1, 'action': 'send', 'text': '看一下我的螢幕 SC_SHOT'}) + wait_turn('once') + t = ' '.join(tool_text_after('SC_SHOT')) + check('screenshot runs on the hands', f'[{HANDS}]' in t and '螢幕截圖（' in t, t[:300]) + pics = [b for b in images[before:] if b[:2] == b'\xff\xd8' and len(b) > 1000] + if pics: + open(os.path.join(WORK, 'shot.jpg'), 'wb').write(pics[-1]) + check('the model gets the screenshot as a JPEG', bool(pics), f'{len(images) - before} pictures; saved {os.path.join(WORK, "shot.jpg")}') + if os.environ.get('E2E_TYPE') == '1': + sync('POST', '/act', {'bot': 1, 'action': 'send', 'text': '在我的電腦打字 SC_TYPE'}) + wait_turn('once') + t = ' '.join(tool_text_after('SC_TYPE')) + check('click, typing (Chinese too) and Enter reach the hands', '打字-12' in t, t[-400:]) # 5. Its own computer. sync('POST', '/act', {'bot': 1, 'action': 'send', 'text': '用你自己的 SC_OWN'}) wait_turn('once') diff --git a/deploy/test/wayland/Dockerfile b/deploy/test/wayland/Dockerfile new file mode 100644 index 0000000..372b12e --- /dev/null +++ b/deploy/test/wayland/Dockerfile @@ -0,0 +1,16 @@ +# The remote end-to-end test (../remote_e2e.py) on a Linux Wayland desktop: a headless sway in +# Docker whose runtime folder (XDG_RUNTIME_DIR) is full, as on a desktop that once crashed when +# AnyBot took a screenshot. Brain, relay and devices are the server images' own builds. +# On the server, after `docker compose build` in deploy/server, from app/: +# docker build -t anybot-wl-e2e deploy/test/wayland +# docker run --rm --user 1000:1000 --tmpfs /run/xdg:size=4m,mode=0700,uid=1000,gid=1000 \ +# -v $PWD/deploy:/e2e/app/deploy:ro anybot-wl-e2e dbus-run-session -- /e2e/app/deploy/test/wayland/run.sh +# (add -v :/old/anybot:ro to see how that one does with the full folder) +FROM server-relay:latest AS relay +FROM server-anybot:latest +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends sway foot python3 dbus at-spi2-core fonts-dejavu-core procps && rm -rf /var/lib/apt/lists/* +COPY --from=relay /usr/local/bin/anybot-relay /usr/local/bin/anybot-relay +RUN mkdir -p /e2e/app/target/release /e2e/app/relay/target/release \ + && ln -s /usr/local/bin/anybot /e2e/app/target/release/anybot \ + && ln -s /usr/local/bin/anybot-relay /e2e/app/relay/target/release/anybot-relay +RUN (getent passwd 1000 >/dev/null || useradd -u 1000 -m tester) diff --git a/deploy/test/wayland/run.sh b/deploy/test/wayland/run.sh new file mode 100755 index 0000000..434313e --- /dev/null +++ b/deploy/test/wayland/run.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Inside the container: a headless sway desktop with a full runtime folder (like the crashed one). +export XDG_RUNTIME_DIR=/run/xdg HOME=/tmp/home +mkdir -p $HOME +export WLR_BACKENDS=headless WLR_LIBINPUT_NO_DEVICES=1 WLR_RENDERER=pixman +sway -c /e2e/app/deploy/test/wayland/sway.conf > /tmp/sway.log 2>&1 & +SWAY=$! +for i in $(seq 1 50); do ls $XDG_RUNTIME_DIR | grep -q '^wayland-[0-9]*$' && break; sleep 0.2; done +export WAYLAND_DISPLAY=$(ls $XDG_RUNTIME_DIR | grep '^wayland-[0-9]*$' | head -1) +sleep 2 +dd if=/dev/zero of=$XDG_RUNTIME_DIR/filler bs=64k 2>/dev/null +echo "== runtime folder: $(df -h $XDG_RUNTIME_DIR | tail -1) display=$WAYLAND_DISPLAY" +alive() { kill -0 $SWAY 2>/dev/null && echo " sway still running" || { echo " SWAY DIED"; tail -5 /tmp/sway.log; }; } +shot() { + timeout 30 "$1" --tool computer_use_screenshot '{}' > /tmp/shot-$2.txt 2>&1 + echo "== $2 build, screenshot with the folder full: exit $? :: $(head -c 160 /tmp/shot-$2.txt | tr '\n' ' ')" + alive +} +shot /usr/local/bin/anybot new +echo "== full E2E (brain + relay + 2 devices), with a remote screenshot" +cd /e2e/app && E2E_SCREENSHOT=1 E2E_TYPE=1 E2E_APP=foot python3 -I deploy/test/remote_e2e.py /e2e/app 2>&1 | tail -40 +alive +[ -d /out ] && cp /tmp/anybot-e2e-*/shot.jpg /out/ 2>/dev/null +if [ -x /old/anybot ]; then shot /old/anybot old; fi diff --git a/deploy/test/wayland/sway.conf b/deploy/test/wayland/sway.conf new file mode 100644 index 0000000..373c980 --- /dev/null +++ b/deploy/test/wayland/sway.conf @@ -0,0 +1,2 @@ +output HEADLESS-1 resolution 1280x800 +exec foot -e sh -c 'echo AnyBot Wayland E2E; sleep 100000'