E2E: remote screenshot, click and typing; a Linux Wayland run with a full runtime folder

remote_e2e.py: with E2E_SCREENSHOT=1 a screenshot of the hands comes back to the model as a
JPEG; with E2E_TYPE=1 also a click, typing (Chinese too) and Enter in a terminal there. The fake
model no longer takes a screenshot's picture message for the person speaking. E2E_APP picks the
app to open. deploy/test/wayland: the same test on a headless sway in Docker whose
XDG_RUNTIME_DIR is full (31/31).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

deploy/test/remote_e2e.py

@@ -5,7 +5,7 @@
 # Build first (cargo build --release; and in relay/), then, from app/:
 #   python3 -I deploy/test/remote_e2e.py .
 # On a Mac the terminal needs 輔助使用 (the hands open Calculator through agent-desktop).
-import json, os, shutil, socket, subprocess, sys, tempfile, threading, time, urllib.request, urllib.error
+import base64, json, os, re, shutil, socket, subprocess, sys, tempfile, threading, time, urllib.request, urllib.error
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 APP = os.path.abspath(sys.argv[1])
@@ -14,7 +14,14 @@ RELAY = os.path.join(APP, 'relay/target/release/anybot-relay')
 WORK = tempfile.mkdtemp(prefix='anybot-e2e-')
 USER, PW = 'e2euser', 'e2e-pass-12345'
 HANDS = 'E2E手腳'
+APP_NAME = os.environ.get('E2E_APP', 'Calculator')
+# E2E_SCREENSHOT=1: also a screenshot of the hands' screen, sent back to the brain (needs a
+# screen: on Linux a Wayland or X11 session, on a Mac 螢幕錄製 for the terminal).
+SHOT = os.environ.get('E2E_SCREENSHOT') == '1'
+# E2E_TYPE=1 (with E2E_SCREENSHOT): also click, type and press Enter in a terminal on the right
+# half of the hands' screen (the Linux Wayland test opens one there).
 results = []
+images = []
 
 def free_port():
     s = socket.socket(); s.bind(('127.0.0.1', 0)); p = s.getsockname()[1]; s.close(); return p
@@ -29,7 +36,14 @@ def tool_call(name, args):
 
 SCRIPTS = {
     'SC_LIST': [tool_call('list_computers', {})],
-    'SC_OPEN': [tool_call('computer_use_open_app', {'app': 'Calculator', 'computer': HANDS})],
+    'SC_OPEN': [tool_call('computer_use_open_app', {'app': APP_NAME, 'computer': HANDS})],
+    'SC_SHOT': [tool_call('computer_use_screenshot', {'computer': HANDS})],
+    # Mouse and keyboard there: click the terminal on the right, type a command (Chinese too), Enter.
+    'SC_TYPE': [tool_call('computer_use_screenshot', {'computer': HANDS}),
+                tool_call('computer_use_click', {'x': 768, 'y': 300, 'computer': HANDS}),
+                tool_call('computer_use_typing', {'value': 'echo 打字-$((3*4)) > /tmp/e2e-typed.txt', 'computer': HANDS}),
+                tool_call('computer_use_press_key', {'key': 'return', 'computer': HANDS}),
+                tool_call('shell', {'command': 'sleep 1; cat /tmp/e2e-typed.txt', 'computer': HANDS})],
     'SC_SHELL': [tool_call('shell', {'command': 'echo e2e-$((6*7))', 'computer': HANDS})],
     'SC_REDIRECT': [tool_call('shell', {'command': 'echo moved-here', 'computer': HANDS})],
     'SC_HANDOFF': [tool_call('hand_over_computer', {'instruction': 'E2E：請按「我完成了」', 'computer': HANDS})],
@@ -45,9 +59,15 @@ class Model(BaseHTTPRequestHandler):
         self.send_response(200); self.send_header('Content-Type', 'application/json'); self.send_header('Content-Length', str(len(body))); self.end_headers(); self.wfile.write(body)
     def do_POST(self):
         n = int(self.headers.get('Content-Length', '0') or 0)
-        req = json.loads(self.rfile.read(n) or b'{}')
+        raw = self.rfile.read(n) or b'{}'
+        req = json.loads(raw)
+        # Pictures the model was shown (a screenshot from another computer comes back as one).
+        for m in re.finditer(rb'data:image/jpeg;base64,([A-Za-z0-9+/=]+)', raw):
+            images.append(base64.b64decode(m.group(1)))
         msgs = req.get('messages', [])
-        last_user = max((i for i, m in enumerate(msgs) if m.get('role') == 'user'), default=-1)
+        # The person's last message (a screenshot comes back as a user message of pictures: not them).
+        said = lambda m: m.get('role') == 'user' and not (isinstance(m.get('content'), list) and any(isinstance(p, dict) and p.get('type') == 'image_url' for p in m['content']))
+        last_user = max((i for i, m in enumerate(msgs) if said(m)), default=-1)
         text = msgs[last_user].get('content', '') if last_user >= 0 else ''
         if isinstance(text, list):
             text = ' '.join(p.get('text', '') for p in text if isinstance(p, dict))
@@ -250,6 +270,22 @@ def run():
     wait_turn('once')
     t = ' '.join(tool_text_after('SC_SHELL'))
     check('shell runs on the hands', 'e2e-42' in t and f'[{HANDS}]' in t, t[:300])
+    # 3b. A screenshot there, back to the brain and on to the model as a picture.
+    if SHOT:
+        before = len(images)
+        sync('POST', '/act', {'bot': 1, 'action': 'send', 'text': '看一下我的螢幕 SC_SHOT'})
+        wait_turn('once')
+        t = ' '.join(tool_text_after('SC_SHOT'))
+        check('screenshot runs on the hands', f'[{HANDS}]' in t and '螢幕截圖（' in t, t[:300])
+        pics = [b for b in images[before:] if b[:2] == b'\xff\xd8' and len(b) > 1000]
+        if pics:
+            open(os.path.join(WORK, 'shot.jpg'), 'wb').write(pics[-1])
+        check('the model gets the screenshot as a JPEG', bool(pics), f'{len(images) - before} pictures; saved {os.path.join(WORK, "shot.jpg")}')
+        if os.environ.get('E2E_TYPE') == '1':
+            sync('POST', '/act', {'bot': 1, 'action': 'send', 'text': '在我的電腦打字 SC_TYPE'})
+            wait_turn('once')
+            t = ' '.join(tool_text_after('SC_TYPE'))
+            check('click, typing (Chinese too) and Enter reach the hands', '打字-12' in t, t[-400:])
     # 5. Its own computer.
     sync('POST', '/act', {'bot': 1, 'action': 'send', 'text': '用你自己的 SC_OWN'})
     wait_turn('once')

deploy/test/wayland/Dockerfile

@@ -0,0 +1,16 @@
+# The remote end-to-end test (../remote_e2e.py) on a Linux Wayland desktop: a headless sway in
+# Docker whose runtime folder (XDG_RUNTIME_DIR) is full, as on a desktop that once crashed when
+# AnyBot took a screenshot. Brain, relay and devices are the server images' own builds.
+# On the server, after `docker compose build` in deploy/server, from app/:
+#   docker build -t anybot-wl-e2e deploy/test/wayland
+#   docker run --rm --user 1000:1000 --tmpfs /run/xdg:size=4m,mode=0700,uid=1000,gid=1000 \
+#     -v $PWD/deploy:/e2e/app/deploy:ro anybot-wl-e2e dbus-run-session -- /e2e/app/deploy/test/wayland/run.sh
+# (add -v <an older anybot>:/old/anybot:ro to see how that one does with the full folder)
+FROM server-relay:latest AS relay
+FROM server-anybot:latest
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends sway foot python3 dbus at-spi2-core fonts-dejavu-core procps && rm -rf /var/lib/apt/lists/*
+COPY --from=relay /usr/local/bin/anybot-relay /usr/local/bin/anybot-relay
+RUN mkdir -p /e2e/app/target/release /e2e/app/relay/target/release \
+ && ln -s /usr/local/bin/anybot /e2e/app/target/release/anybot \
+ && ln -s /usr/local/bin/anybot-relay /e2e/app/relay/target/release/anybot-relay
+RUN (getent passwd 1000 >/dev/null || useradd -u 1000 -m tester)

deploy/test/wayland/run.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# Inside the container: a headless sway desktop with a full runtime folder (like the crashed one).
+export XDG_RUNTIME_DIR=/run/xdg HOME=/tmp/home
+mkdir -p $HOME
+export WLR_BACKENDS=headless WLR_LIBINPUT_NO_DEVICES=1 WLR_RENDERER=pixman
+sway -c /e2e/app/deploy/test/wayland/sway.conf > /tmp/sway.log 2>&1 &
+SWAY=$!
+for i in $(seq 1 50); do ls $XDG_RUNTIME_DIR | grep -q '^wayland-[0-9]*$' && break; sleep 0.2; done
+export WAYLAND_DISPLAY=$(ls $XDG_RUNTIME_DIR | grep '^wayland-[0-9]*$' | head -1)
+sleep 2
+dd if=/dev/zero of=$XDG_RUNTIME_DIR/filler bs=64k 2>/dev/null
+echo "== runtime folder: $(df -h $XDG_RUNTIME_DIR | tail -1)   display=$WAYLAND_DISPLAY"
+alive() { kill -0 $SWAY 2>/dev/null && echo "   sway still running" || { echo "   SWAY DIED"; tail -5 /tmp/sway.log; }; }
+shot() {
+  timeout 30 "$1" --tool computer_use_screenshot '{}' > /tmp/shot-$2.txt 2>&1
+  echo "== $2 build, screenshot with the folder full: exit $? :: $(head -c 160 /tmp/shot-$2.txt | tr '\n' ' ')"
+  alive
+}
+shot /usr/local/bin/anybot new
+echo "== full E2E (brain + relay + 2 devices), with a remote screenshot"
+cd /e2e/app && E2E_SCREENSHOT=1 E2E_TYPE=1 E2E_APP=foot python3 -I deploy/test/remote_e2e.py /e2e/app 2>&1 | tail -40
+alive
+[ -d /out ] && cp /tmp/anybot-e2e-*/shot.jpg /out/ 2>/dev/null
+if [ -x /old/anybot ]; then shot /old/anybot old; fi

deploy/test/wayland/sway.conf

@@ -0,0 +1,2 @@
+output HEADLESS-1 resolution 1280x800
+exec foot -e sh -c 'echo AnyBot Wayland E2E; sleep 100000'