Screen steps name their controls first, and a batch takes a whole row of keys; four held-out tasks

A quick model looking numbers up pressed 1, 2, 3, 1 for 1234, and 1234 × 5678 = didn't fit in
an 8-action batch (it split it and lost a key). The click and batch descriptions now lead with
the control's name (name "4"), and a batch takes up to 16 actions.

The benchmark gets four tasks on things none of the others use (Mousepad and its save dialog,
a Chromium bookmark, ImageMagick, a page of nested web components), run only when asked for
(--tasks N1,…): a check that the changes carry over to what they weren't tuned on.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS

bench/computer-use/run.py

@@ -116,6 +116,46 @@ TASKS = [
      "check": lambda dh, ans: form_code() in ans},
 ]
 
+COMPONENTS = r"""<!doctype html><meta charset=utf-8><title>設定</title>
+<h1>帳號設定</h1><app-settings></app-settings><p id=out>尚未儲存</p>
+<script>
+customElements.define('lang-pick', class extends HTMLElement { constructor() { super(); const r = this.attachShadow({mode: 'open'});
+  r.innerHTML = '<label>語言 <select id=l><option>中文</option><option>English</option><option>日本語</option></select></label>'; }
+  get value() { return this.shadowRoot.getElementById('l').value; } });
+customElements.define('app-settings', class extends HTMLElement { constructor() { super(); const r = this.attachShadow({mode: 'open'});
+  r.innerHTML = '<div role=switch tabindex=0 aria-label="深色模式" aria-checked=false style="display:inline-block;width:44px;height:22px;background:#ccc;cursor:pointer"></div> 深色模式<br><lang-pick></lang-pick><br><button id=save>儲存</button>';
+  const sw = r.querySelector('[role=switch]'); sw.addEventListener('click', () => sw.setAttribute('aria-checked', sw.getAttribute('aria-checked') === 'true' ? 'false' : 'true'));
+  r.getElementById('save').addEventListener('click', () => { const dark = sw.getAttribute('aria-checked') === 'true'; const lang = r.querySelector('lang-pick').value;
+    const code = (dark ? 'D' : 'L') + ({'中文': 'Z', 'English': 'E', '日本語': 'J'})[lang] + '7Q2';
+    document.getElementById('out').textContent = '已儲存，代碼：' + code; }); } });
+</script>"""
+
+# Held out: apps, tools and page shapes none of the tasks above use (and nothing was tuned on).
+HELDOUT = [
+    {"id": "N1", "route": "screen", "name": "Mousepad 存新檔",
+     "setup": "(command -v mousepad >/dev/null || (apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq mousepad >/dev/null)); rm -f /home/bot/bench/hello.txt; mkdir -p /home/bot/bench; pkill -x mousepad; true",
+     "ask": "用你電腦上的文字編輯器 Mousepad 開一個新檔，打「你好，AnyBot」，存成 ~/bench/hello.txt。",
+     "check": lambda dh, ans: "你好，AnyBot" in dx("cat ~/bench/hello.txt 2>/dev/null", dh)[1]},
+    {"id": "N2", "route": "screen", "name": "Chromium 加書籤",
+     "setup": "pkill -f '^/usr/lib/chromium/chromium'; sleep 2; python3 - <<'PY'\n"
+              "import json\nfor p in ['/home/bot/.config/anybot-profile/Default/Bookmarks']:\n"
+              "    try:\n        d = json.load(open(p))\n    except Exception:\n        continue\n"
+              "    for root in d.get('roots', {}).values():\n        if isinstance(root, dict) and 'children' in root:\n            root['children'] = [c for c in root['children'] if 'example.com' not in c.get('url', '')]\n"
+              "    d.pop('checksum', None); json.dump(d, open(p, 'w'))\nPY",
+     "ask": "在你電腦的 Chromium 打開 https://example.com，把這一頁加入書籤。",
+     "check": lambda dh, ans: "example.com" in dx("for i in $(seq 15); do grep -q example.com ~/.config/anybot-profile/Default/Bookmarks 2>/dev/null && { echo example.com; break; }; sleep 1; done", dh)[1]},
+    {"id": "N3", "route": "command", "name": "ImageMagick 紅色圖",
+     "setup": "rm -f /home/bot/bench/red.png; mkdir -p /home/bot/bench",
+     "ask": "幫我在你的電腦上做一張 200×100 像素的紅色 PNG 圖片，存成 ~/bench/red.png，再告訴我圖片的尺寸。",
+     "check": lambda dh, ans: dx("identify -format '%wx%h ' ~/bench/red.png && convert ~/bench/red.png -format '%[pixel:p{10,10}]' info:", dh)[1].replace(" ", "").lower() in ("200x100srgb(255,0,0)", "200x100red")},
+    {"id": "N4", "route": "browser", "name": "Web components 設定頁",
+     "setup": "mkdir -p /home/bot/bench/comp && cat > /home/bot/bench/comp/index.html <<'EOF'\n" + COMPONENTS + "\nEOF\n"
+              "pkill -f '^python3 -m http.server 8771'; sleep 0.5; cd /home/bot/bench/comp && (su bot -c 'setsid python3 -m http.server 8771 --bind 127.0.0.1 >/dev/null 2>&1 &'); "
+              "for i in 1 2 3 4 5 6 7 8 9 10; do curl -s -o /dev/null http://127.0.0.1:8771/ && break; sleep 0.5; done",
+     "ask": "到 http://127.0.0.1:8771 把深色模式打開、語言改成日本語，按儲存，告訴我頁面上顯示的代碼。",
+     "check": lambda dh, ans: "DJ7Q2" in ans},
+]
+
 ROUTE = [("command", ("shell", "read_file", "write_file", "edit_file", "list_dir", "search", "read_terminal", "write_terminal")),
          ("screen", ("computer_use_",)),
          ("browser", ("browser_", "web_fetch"))]
@@ -187,7 +227,7 @@ def main():
     if os.path.realpath(a.data).startswith(real):
         sys.exit("--data must be a scratch copy, not the real data folder")
     want = {t.strip() for t in a.tasks.split(",") if t.strip()}
-    tasks = [t for t in TASKS if not want or t["id"] in want]
+    tasks = [t for t in TASKS + HELDOUT if t["id"] in want] if want else TASKS
     results = json.load(open(a.out)) if os.path.exists(a.out) else []
     env = dict(os.environ, ANYBOT_DATA_DIR=a.data, ANYBOT_ASK_SECS=str(a.secs))
     kept_skills = []

src/computer.rs

@@ -19,7 +19,7 @@ pub const NAMES: [&str; 12] = ["computer_use_screenshot", "computer_use_zoom", "
 pub const SETTLE_MS: u64 = 500;
 /// Most actions one `computer_use_batch` call may carry (Anthropic's computer_batch guidance:
 /// short self-contained sequences, one picture at the end).
-pub const MAX_BATCH: usize = 8;
+pub const MAX_BATCH: usize = 16;
 
 /// A batch step's short action name → the tool it runs. Screen reads, apps and permissions stay
 /// single steps on purpose: a batch is for input you already planned from the current picture.
@@ -293,7 +293,7 @@ pub fn specs_for(computers: &[String], default: &str, chosen: bool) -> Vec<Value
             "parameters": {"type": "object", "properties": {"x": {"type": "number", "description": "Left edge, in the latest picture's pixels"}, "y": {"type": "number", "description": "Top edge"}, "width": {"type": "number"}, "height": {"type": "number"}}, "required": ["x", "y", "width", "height"]}}),
         json!({"name": "computer_use_show", "description": "Show the person a computer's screen: a picture of it as they would see it (no numbers) goes into the chat for them, and you see it too. Use it when they ask to see a screen (截圖給我看、讓我看看畫面). One call, no file needed.",
             "parameters": {"type": "object", "properties": {}}}),
-        json!({"name": "computer_use_click", "description": "Click a control by its number (ref) from the latest picture. Only for something without a number (a spot in a picture or game): x, y in the picture's pixels. Returns the next picture.",
+        json!({"name": "computer_use_click", "description": "Click a control: by its name as the latest list shows it (name, e.g. 4, Close, 自動更新 — easiest, no number to look up), or by its number (ref) when two share a name. Only for something with neither (a spot in a picture or game): x, y in the picture's pixels. Returns the next picture.",
             "parameters": {"type": "object", "properties": {"ref": rf, "name": named, "x": {"type": "number"}, "y": {"type": "number"}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "count": {"type": "integer", "description": "2 for a double click"}, "modifier_keys": mods}}}),
         json!({"name": "computer_use_typing", "description": "Type text (any language) into a field by its number (ref; it is clicked first), or where the cursor is. submit=true presses return after it (send, search, run a command). Returns the next picture.",
             "parameters": {"type": "object", "properties": {"value": {"type": "string"}, "ref": rf, "name": named, "submit": {"type": "boolean"}}, "required": ["value"]}}),
@@ -309,7 +309,7 @@ pub fn specs_for(computers: &[String], default: &str, chosen: bool) -> Vec<Value
             "parameters": {"type": "object", "properties": {"path": {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "button": {"type": "string", "enum": ["left", "right"]}, "modifier_keys": mods}, "required": ["path"]}}),
         json!({"name": "computer_use_check_permissions", "description": "Check whether the screen can be seen and the mouse and keyboard used on that computer.",
             "parameters": {"type": "object", "properties": {}}}),
-        json!({"name": "computer_use_batch", "description": format!("Do a short planned sequence of steps in one call (max {MAX_BATCH}): each item is one action with its own arguments, e.g. {{\"action\":\"click\",\"ref\":\"12\"}}, {{\"action\":\"typing\",\"ref\":\"15\",\"value\":\"cat\"}}, {{\"action\":\"press_key\",\"key\":\"tab\"}}. Use it when the current picture already tells you every step (filling a form, a known shortcut chain): they run in order, the first step that fails stops the rest, and you get one picture at the end. Do not use it to explore, or where step 2 depends on what step 1's screen shows — act one step at a time there."),
+        json!({"name": "computer_use_batch", "description": format!("Do a planned sequence of steps in one call (max {MAX_BATCH}): each item is one action with its own arguments, best naming the control, e.g. {{\"action\":\"click\",\"name\":\"7\"}}, {{\"action\":\"typing\",\"name\":\"Search\",\"value\":\"cat\"}}, {{\"action\":\"press_key\",\"key\":\"tab\"}}. Use it when the current picture already tells you every step (a row of keys to press, filling a form, a known shortcut chain): put the whole sequence in one call, they run in order, the first step that fails stops the rest, and you get one picture at the end. Several separate clicks in one step are slower and easier to get wrong than one batch. Do not use it to explore, or where step 2 depends on what step 1's screen shows — act one step at a time there."),
             "parameters": {"type": "object", "properties": {"actions": {"type": "array", "maxItems": MAX_BATCH, "items": {"type": "object", "properties": {"action": {"type": "string", "enum": ["click", "typing", "press_key", "scroll", "mouse_move", "drag"]}, "ref": rf.clone(), "name": named.clone(), "x": {"type": "number"}, "y": {"type": "number"}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "count": {"type": "integer"}, "value": {"type": "string"}, "submit": {"type": "boolean"}, "key": {"type": "string"}, "hold_duration_ms": {"type": "integer"}, "direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "amount": {"type": "integer"}, "path": {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "modifier_keys": mods.clone()}, "required": ["action"]}}}, "required": ["actions"]}}),
     ];
     // Always named: each computer goes by its own name (the instructions say which is which),