People watch and help on the bots' computer through VNC (noVNC), carried by the account server

The core: the bot works the computer; a person watches it like a colleague, and takes over
when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on
phones), so the pictures sent one by one go, and with them their own input path.

- vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends
  with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and
  offers the viewer no password, so the password never leaves the brain. The person's clicks
  and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box
  (any language) is pasted in the box.
- Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled);
  /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the
  app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that
  AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection
  wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to
  160 ms more than needed.
- Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on
  the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu.
- Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by
  ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the
  live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone.
- Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window
  through accessibility and failed without 輔助使用 now and then.

Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer
signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median;
headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Cargo.lock

@@ -345,6 +345,7 @@ dependencies = [
  "chrono-tz",
  "cpal",
  "crossterm",
+ "des",
  "dirs",
  "eframe",
  "flate2",
@@ -1356,6 +1357,15 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "des"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e"
+dependencies = [
+ "cipher",
+]
+
 [[package]]
 name = "digest"
 version = "0.10.7"

Cargo.toml

@@ -25,6 +25,8 @@ serde_json = "1"
 dirs = "7"
 image = { version = "0.25", default-features = false, features = ["png", "jpeg", "gif", "webp", "bmp", "tiff", "ico"] }
 base64 = "0.22"
+# VNC sign-in (the 16-byte challenge) for people watching the bots' computer.
+des = "0.8"
 chrono = "0.4"
 chrono-tz = "0.10"
 iana-time-zone = "0.1"

mobile/index.html

@@ -272,7 +272,7 @@ a.plink{color:var(--accent);text-decoration:underline;text-underline-offset:3px;
   <section class="view" id="v-screen">
     <header>
       <button class="icon-btn" id="b-back2"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M15 5l-7 7 7 7"/></svg></button>
-      <div class="seg" id="seg"><button data-w="box" class="on">Bot 的電腦</button><button data-w="mac">這台 Mac</button></div>
+      <div class="title">這台 Mac</div>
       <span style="flex:1"></span>
       <span class="sub" id="screen-live"></span>
     </header>
@@ -287,7 +287,6 @@ a.plink{color:var(--accent);text-decoration:underline;text-underline-offset:3px;
       <button class="btn" data-scroll="up">↑ 捲動</button>
       <button class="btn" data-scroll="down">↓ 捲動</button>
       <button class="btn" id="k-right">右鍵：關</button>
-      <button class="btn" id="k-wake">叫醒</button>
     </div>
   </section>
 </div>
@@ -295,6 +294,7 @@ a.plink{color:var(--accent);text-decoration:underline;text-underline-offset:3px;
 <div class="sheet-bg" id="sheet"><div class="sheet">
   <button id="m-notify">開啟通知</button>
   <div class="note" id="m-note">Bot 需要你允許、或在你不在時做完事，手機會跳通知。</div>
+  <button id="m-mac" style="display:none">看這台 Mac 的螢幕</button>
   <button id="m-pair" style="display:none">加一台電腦（Linux、伺服器）</button>
   <button id="m-host" style="display:none">切換到別台 AnyBot</button>
   <button class="no" id="m-out">登出這支手機</button>
@@ -692,7 +692,7 @@ function renderAsk() {
   // 需要處理 (a step only you can do on a computer): its screen is one tap away, then 我完成了 / 跳過.
   if (a.kind === "handoff") {
     $("ask").innerHTML = `<div class="ask"><div class="q">需要處理 · ${esc(a.computer || "")}</div><div class="why">${esc(a.reason)}</div><div class="btns"><button class="btn" data-screen="1">接管</button><button class="btn main" data-a="once">我完成了</button><button class="btn no" data-a="reject" title="取消此要求且不執行該步驟；Bot 會跳過它並繼續">跳過</button></div></div>`;
-    $("ask").querySelector("[data-screen]").onclick = () => { S.back = S.view; show("screen"); $("seg").children[0].textContent = ownComputer(); $("seg").children[1].style.display = S.mac ? "" : "none"; startFrames(); };
+    $("ask").querySelector("[data-screen]").onclick = () => openBox(true);
   } else {
   $("ask").innerHTML = `<div class="ask"><div class="q">${esc(a.title)}</div><div class="why">${esc(a.reason)}</div><div class="btns"><button class="btn main" data-a="once">允許一次</button>${a.can_always ? `<button class="btn" data-a="always">一律允許</button>` : ""}<button class="btn no" data-a="reject">拒絕</button></div></div>`;
   }
@@ -737,36 +737,33 @@ function renderChips() {
 // ---------- screens ----------
 // Each bot's own computer goes by the bot's name (「Master 的電腦」).
 function ownComputer() { const n = botName(S.open); return n ? n + " 的電腦" : "Bot 的電腦"; }
-$("b-screen").onclick = $("b-screen2").onclick = () => { S.back = S.view; show("screen"); $("seg").children[0].textContent = ownComputer(); $("seg").children[1].style.display = S.mac ? "" : "none"; startFrames(); };
+// The bots' computer: live in the viewer page (VNC), where you can also take over.
+function openBox(control) {
+  const at = S.relay ? "host=" + encodeURIComponent(S.host) : "lan=1";
+  location.href = "screen#" + at + "&name=" + encodeURIComponent(ownComputer()) + (control ? "&control=1" : "");
+}
+$("b-screen").onclick = $("b-screen2").onclick = () => openBox(false);
+$("m-mac").onclick = () => { $("sheet").classList.remove("on"); S.back = S.view; S.screen = "mac"; show("screen"); startFrames(); };
 $("b-back2").onclick = () => show(S.back === "chat" ? "chat" : "list");
-document.querySelectorAll("#seg button").forEach((b) => b.onclick = () => { document.querySelectorAll("#seg button").forEach((x) => x.classList.toggle("on", x === b)); S.screen = b.dataset.w; $("k-wake").style.display = S.screen === "box" ? "" : "none"; startFrames(); });
+// This Mac's screen: a picture at a time.
-// The bots' computer sends a picture the moment its screen changes: ask for the one after the
-// picture shown (X-Seq), again as soon as it arrives. (It was one picture a second, so a tap
-// showed its result a second or two later.) This computer's own screen: a picture at a time.
 let lastUrl = null;
 const nap = (ms) => new Promise((r) => setTimeout(r, ms));
 function startFrames() { stopFrames(); const run = S.frameRun = {}; frames(run); }
 function stopFrames() { S.frameRun = null; }
 async function frames(run) {
-  let seq = 0;
   while (S.frameRun === run) {
     if (document.hidden) { await nap(800); continue; }
     try {
-      const live = S.screen === "box";
+      const r = await api(base() + "api/screen/mac", {raw: true});
-      const r = await api(base() + "api/screen/" + S.screen + (live ? "?after=" + seq : ""), {raw: true});
       if (S.frameRun !== run) return;
       if (!r.ok) { const v = await r.json().catch(() => ({})); $("stage").innerHTML = `<div class="msg">${esc(v.error || "看不到畫面")}</div>`; $("screen-live").textContent = ""; await nap(1500); continue; }
-      const n = +(r.headers.get("X-Seq") || 0);
-      if (live && n && n === seq) continue;
-      seq = n;
       S.size = (r.headers.get("X-Size") || "0x0").split("x").map(Number);
       const url = URL.createObjectURL(await r.blob());
       let img = $("stage").querySelector("img");
       if (!img) { $("stage").innerHTML = ""; img = document.createElement("img"); img.draggable = false; $("stage").appendChild(img); bindStage(img); }
       img.src = url; if (lastUrl) URL.revokeObjectURL(lastUrl); lastUrl = url;
       $("screen-live").innerHTML = `<span class="live"></span>即時`;
-      // At most about 8 pictures a second (an animated page would send all it can).
+      await nap(700);
-      await nap(!live || !n ? 700 : 120);
     } catch (e) { $("screen-live").textContent = ""; await nap(1500); }
   }
 }
@@ -788,7 +785,7 @@ function act(img, e, button) {
   input2({kind: "click", x: p.x, y: p.y, button});
 }
 async function input2(body) {
-  try { await api(base() + "api/screen/" + S.screen, {json: body}); } catch (e) { toast(e.message); }
+  try { await api(base() + "api/screen/mac", {json: body}); } catch (e) { toast(e.message); }
 }
 document.querySelectorAll("[data-key]").forEach((b) => b.onclick = () => input2({kind: "key", key: b.dataset.key}));
 document.querySelectorAll("[data-scroll]").forEach((b) => b.onclick = () => input2({kind: "scroll", x: Math.round(S.size[0] / 2), y: Math.round(S.size[1] / 2), direction: b.dataset.scroll, amount: 5}));
@@ -796,9 +793,8 @@ $("k-type").onclick = () => { $("typebar").classList.toggle("on"); if ($("typeba
 $("b-type").onclick = () => { const t = $("typein").value; if (!t) return; $("typein").value = ""; input2({kind: "type", text: t}); };
 $("typein").addEventListener("keydown", (e) => { if (e.key === "Enter" && !e.isComposing) { e.preventDefault(); $("b-type").click(); } });
 $("k-right").onclick = () => { S.right = !S.right; $("k-right").textContent = "右鍵：" + (S.right ? "開" : "關"); };
-$("k-wake").onclick = () => { toast("正在叫醒" + ownComputer() + "…"); input2({kind: "wake"}); };
 // ---------- menu, notifications ----------
-$("b-more").onclick = () => { $("m-host").style.display = S.relay ? "" : "none"; $("m-pair").style.display = S.relay ? "" : "none"; notifyLabel(); $("sheet").classList.add("on"); };
+$("b-more").onclick = () => { $("m-mac").style.display = S.mac ? "" : "none"; $("m-host").style.display = S.relay ? "" : "none"; $("m-pair").style.display = S.relay ? "" : "none"; notifyLabel(); $("sheet").classList.add("on"); };
 $("sheet").onclick = (e) => { if (e.target.id === "sheet") $("sheet").classList.remove("on"); };
 $("m-cancel").onclick = () => $("sheet").classList.remove("on");
 $("m-out").onclick = () => { $("sheet").classList.remove("on"); if (confirm("要登出這支手機嗎？")) signOut(); };

mobile/screen.html

@@ -0,0 +1,109 @@
+<!doctype html>
+<html lang="zh-Hant">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover, user-scalable=no">
+<meta name="theme-color" content="#16181d">
+<title>Bot 的電腦</title>
+<link rel="icon" href="icon.svg">
+<style>
+  :root { --bg: #16181d; --bar: #1f2228; --line: #2e323a; --text: #eceef2; --dim: #9aa1ad; --accent: #4f8cff; --warn: #e5a13a; --ok: #3fb67a; }
+  * { box-sizing: border-box; }
+  html, body { margin: 0; height: 100%; background: var(--bg); color: var(--text); font: 15px/1.4 -apple-system, BlinkMacSystemFont, "PingFang TC", "Noto Sans TC", sans-serif; overflow: hidden; }
+  body { display: flex; flex-direction: column; height: 100dvh; }
+  header { display: flex; align-items: center; gap: 8px; padding: 8px 12px; padding-top: max(8px, env(safe-area-inset-top)); background: var(--bar); border-bottom: 1px solid var(--line); }
+  header .name { font-weight: 600; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
+  header .state { color: var(--dim); font-size: 13px; white-space: nowrap; display: flex; align-items: center; gap: 6px; }
+  .dot { width: 8px; height: 8px; border-radius: 50%; background: var(--dim); flex: none; }
+  .dot.live { background: var(--ok); }
+  .dot.hold { background: var(--warn); }
+  .grow { flex: 1; min-width: 0; display: flex; flex-direction: column; }
+  button { font: inherit; color: var(--text); background: #2b2f37; border: 1px solid var(--line); border-radius: 8px; padding: 6px 12px; cursor: pointer; white-space: nowrap; }
+  button.main { background: var(--accent); border-color: var(--accent); color: #fff; font-weight: 600; }
+  button.icon { padding: 6px 10px; }
+  #screen { flex: 1; min-height: 0; position: relative; background: #000; }
+  #screen > div { background: #000 !important; }
+  #msg { position: absolute; inset: 0; display: flex; align-items: center; justify-content: center; text-align: center; padding: 24px; color: var(--dim); pointer-events: none; }
+  #type { display: none; gap: 6px; padding: 8px 12px; padding-bottom: max(8px, env(safe-area-inset-bottom)); background: var(--bar); border-top: 1px solid var(--line); flex-wrap: wrap; }
+  #type.on { display: flex; }
+  #type input { flex: 1 1 160px; min-width: 0; font: inherit; color: var(--text); background: #111318; border: 1px solid var(--line); border-radius: 8px; padding: 7px 10px; }
+  @media (max-width: 420px) { header .hint { display: none; } }
+</style>
+</head>
+<body>
+<header>
+  <button class="icon" id="back" title="返回">←</button>
+  <div class="grow"><div class="name" id="name">Bot 的電腦</div><div class="state"><span class="dot" id="dot"></span><span id="state">連線中…</span></div></div>
+  <button id="kb" title="打字（任何語言）">打字</button>
+  <button class="main" id="take">接管</button>
+</header>
+<div id="screen"><div id="msg">連線中…</div></div>
+<div id="type">
+  <input id="text" placeholder="要打的字（中文也可以），按送出" autocomplete="off" enterkeyhint="send">
+  <button class="main" id="send">送出</button>
+  <button data-k="Return">Enter</button><button data-k="Escape">Esc</button><button data-k="Tab">Tab</button><button data-k="BackSpace">⌫</button>
+</div>
+<script type="module">
+import RFB from "./novnc.js";
+
+// #host=…&name=…&ticket=… (the app's link) or #host=… with this site's sign-in (the phone page);
+// #lan=1 on an AnyBot's own phone page. &control=1 starts with control (a bot asked for help).
+const q = new URLSearchParams(location.hash.slice(1));
+const name = q.get("name") || "Bot 的電腦";
+const token = localStorage.getItem("anybot.token") || "";
+const wsBase = (location.protocol === "https:" ? "wss://" : "ws://") + location.host;
+const url = q.get("lan") === "1"
+  ? `${wsBase}/api/vnc?t=${encodeURIComponent(token)}`
+  : `${wsBase}/relay/vnc/${encodeURIComponent(q.get("host") || "")}?` + (q.get("ticket") ? "ticket=" + encodeURIComponent(q.get("ticket")) : "t=" + encodeURIComponent(token));
+const $ = (id) => document.getElementById(id);
+document.title = name;
+$("name").textContent = name;
+
+let rfb = null, ws = null, control = q.get("control") === "1", tries = 0, connected = false;
+const KEYS = { Return: 0xff0d, Escape: 0xff1b, Tab: 0xff09, BackSpace: 0xff08 };
+
+function state(text, kind) { $("state").textContent = text; $("dot").className = "dot" + (kind ? " " + kind : ""); }
+function message(text) { $("msg").textContent = text; $("msg").style.display = text ? "" : "none"; }
+function showControl() {
+  $("take").textContent = control ? "交還給 Bot" : "接管";
+  $("take").className = control ? "" : "main";
+  $("kb").style.display = control ? "" : "none";
+  if (!control) $("type").classList.remove("on");
+  if (rfb) { rfb.viewOnly = !control; if (control) rfb.focus(); }
+  if (connected) state(control ? "你在操作（Bot 會等你停下）" : "即時觀看", control ? "hold" : "live");
+}
+
+function connect() {
+  message("連線中…"); state("連線中…");
+  ws = new WebSocket(url);
+  rfb = new RFB($("screen"), ws, { shared: true });
+  rfb.scaleViewport = true;
+  rfb.resizeSession = false;
+  rfb.qualityLevel = 6;
+  rfb.compressionLevel = 2;
+  rfb.showDotCursor = true;
+  rfb.viewOnly = !control;
+  rfb.addEventListener("connect", () => { tries = 0; connected = true; message(""); showControl(); });
+  rfb.addEventListener("disconnect", () => {
+    rfb = null; connected = false;
+    tries += 1;
+    state("連線中斷，正在重新連線…");
+    message(tries > 3 ? "連不上這台電腦。它的 AnyBot 可能關了，或連結已過期：回到 AnyBot 再打開一次。" : "連線中斷，正在重新連線…");
+    setTimeout(connect, Math.min(1000 * tries, 8000));
+  });
+}
+
+$("take").onclick = () => { control = !control; showControl(); };
+$("kb").onclick = () => { $("type").classList.toggle("on"); if ($("type").classList.contains("on")) $("text").focus(); };
+// Typed text goes over the same connection as words (any language), pasted in the bots' computer.
+function send() { const t = $("text").value; if (!t || !ws || ws.readyState !== 1) return; ws.send(t); $("text").value = ""; }
+$("send").onclick = send;
+$("text").addEventListener("keydown", (e) => { if (e.key === "Enter" && !e.isComposing) { e.preventDefault(); send(); } });
+document.querySelectorAll("[data-k]").forEach((b) => b.onclick = () => { if (rfb) rfb.sendKey(KEYS[b.dataset.k], b.dataset.k); });
+$("back").onclick = () => { if (history.length > 1) history.back(); else window.close(); };
+
+showControl();
+connect();
+</script>
+</body>
+</html>

relay/src/main.rs

@@ -29,6 +29,9 @@ use std::sync::{Arc, Condvar, Mutex};
 use std::time::{Duration, Instant};
 
 const PAGE: &str = include_str!("../../mobile/index.html");
+/// The bots' computer, watched and used in a browser (noVNC).
+const SCREEN_PAGE: &str = include_str!("../../mobile/screen.html");
+const NOVNC: &str = include_str!("../../mobile/novnc.js");
 const MANIFEST: &str = r##"{"name":"AnyBot","short_name":"AnyBot","start_url":"./","display":"standalone","background_color":"#121212","theme_color":"#121212","icons":[{"src":"icon.svg","sizes":"any","type":"image/svg+xml"},{"src":"icon-192.png","sizes":"192x192","type":"image/png","purpose":"any maskable"},{"src":"icon-512.png","sizes":"512x512","type":"image/png","purpose":"any maskable"}]}"##;
 /// AnyBot's icon (the same drawing as the app's), and PNGs for the iPhone home screen and Android.
 const ICON: &str = include_str!("../../mobile/icon.svg");
@@ -200,6 +203,10 @@ struct Relay {
     /// The brain each AnyBot shows (when it is a screen of another), and which lend their hands.
     uses: Mutex<HashMap<String, String>>,
     hands: Mutex<std::collections::HashSet<String>>,
+    /// People watching a bots' computer: stream id → (where the host's side goes, that host).
+    streams: Mutex<HashMap<String, (Sender<Vec<u8>>, String)>>,
+    /// One-time links for a browser to watch a host's computer: ticket → (account, host, made).
+    tickets: Mutex<HashMap<String, (String, String, Instant)>>,
     /// 配對碼: code → (account, the AnyBot it is for, when made). Ten minutes, once.
     pairs: Mutex<HashMap<String, (String, String, Instant)>>,
     /// When each host last asked (in memory; saved now and then).
@@ -210,6 +217,32 @@ struct Relay {
     contact: String,
 }
 
+/// How long a link the app opens for a browser works (reconnects included).
+const TICKET_LIFE: Duration = Duration::from_secs(600);
+
+/// How long a connection's loop waits to read before sending what is queued: 2 ms while things
+/// move (a live screen), 40 ms when quiet. A fixed 40 ms added up to 40 ms to every message.
+struct Pace {
+    moved: Instant,
+    busy: Option<bool>,
+}
+
+impl Pace {
+    fn new() -> Pace {
+        Pace { moved: Instant::now() - Duration::from_secs(60), busy: None }
+    }
+    fn moved(&mut self) {
+        self.moved = Instant::now();
+    }
+    fn tune(&mut self, c: &std::net::TcpStream) {
+        let busy = self.moved.elapsed() < Duration::from_secs(2);
+        if self.busy != Some(busy) {
+            self.busy = Some(busy);
+            let _ = c.set_read_timeout(Some(Duration::from_millis(if busy { 2 } else { 40 })));
+        }
+    }
+}
+
 fn now() -> i64 {
     chrono::Utc::now().timestamp()
 }
@@ -229,6 +262,8 @@ impl Relay {
             flights: Mutex::new(HashMap::new()),
             uses: Mutex::new(HashMap::new()),
             hands: Mutex::new(std::collections::HashSet::new()),
+            streams: Mutex::new(HashMap::new()),
+            tickets: Mutex::new(HashMap::new()),
             pairs: Mutex::new(HashMap::new()),
             seen: Mutex::new(HashMap::new()),
             fails: Mutex::new(HashMap::new()),
@@ -294,6 +329,95 @@ impl Relay {
         e.1 = Instant::now();
     }
 
+    /// A person's browser watching (and maybe using) an AnyBot's computer: its VNC stream goes
+    /// to that AnyBot over its lasting connection, and back. Signed in by the account session
+    /// (?t=, the phone page) or a ticket the account's app asked for (?ticket=); owners and
+    /// admins only.
+    fn vnc_viewer(self: &Arc<Self>, req: Req, mut c: std::net::TcpStream, host: &str) {
+        use std::io::Write;
+        use tungstenite::{protocol::Role, Message, WebSocket};
+        let user = match req.query.get("ticket").filter(|t| !t.is_empty()) {
+            Some(t) => self.tickets.lock().unwrap().get(t).filter(|(_, h, made)| h == host && made.elapsed() < TICKET_LIFE).map(|(u, _, _)| u.clone()),
+            None => self.session(&req),
+        };
+        let Some(user) = user else { return httpd::write_response(&mut c, &Resp::error(401, "請先登入")) };
+        let allowed = self.data.lock().unwrap().hosts.iter().any(|x| x.id == host && matches!(x.role_of(&user).as_deref(), Some("owner") | Some("admin")));
+        if !allowed {
+            return httpd::write_response(&mut c, &Resp::error(404, "找不到這台 AnyBot"));
+        }
+        let Some(to_host) = self.sockets.lock().unwrap().get(host).map(|s| s.tx.clone()) else {
+            return httpd::write_response(&mut c, &Resp::error(503, "這台 AnyBot 現在沒有連線"));
+        };
+        let key = req.header("sec-websocket-key").to_string();
+        if key.is_empty() {
+            return httpd::write_response(&mut c, &Resp::error(400, "bad request"));
+        }
+        let accept = tungstenite::handshake::derive_accept_key(key.as_bytes());
+        if c.write_all(format!("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: {accept}\r\n\r\n").as_bytes()).is_err() {
+            return;
+        }
+        let _ = c.set_nodelay(true);
+        let mut ws = WebSocket::from_raw_socket(c, Role::Server, None);
+        let sid = httpd::random_hex(8);
+        let (tx, rx) = channel::<Vec<u8>>();
+        self.streams.lock().unwrap().insert(sid.clone(), (tx, host.to_string()));
+        if to_host.send(wire::pack(&json!({"t": "vopen", "sid": sid}), &[])).is_err() {
+            self.streams.lock().unwrap().remove(&sid);
+            return;
+        }
+        let (mut heard, mut pinged) = (Instant::now(), Instant::now());
+        let mut pace = Pace::new();
+        'live: loop {
+            pace.tune(ws.get_ref());
+            let up = match ws.read() {
+                Ok(Message::Binary(b)) => Some(("vdata", b.to_vec())),
+                Ok(Message::Text(t)) => Some(("vtext", t.as_bytes().to_vec())),
+                Ok(Message::Close(_)) => break,
+                Ok(_) => None,
+                Err(tungstenite::Error::Io(e)) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => None,
+                Err(_) => break,
+            };
+            if let Some((t, b)) = up {
+                heard = Instant::now();
+                pace.moved();
+                if to_host.send(wire::pack(&json!({"t": t, "sid": sid}), &b)).is_err() {
+                    break;
+                }
+            }
+            loop {
+                match rx.try_recv() {
+                    Ok(b) => {
+                        pace.moved();
+                        if ws.write(Message::Binary(b.into())).is_err() {
+                            break 'live;
+                        }
+                    }
+                    Err(std::sync::mpsc::TryRecvError::Empty) => break,
+                    // The host closed it (or went away).
+                    Err(std::sync::mpsc::TryRecvError::Disconnected) => break 'live,
+                }
+            }
+            if pinged.elapsed().as_secs() >= WS_PING_SECS {
+                pinged = Instant::now();
+                if ws.write(Message::Ping(Vec::new().into())).is_err() {
+                    break;
+                }
+            }
+            match ws.flush() {
+                Err(tungstenite::Error::Io(e)) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => {}
+                Err(_) => break,
+                Ok(()) => {}
+            }
+            if heard.elapsed().as_secs() >= WS_DEAD_SECS {
+                break;
+            }
+        }
+        let _ = ws.close(None);
+        let _ = ws.flush();
+        self.streams.lock().unwrap().remove(&sid);
+        let _ = to_host.send(wire::pack(&json!({"t": "vclose", "sid": sid}), &[]));
+    }
+
     /// Hand a request to a host and wait for its answer.
     fn forward(&self, host_id: &str, method: &str, path: &str, query: &HashMap<String, String>, body: &[u8], who: (&str, &str)) -> Resp {
         self.forward_raw(host_id, method, path, query, body, false, who, false, "").into_resp()
@@ -358,6 +482,10 @@ impl Relay {
     fn ws(self: &Arc<Self>, req: Req, mut c: std::net::TcpStream) {
         use std::io::Write;
         use tungstenite::{protocol::Role, Message, WebSocket};
+        if let Some(host) = req.path.strip_prefix("/relay/vnc/") {
+            let host = host.to_string();
+            return self.vnc_viewer(req, c, &host);
+        }
         if req.path != "/relay/host/ws" {
             return httpd::write_response(&mut c, &Resp::error(404, "沒有這個網址"));
         }
@@ -379,10 +507,13 @@ impl Relay {
         self.sockets.lock().unwrap().insert(h.id.clone(), Sock { tx: tx.clone(), serial });
         self.seen.lock().unwrap().insert(h.id.clone(), Instant::now());
         let (mut heard, mut pinged, mut marked) = (Instant::now(), Instant::now(), Instant::now());
+        let mut pace = Pace::new();
         'live: loop {
+            pace.tune(ws.get_ref());
             match ws.read() {
                 Ok(Message::Binary(b)) => {
                     heard = Instant::now();
+                    pace.moved();
                     self.on_frame(&h, &b, &tx);
                 }
                 Ok(Message::Close(_)) => break,
@@ -391,6 +522,7 @@ impl Relay {
                 Err(_) => break,
             }
             while let Ok(f) = rx.try_recv() {
+                pace.moved();
                 if ws.write(Message::Binary(f.into())).is_err() {
                     break 'live;
                 }
@@ -414,7 +546,8 @@ impl Relay {
                 self.seen.lock().unwrap().insert(h.id.clone(), Instant::now());
             }
         }
-        // Gone: whoever waits on it hears so now, not after a timeout.
+        // Gone: its viewers' streams end, and whoever waits on it hears so now, not after a timeout.
+        self.streams.lock().unwrap().retain(|_, (_, host)| *host != h.id);
         {
             let mut socks = self.sockets.lock().unwrap();
             if socks.get(&h.id).map(|s| s.serial) == Some(serial) {
@@ -434,6 +567,20 @@ impl Relay {
         let Some((head, body)) = wire::unpack(b) else { return };
         let s = |k: &str| head[k].as_str().unwrap_or("").to_string();
         match head["t"].as_str().unwrap_or("") {
+            // A viewer's stream: only the host it was opened on may write to it.
+            "vdata" => {
+                if let Some((v, host)) = self.streams.lock().unwrap().get(&s("sid")) {
+                    if *host == h.id {
+                        let _ = v.send(body);
+                    }
+                }
+            }
+            "vclose" => {
+                let mut all = self.streams.lock().unwrap();
+                if all.get(&s("sid")).map(|(_, host)| *host == h.id).unwrap_or(false) {
+                    all.remove(&s("sid"));
+                }
+            }
             "reply" => {
                 // Only the host the job went to may answer it.
                 let w = {
@@ -499,6 +646,25 @@ impl Relay {
             ("GET", []) | ("GET", ["index.html"]) => Resp::bytes("text/html; charset=utf-8", PAGE.as_bytes().to_vec())
                 .with_header("Content-Security-Policy", "default-src 'self'; img-src 'self' blob: data:; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline'; connect-src 'self'")
                 .with_header("Strict-Transport-Security", "max-age=31536000"),
+            ("GET", ["screen"]) => Resp::bytes("text/html; charset=utf-8", SCREEN_PAGE.as_bytes().to_vec())
+                .with_header("Content-Security-Policy", "default-src 'self'; img-src 'self' blob: data:; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline'; connect-src 'self' wss: ws:")
+                .with_header("Strict-Transport-Security", "max-age=31536000"),
+            ("GET", ["novnc.js"]) => Resp::bytes("text/javascript; charset=utf-8", NOVNC.as_bytes().to_vec()).with_header("Cache-Control", "public, max-age=86400"),
+            // A link for this account's browser to watch one of its AnyBots' computer (the app
+            // opens it; the browser has no session of its own).
+            ("POST", ["relay", "screen-ticket"]) => {
+                let Some(h) = self.host(&req) else { return Resp::error(401, "這台電腦的金鑰無效") };
+                let want = req.json()["host"].as_str().unwrap_or("").to_string();
+                let ok = self.data.lock().unwrap().hosts.iter().any(|x| x.id == want && matches!(x.role_of(&h.user).as_deref(), Some("owner") | Some("admin")));
+                if !ok {
+                    return Resp::error(404, "找不到這台 AnyBot");
+                }
+                let t = httpd::random_hex(24);
+                let mut all = self.tickets.lock().unwrap();
+                all.retain(|_, (_, _, made)| made.elapsed() < TICKET_LIFE);
+                all.insert(t.clone(), (h.user.clone(), want, Instant::now()));
+                Resp::json(&json!({"ticket": t}))
+            }
             ("GET", ["manifest.webmanifest"]) => Resp::bytes("application/manifest+json", MANIFEST.as_bytes().to_vec()),
             ("GET", ["icon.svg"]) => Resp::bytes("image/svg+xml", ICON.as_bytes().to_vec()),
             ("GET", [name]) if ICON_PNGS.iter().any(|(n, _)| n == name) => Resp::bytes("image/png", ICON_PNGS.iter().find(|(n, _)| n == name).map(|(_, b)| b.to_vec()).unwrap_or_default()),

src/app.rs

@@ -371,13 +371,6 @@ impl App {
                     app.state.selected = 2;
                     app.show_routines = Some(2);
                 }
-                Ok("viewer") => {
-                    crate::cloudbox::start_background();
-                    let _ = crate::cloudbox::ensure_running();
-                    app.cu.demo_live = true;
-                    app.cu.viewer_open = true;
-                    app.cu.fit = true;
-                }
                 Ok("attach") => {
                     app.state.selected = 1;
                     let dir = std::env::temp_dir().join("anybot-shot-attach");
@@ -511,7 +504,6 @@ impl App {
                     // The real box, so the 電腦 card shows its screen.
                     crate::cloudbox::start_background();
                     let _ = crate::cloudbox::ensure_running();
-                    app.cu.demo_live = true;
                     app.state.selected = 1;
                     app.panel = Some(panel_ui::PanelTab::Activity);
                 }
@@ -3427,11 +3419,6 @@ impl App {
                 ctx.send_viewport_cmd(egui::ViewportCommand::InnerSize(vec2(w, h)));
             }
         }
-        // Real data: the computer viewer opens a little after start, as a person would open it (a
-        // screen of a brain has said so by then, so it shows the brain's computer).
-        if self.frames == 10 && !self.demo && std::env::var("ANYBOT_SHOT_VIEW").as_deref() == Ok("viewer") {
-            self.open_viewer();
-        }
         // ANYBOT_SHOT_PAGE=id: open that page late (a screen gets its pages from the brain first).
         if self.frames == need * 3 / 4 {
             if let Some(id) = std::env::var("ANYBOT_SHOT_PAGE").ok().and_then(|v| v.parse().ok()) {
@@ -3767,7 +3754,6 @@ impl eframe::App for App {
         self.delete_dialog(ctx);
         self.routines_window(ctx);
         self.rebuild_dialog(ctx);
-        self.viewer_window(ctx);
         self.local_control_bar(ctx);
         self.teach_bar(ctx);
         self.approval_popup(ctx);

src/app/computers_ui.rs

@@ -7,9 +7,6 @@
 use super::*;
 use crate::cloudbox::{self, Rebuild, Run};
 
-/// While the viewer is open it redraws at least this often (new pictures ask for a redraw
-/// themselves as they arrive).
-const VIEWER_FRAME_MS: u64 = 450;
 /// The floating bar stays this long after the bot's last mouse/keyboard action on this Mac.
 const LOCAL_BAR_SECS: u64 = 8;
 
@@ -30,16 +27,8 @@ pub struct ComputerUi {
     pub confirm_reset: bool,
     /// 一律允許的應用程式 is expanded.
     pub apps_open: bool,
-    pub viewer_open: bool,
+    /// 接管 during a hand-off: this app told the bots' computer to let the person have it.
-    /// 取得控制權: the person's clicks and keys go to the box.
     pub control: bool,
-    pub fit: bool,
-    texture: Option<egui::TextureHandle>,
-    frame: Arc<Mutex<Option<(u64, egui::ColorImage)>>>,
-    shown_seq: u64,
-    frame_size: [usize; 2],
-    worker: Option<Arc<AtomicBool>>,
-    input_tx: Option<Sender<Input>>,
     pub confirm: Option<Rebuild>,
     pub note: Arc<Mutex<String>>,
     disk: Arc<Mutex<Option<String>>>,
@@ -51,8 +40,6 @@ pub struct ComputerUi {
     teach_name: String,
     teach_for: Option<u64>,
     teach_error: String,
-    /// Screenshot mode: show the real box in the viewer.
-    pub demo_live: bool,
     schedule_edit: String,
     pub update_ready: bool,
     update_checked: Option<Instant>,
@@ -62,16 +49,6 @@ pub struct ComputerUi {
     thumb_img: Arc<Mutex<Option<egui::ColorImage>>>,
     thumb_busy: Arc<AtomicBool>,
     thumb_at: Option<Instant>,
-    /// The viewer shows the brain's computer (this window is a screen of a brain): how to reach it.
-    remote: Option<crate::remote::Settings>,
-}
-
-enum Input {
-    Click(i64, i64, &'static str, u32),
-    Move(i64, i64),
-    Scroll(i64, i64, bool, u32),
-    Key(String),
-    Text(String),
 }
 
 fn status_color(r: &Run, p: Palette) -> Color32 {
@@ -103,16 +80,6 @@ fn status_pill(ui: &mut egui::Ui, r: &Run, p: Palette) {
 }
 
 /// A little monitor drawn with lines (no emoji font needed).
-fn monitor_icon(ui: &mut egui::Ui, size: f32, col: Color32) {
-    let (rect, _) = ui.allocate_exact_size(vec2(size, size), egui::Sense::hover());
-    let s = size / 20.0;
-    let screen = egui::Rect::from_min_size(rect.min + vec2(2.0 * s, 3.0 * s), vec2(16.0 * s, 11.0 * s));
-    ui.painter().rect_stroke(screen, 2.5 * s, Stroke::new(1.5, col), egui::StrokeKind::Inside);
-    ui.painter().line_segment([rect.min + vec2(10.0 * s, 14.0 * s), rect.min + vec2(10.0 * s, 17.0 * s)], Stroke::new(1.5, col));
-    ui.painter().line_segment([rect.min + vec2(6.5 * s, 17.5 * s), rect.min + vec2(13.5 * s, 17.5 * s)], Stroke::new(1.5, col));
-}
-
-
 impl App {
     /// Called once at start.
     pub(super) fn computers_start(&mut self) {
@@ -946,455 +913,67 @@ impl App {
         self.cu.thumb.as_ref().map(|t| (t.id(), t.size_vec2()))
     }
 
+    /// 看畫面: the bots' computer in the browser (noVNC), live, where the person can also take
+    /// over. On a screen of a brain it is the brain's computer, through the account server (a
+    /// link made for this once); here it is this computer's own box.
     pub(super) fn open_viewer(&mut self) {
-        self.cu.viewer_open = true;
+        self.open_screen(false);
-        self.cu.fit = true;
+    }
+
+    fn open_screen(&mut self, control: bool) {
         self.show_settings = false;
-        // A screen of a brain watches the brain's computer (where its bots really work).
+        let name = self.state.bots.iter().find(|b| b.id == self.state.selected).map(|b| format!("{} 的電腦", b.name)).unwrap_or_else(|| "Bot 的電腦".into());
         if self.mir.on {
             let s = self.state.remote.clone();
-            self.cu.remote = Some(s.clone());
+            self.box_task(move || {
-            if let Some(stop) = self.cu.worker.take() {
+                let ticket = crate::remote::screen_ticket(&s)?;
-                stop.store(true, Ordering::SeqCst);
+                let enc = |t: &str| t.bytes().map(|b| if b.is_ascii_alphanumeric() { (b as char).to_string() } else { format!("%{b:02X}") }).collect::<String>();
-            }
+                let url = format!("{}/screen#host={}&ticket={ticket}&name={}{}", s.relay_url.trim_end_matches('/'), s.brain_id, enc(&name), if control { "&control=1" } else { "" });
-            self.cu.texture = None;
+                crate::preview::open_with_system(&url);
-            std::thread::spawn(move || {
+                Ok(String::new())
-                let _ = crate::remote::sync_post(&s, "screen", &serde_json::json!({"kind": "viewer", "on": true}));
-                let _ = crate::remote::sync_post(&s, "screen", &serde_json::json!({"kind": "wake"}));
             });
             return;
         }
-        if self.cu.remote.take().is_some() {
+        if self.demo {
-            if let Some(stop) = self.cu.worker.take() {
+            return;
-                stop.store(true, Ordering::SeqCst);
-            }
-            self.cu.texture = None;
-        }
-        cloudbox::set_viewer_open(true);
-        if !cloudbox::run_state().is_running() && !self.demo {
-            self.box_task(|| cloudbox::ensure_running().map(|_| String::new()));
         }
+        self.box_task(move || {
+            let url = cloudbox::viewer_url(control)?;
+            crate::preview::open_with_system(&url);
+            Ok(String::new())
+        });
     }
 
-    /// 接管: open the computer and take control (the bot waits for the person anyway).
+    /// 接管: open the computer with control (the bot waits for the person meanwhile).
     pub(super) fn take_over_computer(&mut self) {
-        self.open_viewer();
+        self.open_screen(true);
         if self.cu.control {
             return;
         }
         self.cu.control = true;
-        match self.cu.remote.clone() {
+        self.hold_computer(true);
-            Some(s) => {
-                std::thread::spawn(move || {
-                    let _ = crate::remote::sync_post(&s, "screen", &serde_json::json!({"kind": "hold", "on": true}));
-                });
-            }
-            None => cloudbox::HELD_BY_PERSON.store(true, Ordering::SeqCst),
-        }
     }
 
-    /// 我完成了 / 跳過: the bot gets its computer back (its steps would otherwise wait for the
+    /// 我完成了 / 跳過: the bot gets its computer back.
-    /// person to let go).
     pub(super) fn give_back_computer(&mut self) {
         if !self.cu.control {
             return;
         }
         self.cu.control = false;
-        match self.cu.remote.clone() {
+        self.hold_computer(false);
-            Some(s) => {
-                std::thread::spawn(move || {
-                    let _ = crate::remote::sync_post(&s, "screen", &serde_json::json!({"kind": "hold", "on": false}));
-                });
-            }
-            None => cloudbox::HELD_BY_PERSON.store(false, Ordering::SeqCst),
-        }
     }
 
-    fn close_viewer(&mut self) {
+    fn hold_computer(&self, on: bool) {
-        self.cu.viewer_open = false;
+        if self.mir.on {
-        self.cu.control = false;
+            let s = self.state.remote.clone();
-        if let Some(s) = self.cu.remote.clone() {
             std::thread::spawn(move || {
-                let _ = crate::remote::sync_post(&s, "screen", &serde_json::json!({"kind": "hold", "on": false}));
+                let _ = crate::remote::sync_post(&s, "screen", &serde_json::json!({"kind": "hold", "on": on}));
-                let _ = crate::remote::sync_post(&s, "screen", &serde_json::json!({"kind": "viewer", "on": false}));
             });
         } else {
-            cloudbox::HELD_BY_PERSON.store(false, Ordering::SeqCst);
+            cloudbox::HELD_BY_PERSON.store(on, Ordering::SeqCst);
-            cloudbox::set_viewer_open(false);
-        }
-        if let Some(stop) = self.cu.worker.take() {
-            stop.store(true, Ordering::SeqCst);
-        }
-        self.cu.input_tx = None;
-    }
-
-    fn ensure_viewer_threads(&mut self, ctx: &egui::Context) {
-        if self.cu.worker.is_some() || (self.demo && !self.cu.demo_live) {
-            return;
-        }
-        let stop = Arc::new(AtomicBool::new(false));
-        self.cu.worker = Some(stop.clone());
-        let slot = self.cu.frame.clone();
-        let c = ctx.clone();
-        let s2 = stop.clone();
-        let brain = self.cu.remote.clone();
-        if let Some(s) = brain.clone() {
-            // The brain's computer: pictures and the person's input go through the account server.
-            let s_frames = s.clone();
-            std::thread::spawn(move || {
-                let s = s_frames;
-                // Ask for the picture after the one shown: the brain answers when the screen
-                // changes (no fixed pause between pictures).
-                let mut seq = 0u64;
-                while !s2.load(Ordering::SeqCst) {
-                    match crate::remote::sync_live(&s, seq) {
-                        Ok((n, jpeg)) if n != seq || n == 0 => {
-                            if let Ok(img) = image::load_from_memory(&jpeg) {
-                                let img = img.to_rgba8();
-                                let size = [img.width() as usize, img.height() as usize];
-                                *slot.lock().unwrap() = Some((n.max(seq + 1), egui::ColorImage::from_rgba_unmultiplied(size, img.as_raw())));
-                                c.request_repaint();
-                            }
-                            // At most about 8 a second over the network (an animated page
-                            // sends all it can); a brain too old to number them: one a second.
-                            std::thread::sleep(std::time::Duration::from_millis(if n == 0 { 1000 } else { 120 }));
-                            seq = n;
-                        }
-                        Ok(_) => {}
-                        Err(_) => {
-                            std::thread::sleep(std::time::Duration::from_millis(1500));
-                            c.request_repaint();
-                        }
-                    }
-                }
-            });
-            let (tx, rx) = channel::<Input>();
-            self.cu.input_tx = Some(tx);
-            std::thread::spawn(move || {
-                let mut last_move = Instant::now();
-                while let Ok(i) = rx.recv() {
-                    let v = match i {
-                        Input::Click(x, y, b, n) => serde_json::json!({"kind": "click", "x": x, "y": y, "button": b, "count": n}),
-                        // Hovering sends many moves; a few a second are enough over the network.
-                        Input::Move(x, y) => {
-                            if last_move.elapsed().as_millis() < 250 {
-                                continue;
-                            }
-                            last_move = Instant::now();
-                            serde_json::json!({"kind": "move", "x": x, "y": y})
-                        }
-                        Input::Scroll(x, y, down, n) => serde_json::json!({"kind": "scroll", "x": x, "y": y, "down": down, "steps": n}),
-                        Input::Key(k) => serde_json::json!({"kind": "key", "key": k}),
-                        Input::Text(t) => serde_json::json!({"kind": "text", "text": t}),
-                    };
-                    let _ = crate::remote::sync_post(&s, "screen", &v);
-                }
-            });
-            return;
-        }
-        std::thread::spawn(move || {
-            let mut seq = 0u64;
-            while !s2.load(Ordering::SeqCst) {
-                match cloudbox::live_frame(seq, std::time::Duration::from_secs(5)) {
-                    Some((n, jpeg)) if n != seq => {
-                        if let Ok(img) = image::load_from_memory(&jpeg) {
-                            let img = img.to_rgba8();
-                            let size = [img.width() as usize, img.height() as usize];
-                            *slot.lock().unwrap() = Some((n, egui::ColorImage::from_rgba_unmultiplied(size, img.as_raw())));
-                            c.request_repaint();
-                        }
-                        seq = n;
-                    }
-                    Some(_) => {}
-                    None => {
-                        std::thread::sleep(std::time::Duration::from_millis(1000));
-                        c.request_repaint();
-                    }
-                }
-            }
-        });
-        let (tx, rx) = channel::<Input>();
-        self.cu.input_tx = Some(tx);
-        std::thread::spawn(move || {
-            while let Ok(i) = rx.recv() {
-                match i {
-                    Input::Click(x, y, b, n) => cloudbox::viewer_click(x, y, b, n),
-                    Input::Move(x, y) => cloudbox::viewer_move(x, y),
-                    Input::Scroll(x, y, down, n) => cloudbox::viewer_scroll(x, y, down, n),
-                    Input::Key(k) => cloudbox::viewer_key(&k),
-                    Input::Text(t) => cloudbox::viewer_text(&t),
-                }
-            }
-        });
-    }
-
-    pub(super) fn viewer_window(&mut self, ctx: &egui::Context) {
-        if !self.cu.viewer_open {
-            return;
-        }
-        self.ensure_viewer_threads(ctx);
-        let p = self.pal;
-        if let Some((seq, img)) = self.cu.frame.lock().unwrap().take() {
-            if seq != self.cu.shown_seq {
-                self.cu.shown_seq = seq;
-                self.cu.frame_size = img.size;
-                match &mut self.cu.texture {
-                    Some(t) => t.set(img, egui::TextureOptions::LINEAR),
-                    None => self.cu.texture = Some(ctx.load_texture("box-screen", img, egui::TextureOptions::LINEAR)),
-                }
-            }
-        }
-        let brain = self.cu.remote.is_some();
-        // The brain's computer reports its state with the brain's answers.
-        let run = if brain {
-            match (self.mir.box_state["running"].as_bool().unwrap_or(false), self.mir.box_state["available"].as_bool().unwrap_or(true)) {
-                (true, _) => Run::Running,
-                (false, false) => Run::NoDocker("大腦那台沒有 Docker".into()),
-                _ => Run::Asleep,
-            }
-        } else {
-            cloudbox::run_state()
-        };
-        let screen = ctx.content_rect();
-        let open = true;
-        let default = vec2((screen.width() * 0.62).min(900.0), (screen.width() * 0.62).min(900.0) * 0.625 + 52.0);
-        let busy = if brain { self.mir.box_state["busy"].as_bool().unwrap_or(false) } else { cloudbox::is_busy() };
-        let title = self.state.bot(self.state.selected).map(|b| crate::model::own_computer_name(&b.name)).unwrap_or_else(|| crate::model::BOT_COMPUTER_NAME.to_string());
-
-        let mut close_clicked = false;
-        let mut done_clicked = false;
-        let selected = self.state.selected;
-        let handoff = self.approvals.get(&selected).map(|a| a.kind == crate::providers::ApprovalKind::Handoff).unwrap_or(false);
-        egui::Window::new("box-viewer")
-            .id(egui::Id::unique("box-viewer"))
-            .title_bar(false)
-            .default_size(default)
-            .default_pos(screen.center() - default / 2.0)
-            .min_size(vec2(420.0, 300.0))
-            .resizable(true)
-            .collapsible(false)
-            .frame(
-                Frame::new()
-                    .fill(p.elevated)
-                    .corner_radius(14)
-                    .inner_margin(Margin::symmetric(14, 12))
-                    .stroke(Stroke::new(1.0, p.border))
-                    .shadow(egui::epaint::Shadow { offset: [0, 14], blur: 40, spread: 0, color: Color32::from_black_alpha(60) }),
-            )
-            .show(ctx, |ui| {
-                // Buttons first, from the right; the title and state get what is left and end in
-                // … when it's short (the state used to slide under 「取得控制權」 in a narrow window).
-                ui.horizontal(|ui| {
-                    ui.with_layout(Layout::right_to_left(Align::Center), |ui| {
-                        let (rect, r) = ui.allocate_exact_size(vec2(28.0, 28.0), egui::Sense::click());
-                        if r.hovered() {
-                            ui.painter().rect_filled(rect, 8, p.hover);
-                        }
-                        let c = rect.center();
-                        let st = Stroke::new(1.5, if r.hovered() { p.text } else { p.text2 });
-                        ui.painter().line_segment([c + vec2(-5.0, -5.0), c + vec2(5.0, 5.0)], st);
-                        ui.painter().line_segment([c + vec2(-5.0, 5.0), c + vec2(5.0, -5.0)], st);
-                        if r.on_hover_text(t("關閉")).clicked() {
-                            close_clicked = true;
-                        }
-                        ui.add_space(4.0);
-                        // The bot is waiting for the person here: done, and it carries on.
-                        if handoff && ui.add(egui::Button::new(RichText::new(t("我完成了")).strong().color(p.on_primary)).fill(p.primary).corner_radius(8).min_size(vec2(84.0, 30.0))).clicked() {
-                            done_clicked = true;
-                        }
-                        if !brain && ui.add(secondary_button("全螢幕")).on_hover_text(t("在瀏覽器用 noVNC 全螢幕開啟")).clicked() {
-                            let control = self.cu.control;
-                            self.box_task(move || {
-                                let url = cloudbox::viewer_url(control)?;
-                                crate::preview::open_with_system(&url);
-                                Ok(String::new())
-                            });
-                        }
-                        let fit_label = if self.cu.fit { "實際大小" } else { "調整至視窗大小" };
-                        if ui.add(secondary_button(fit_label)).clicked() {
-                            self.cu.fit = !self.cu.fit;
-                        }
-                        let (label, fill, col) = if self.cu.control { ("交還給 Bot", p.accent, Color32::WHITE) } else { ("取得控制權", p.primary, p.on_primary) };
-                        let btn = egui::Button::new(RichText::new(label).strong().color(col)).fill(fill).corner_radius(8).min_size(vec2(96.0, 30.0));
-                        if ui.add_enabled(run.is_running(), btn).on_hover_text(if self.cu.control { "Bot 可以繼續操作" } else { "你自己點擊和輸入；Bot 的操作會先暫停" }).clicked() {
-                            self.cu.control = !self.cu.control;
-                            match self.cu.remote.clone() {
-                                Some(s) => {
-                                    let on = self.cu.control;
-                                    std::thread::spawn(move || {
-                                        let _ = crate::remote::sync_post(&s, "screen", &serde_json::json!({"kind": "hold", "on": on}));
-                                    });
-                                }
-                                None => cloudbox::HELD_BY_PERSON.store(self.cu.control, Ordering::SeqCst),
-                            }
-                        }
-                        ui.add_space(8.0);
-                        ui.with_layout(Layout::left_to_right(Align::Center), |ui| {
-                            monitor_icon(ui, 20.0, p.text);
-                            let note = if self.cu.control { Some(t("你正在操作，Bot 已暫停")) } else if busy { Some(t("Bot 正在操作")) } else { None };
-                            let room = (ui.available_width() - 96.0 - if note.is_some() { 120.0 } else { 0.0 }).max(60.0);
-                            ui.label(RichText::new(super::sidebar_ui::shorten_width(ui, &title, room, FONT_LG)).strong().size(FONT_LG));
-                            if ui.available_width() > 80.0 {
-                                status_pill(ui, &run, p);
-                            }
-                            if let Some(n) = note.filter(|_| ui.available_width() > 110.0) {
-                                crate::icons::label(ui, if self.cu.control { crate::icons::Icon::Users } else { crate::icons::Icon::Bot }, &n, FONT_XS, p.accent);
-                            }
-                        });
-                    });
-                });
-                ui.add_space(6.0);
-                let avail = ui.available_size();
-                match (self.cu.texture.as_ref().map(|t| t.id()), run.is_running()) {
-                    (Some(tex_id), true) => {
-                        let [fw, fh] = self.cu.frame_size;
-                        let (fw, fh) = (fw.max(1) as f32, fh.max(1) as f32);
-                        let size = if self.cu.fit {
-                            let k = (avail.x / fw).min((avail.y.max(120.0)) / fh);
-                            vec2(fw * k, fh * k)
-                        } else {
-                            vec2(fw, fh)
-                        };
-                        egui::ScrollArea::both().auto_shrink([false, false]).show(ui, |ui| {
-                            let sense = if self.cu.control { egui::Sense::click_and_drag() } else { egui::Sense::hover() };
-                            // Centred in the window (it sat at the left with an empty band beside it).
-                            let resp = if self.cu.fit {
-                                let pad = ((ui.available_width() - size.x) / 2.0).max(0.0);
-                                ui.horizontal(|ui| {
-                                    ui.add_space(pad);
-                                    ui.add(egui::Image::new((tex_id, size)).corner_radius(8).sense(sense))
-                                })
-                                .inner
-                            } else {
-                                ui.add(egui::Image::new((tex_id, size)).corner_radius(8).sense(sense))
-                            };
-                            if self.cu.control {
-                                ui.painter().rect_stroke(resp.rect, 8, Stroke::new(2.0, p.accent), egui::StrokeKind::Outside);
-                                self.forward_input(ui, &resp, (fw, fh));
-                            }
-                        });
-                    }
-                    _ => {
-                        let (rect, _) = ui.allocate_exact_size(vec2(avail.x, avail.y.max(200.0)), egui::Sense::hover());
-                        ui.painter().rect_filled(rect, 8, Color32::from_rgb(0x2b, 0x2f, 0x36));
-                        let text = match &run {
-                            Run::Running => "正在連線…".to_string(),
-                            Run::NoDocker(why) => why.clone(),
-                            Run::Failed(e) => format!("發生問題：{e}"),
-                            other => format!("{}…", other.label()),
-                        };
-                        ui.painter().text(rect.center(), egui::Align2::CENTER_CENTER, text, egui::FontId::proportional(FONT_BASE), Color32::from_gray(220));
-                        if matches!(run, Run::Asleep | Run::Absent) {
-                            let b = egui::Rect::from_center_size(rect.center() + vec2(0.0, 34.0), vec2(90.0, 30.0));
-                            if ui.put(b, primary_button("喚醒", p)).clicked() {
-                                match self.cu.remote.clone() {
-                                    Some(s) => {
-                                        std::thread::spawn(move || {
-                                            let _ = crate::remote::sync_post(&s, "screen", &serde_json::json!({"kind": "wake"}));
-                                        });
-                                    }
-                                    None => self.box_task(|| cloudbox::ensure_running().map(|_| String::new())),
-                                }
-                            }
-                        }
-                    }
-                }
-            });
-        if done_clicked {
-            self.give_back_computer();
-            self.answer(ctx, selected, crate::providers::ApprovalAnswer::Once);
-            self.close_viewer();
-        } else if close_clicked || !open {
-            self.close_viewer();
-        }
-        ctx.request_repaint_after(std::time::Duration::from_millis(VIEWER_FRAME_MS));
-    }
-
-    /// Clicks, scrolls and keys on the picture go to the box while the person has control.
-    fn forward_input(&mut self, ui: &mut egui::Ui, resp: &egui::Response, frame: (f32, f32)) {
-        let Some(tx) = self.cu.input_tx.clone() else { return };
-        let to_box = |pos: egui::Pos2| -> (i64, i64) {
-            let r = resp.rect;
-            (((pos.x - r.left()) / r.width() * frame.0).round() as i64, ((pos.y - r.top()) / r.height() * frame.1).round() as i64)
-        };
-        if resp.clicked() || resp.secondary_clicked() || resp.double_clicked() {
-            resp.request_focus();
-        }
-        if let Some(pos) = resp.interact_pointer_pos().or(resp.hover_pos()) {
-            let (x, y) = to_box(pos);
-            if resp.double_clicked() {
-                let _ = tx.send(Input::Click(x, y, "left", 2));
-            } else if resp.clicked() {
-                let _ = tx.send(Input::Click(x, y, "left", 1));
-            } else if resp.secondary_clicked() {
-                let _ = tx.send(Input::Click(x, y, "right", 1));
-            } else if resp.hovered() {
-                let d = ui.input(|i| i.smooth_scroll_delta.y);
-                if d.abs() > 0.5 {
-                    let _ = tx.send(Input::Scroll(x, y, d < 0.0, ((d.abs() / 40.0).ceil() as u32).max(1)));
-                } else if ui.input(|i| i.pointer.delta().length() > 3.0) {
-                    let _ = tx.send(Input::Move(x, y));
-                }
-            }
-        }
-        if !resp.has_focus() {
-            return;
-        }
-        ui.memory_mut(|m| m.set_focus_lock_filter(resp.id, egui::EventFilter { tab: true, horizontal_arrows: true, vertical_arrows: true, escape: true }));
-        let events = ui.input(|i| i.events.clone());
-        for e in events {
-            match e {
-                egui::Event::Text(t) => {
-                    let _ = tx.send(Input::Text(t));
-                }
-                egui::Event::Paste(t) => {
-                    let _ = tx.send(Input::Text(t));
-                }
-                egui::Event::Key { key, pressed: true, modifiers, .. } => {
-                    let name = match key {
-                        Key::Enter => "Return",
-                        Key::Tab => "Tab",
-                        Key::Backspace => "BackSpace",
-                        Key::Delete => "Delete",
-                        Key::Escape => "Escape",
-                        Key::ArrowUp => "Up",
-                        Key::ArrowDown => "Down",
-                        Key::ArrowLeft => "Left",
-                        Key::ArrowRight => "Right",
-                        Key::Home => "Home",
-                        Key::End => "End",
-                        Key::PageUp => "Prior",
-                        Key::PageDown => "Next",
-                        _ if modifiers.command || modifiers.ctrl || modifiers.alt => {
-                            let n = key.name();
-                            if n.chars().count() == 1 {
-                                n
-                            } else {
-                                continue;
-                            }
-                        }
-                        _ => continue,
-                    };
-                    let mut combo = vec![];
-                    if modifiers.command || modifiers.ctrl {
-                        combo.push("ctrl".to_string());
-                    }
-                    if modifiers.alt {
-                        combo.push("alt".to_string());
-                    }
-                    if modifiers.shift && (name.len() > 1 || !combo.is_empty()) {
-                        combo.push("shift".to_string());
-                    }
-                    combo.push(if name.len() == 1 { name.to_lowercase() } else { name.to_string() });
-                    let _ = tx.send(Input::Key(combo.join("+")));
-                }
-                _ => {}
-            }
         }
     }
 
-    // ---------- the floating bar while a bot drives this Mac ----------
-
     pub(super) fn local_control_bar(&mut self, ctx: &egui::Context) {
         if self.demo {
             return;

src/cloudbox.rs

@@ -206,6 +206,16 @@ pub fn vnc_password() -> String {
     pw
 }
 
+/// The password the box's VNC server really has: the one it was made with (its VNC_PASSWORD), so a
+/// viewer signs in even when this AnyBot's settings were made anew; else this AnyBot's own.
+pub fn box_vnc_password() -> String {
+    dk_timeout(&["exec", container(), "printenv", "VNC_PASSWORD"], 5_000)
+        .ok()
+        .map(|b| String::from_utf8_lossy(&b).trim().to_string())
+        .filter(|p| !p.is_empty())
+        .unwrap_or_else(vnc_password)
+}
+
 static BUSY: AtomicUsize = AtomicUsize::new(0);
 /// The person took control in the viewer: the bot's screen actions wait.
 pub static HELD_BY_PERSON: AtomicBool = AtomicBool::new(false);
@@ -417,6 +427,13 @@ fn docker() -> Command {
     c
 }
 
+/// A program in the box with its stdin and stdout piped to this side (as the bot user).
+pub fn exec_piped(args: &[&str]) -> Result<std::process::Child, String> {
+    let mut a = vec!["exec", "-i", "-u", "bot", container()];
+    a.extend_from_slice(args);
+    docker().args(&a).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().map_err(|e| format!("找不到 docker：{e}"))
+}
+
 /// The `docker …` prefix for shell strings (used by the command tools).
 fn docker_prefix() -> String {
     let host = docker_host();
@@ -1188,164 +1205,16 @@ pub fn thumbnail(width: u32) -> Option<image::RgbaImage> {
     image::open(&path).ok().map(|i| i.to_rgba8())
 }
 
-/// The screen now, for a small picture (no waking: None when the box isn't running): the live
+/// The screen now, for a small picture (no waking: None when the box isn't running). People watch
-/// picture when someone is watching, else one taken now.
+/// it live through VNC (vnc.rs).
 pub fn frame() -> Option<image::RgbaImage> {
     if !run_state().is_running() {
         return None;
     }
-    let jpeg = match live_latest() {
+    let jpeg = dk_timeout(&["exec", container(), "import", "-window", "root", "-display", DISPLAY, "-quality", "80", "jpeg:-"], 5_000).ok()?;
-        Some(j) => j,
-        None => std::sync::Arc::new(dk_timeout(&["exec", container(), "import", "-window", "root", "-display", DISPLAY, "-quality", "80", "jpeg:-"], 5_000).ok()?),
-    };
     image::load_from_memory(&jpeg).ok().map(|i| i.to_rgba8())
 }
 
-// ---------- the live picture (people watching) ----------
-// One lasting `docker exec` takes the desktop as a JPEG inside the box (ImageMagick, in C) a few
-// times a second and sends a picture only when it changed; a viewer asks for "a picture newer
-// than mine" and gets it the moment there is one. It used to be one exec per picture (a 1 MB
-// PNG, decoded and encoded again as a JPEG here) and a fixed pause after each: about one
-// picture a second, so a click showed its result a second or two later. It stops when nobody
-// has asked for a while.
-
-/// JPEG quality of the live picture (text stays readable; 60–90 KB for 1280×800).
-const LIVE_QUALITY: u8 = 70;
-/// Pause between looks at the screen inside the box: short while it is changing (a click's
-/// result, scrolling), longer while it stays still.
-const LIVE_GAP_BUSY: &str = "0.04";
-const LIVE_GAP_STILL: &str = "0.12";
-/// Stop taking pictures when no viewer asked for this long.
-const LIVE_IDLE: Duration = Duration::from_secs(10);
-
-struct Live {
-    seq: u64,
-    jpeg: std::sync::Arc<Vec<u8>>,
-    at: Instant,
-    asked: Instant,
-    running: bool,
-}
-
-fn live() -> &'static (Mutex<Live>, std::sync::Condvar) {
-    static L: OnceLock<(Mutex<Live>, std::sync::Condvar)> = OnceLock::new();
-    L.get_or_init(|| (Mutex::new(Live { seq: 0, jpeg: Default::default(), at: Instant::now(), asked: Instant::now(), running: false }), std::sync::Condvar::new()))
-}
-
-/// The latest live picture, when one was taken in the last few seconds.
-fn live_latest() -> Option<std::sync::Arc<Vec<u8>>> {
-    let l = live().0.lock().unwrap_or_else(|e| e.into_inner());
-    (l.running && l.seq > 0 && l.at.elapsed() < Duration::from_secs(3)).then(|| l.jpeg.clone())
-}
-
-/// A picture of the desktop newer than `after` (0: any), waiting up to `wait` for the screen to
-/// change; the same one again when it didn't. (number, JPEG); None when the box isn't running.
-pub fn live_frame(after: u64, wait: Duration) -> Option<(u64, std::sync::Arc<Vec<u8>>)> {
-    if !run_state().is_running() {
-        return None;
-    }
-    let (m, cv) = live();
-    let mut l = m.lock().unwrap_or_else(|e| e.into_inner());
-    l.asked = Instant::now();
-    if !l.running {
-        l.running = true;
-        std::thread::spawn(live_loop);
-    }
-    let until = Instant::now() + wait;
-    while l.seq == 0 || l.seq <= after {
-        let left = until.saturating_duration_since(Instant::now());
-        if left.is_zero() {
-            break;
-        }
-        l = cv.wait_timeout(l, left).map(|r| r.0).unwrap_or_else(|e| e.into_inner().0);
-    }
-    (l.seq > 0).then(|| (l.seq, l.jpeg.clone()))
-}
-
-/// The box's side: a picture whenever the screen changed, as "<bytes>\n<jpeg>", and "0\n" now
-/// and then while it stays still. Ending the docker exec on this side doesn't end the loop in
-/// the box (a still screen never writes, so it never noticed): it ends the one before it when
-/// it starts, and itself when a write fails (the "0" lines make sure one does).
-fn live_script() -> String {
-    format!(
-        "[ -f /tmp/anybot-live.pid ] && kill $(cat /tmp/anybot-live.pid) 2>/dev/null; echo $$ > /tmp/anybot-live.pid; f=/tmp/anybot-live-$$.jpg; trap 'rm -f $f' EXIT; prev=; still=0; \
-         while :; do import -window root -display {DISPLAY} -quality {LIVE_QUALITY} jpeg:$f 2>/dev/null || exit 3; h=$(cksum < $f); \
-         if [ \"$h\" != \"$prev\" ]; then prev=$h; still=0; {{ wc -c < $f && cat $f; }} || exit 0; sleep {LIVE_GAP_BUSY}; \
-         else still=$((still+1)); if [ $still -ge 15 ]; then still=0; echo 0 || exit 0; fi; sleep {LIVE_GAP_STILL}; fi; done"
-    )
-}
-
-/// End the picture loop in the box (see live_script).
-fn live_stop_in_box() {
-    let _ = dk_timeout(&["exec", container(), "sh", "-c", "[ -f /tmp/anybot-live.pid ] && kill $(cat /tmp/anybot-live.pid) 2>/dev/null; rm -f /tmp/anybot-live.pid; true"], 5_000);
-}
-
-fn live_loop() {
-    use std::io::{BufRead, Read};
-    let (m, cv) = live();
-    let stop = |m: &Mutex<Live>| m.lock().unwrap_or_else(|e| e.into_inner()).running = false;
-    let Ok(mut child) = docker().args(["exec", "-u", "bot", container(), "sh", "-c", &live_script()]).stdout(Stdio::piped()).stderr(Stdio::null()).spawn() else {
-        return stop(m);
-    };
-    let Some(out) = child.stdout.take() else {
-        let _ = child.kill();
-        return stop(m);
-    };
-    // Reading blocks while the screen stays the same; the watcher ends it when nobody asks.
-    let done = std::sync::Arc::new(AtomicBool::new(false));
-    let d2 = done.clone();
-    let reader = std::thread::spawn(move || {
-        let mut r = std::io::BufReader::new(out);
-        let mut line = String::new();
-        while !d2.load(Ordering::SeqCst) {
-            line.clear();
-            if r.read_line(&mut line).unwrap_or(0) == 0 {
-                break;
-            }
-            let Ok(n) = line.trim().parse::<usize>() else { break };
-            // "0": still there, nothing new.
-            if n == 0 {
-                continue;
-            }
-            let mut jpeg = vec![0u8; n];
-            if r.read_exact(&mut jpeg).is_err() {
-                break;
-            }
-            if !jpeg.starts_with(&[0xff, 0xd8]) {
-                continue;
-            }
-            let mut l = m.lock().unwrap_or_else(|e| e.into_inner());
-            l.seq += 1;
-            l.jpeg = std::sync::Arc::new(jpeg);
-            l.at = Instant::now();
-            cv.notify_all();
-        }
-        d2.store(true, Ordering::SeqCst);
-    });
-    loop {
-        std::thread::sleep(Duration::from_millis(500));
-        let idle = m.lock().unwrap_or_else(|e| e.into_inner()).asked.elapsed() > LIVE_IDLE;
-        if idle || done.load(Ordering::SeqCst) || !run_state().is_running() {
-            break;
-        }
-    }
-    done.store(true, Ordering::SeqCst);
-    let _ = child.kill();
-    let _ = child.wait();
-    if run_state().is_running() {
-        live_stop_in_box();
-    }
-    let _ = reader.join();
-    let mut l = m.lock().unwrap_or_else(|e| e.into_inner());
-    l.running = false;
-    // Viewers waiting on a picture that won't come get the last one now.
-    cv.notify_all();
-}
-
-/// The desktop's size (for a viewer mapping its taps), when known.
-pub fn screen_size() -> Option<(u32, u32)> {
-    box_size().ok()
-}
-
 /// The desktop's size in pixels (asked once per box).
 fn box_size() -> Result<(u32, u32), String> {
     if let Some(s) = shared().lock().unwrap_or_else(|e| e.into_inner()).size {
@@ -1605,8 +1474,8 @@ pub fn computer_run(name: &str, v: &Value) -> Result<(String, Option<String>), S
         }
         _ => {}
     }
-    if HELD_BY_PERSON.load(Ordering::SeqCst) {
+    if HELD_BY_PERSON.load(Ordering::SeqCst) || crate::vnc::person_active() {
-        return Err("使用者正在自己操作這台電腦（已取得控制權）。請等他交還控制權後再操作，或先截圖看看畫面。".into());
+        return Err("使用者正在自己操作這台電腦。請等他操作完（停下幾秒後）再繼續，或先截圖看看畫面。".into());
     }
     let (note, picture) = if name == "computer_use_open_app" {
         let note = open_on_box(v["app"].as_str().unwrap_or(""))?;
@@ -1623,85 +1492,13 @@ pub fn computer_run(name: &str, v: &Value) -> Result<(String, Option<String>), S
     }
 }
 
-// ---------- the viewer's own input (when the person takes control) ----------
+// ---------- text a person types in a viewer ----------
-// Same scripts, at the desktop's own pixels (the viewer shows it full size), no picture back,
-// through one lasting shell in the box (a click or a key no longer starts a docker exec of its
-// own: 50–200 ms each, more with a Docker host elsewhere). Typed text rides along as base64.
-
-fn input_shell() -> &'static Mutex<Option<std::process::Child>> {
-    static S: OnceLock<Mutex<Option<std::process::Child>>> = OnceLock::new();
-    S.get_or_init(|| Mutex::new(None))
-}
-
-/// One line for the lasting shell: the step's script, its text (if any) fed on its stdin.
-pub fn input_line(script: &str, stdin: Option<&str>) -> String {
-    let script = script.replace('\n', " ");
-    match stdin {
-        Some(t) => format!("printf %s {} | base64 -d | {{ {script}; }}\n", base64::engine::general_purpose::STANDARD.encode(t)),
-        None => format!("{{ {script}; }} </dev/null\n"),
-    }
-}
-
-fn send_input(script: &str, stdin: Option<&str>) {
-    use std::io::Write;
-    let line = input_line(script, stdin);
-    let mut sh = input_shell().lock().unwrap_or_else(|e| e.into_inner());
-    for _ in 0..2 {
-        let alive = sh.as_mut().map(|c| matches!(c.try_wait(), Ok(None))).unwrap_or(false);
-        if !alive {
-            *sh = docker()
-                .args(["exec", "-i", "-u", "bot", "-e", "DISPLAY=:1", container(), "sh", "-c", "while IFS= read -r l; do eval \"$l\" >/dev/null 2>&1; done"])
-                .stdin(Stdio::piped())
-                .stdout(Stdio::null())
-                .stderr(Stdio::null())
-                .spawn()
-                .ok();
-        }
-        let Some(c) = sh.as_mut() else { return };
-        if c.stdin.as_mut().map(|i| i.write_all(line.as_bytes()).and_then(|_| i.flush()).is_ok()).unwrap_or(false) {
-            return;
-        }
-        // The box restarted under it: start a new one and try once more.
-        let _ = c.kill();
-        let _ = c.wait();
-        *sh = None;
-    }
-}
-
-fn viewer(name: &str, v: Value) {
-    if let Ok((script, stdin, _)) = box_script(name, &v, (1.0, 1.0)) {
-        send_input(&script, stdin.as_deref());
-    }
-    touch();
-}
-
-pub fn viewer_click(x: i64, y: i64, button: &str, count: u32) {
-    viewer("computer_use_click", serde_json::json!({"x": x, "y": y, "button": button, "count": count.clamp(1, 3)}));
-}
-
-pub fn viewer_move(x: i64, y: i64) {
-    viewer("computer_use_mouse_move", serde_json::json!({"x": x, "y": y}));
-}
-
-pub fn viewer_scroll(x: i64, y: i64, down: bool, steps: u32) {
-    viewer("computer_use_scroll", serde_json::json!({"x": x, "y": y, "direction": if down { "down" } else { "up" }, "amount": steps.clamp(1, 20)}));
-}
-
-/// A key from a viewer: an X key name (「BackSpace」「Return」, ctrl+c…) or the bot's names.
-pub fn viewer_key(combo: &str) {
-    let c = combo.trim();
-    if c.is_empty() || !c.chars().all(|ch| ch.is_ascii_alphanumeric() || "+_-".contains(ch)) {
-        return;
-    }
-    // X key names pass straight through (they are what the viewers send).
-    let x = combo_to_x(c).unwrap_or_else(|_| c.to_string());
-    send_input(&format!("xdotool key --clearmodifiers {x}"), None);
-    touch();
-}
 
+/// Text from a person's viewer (any language): typed or pasted into the window in front, like
+/// the bot's typing. (Their mouse and keys go through VNC itself.)
 pub fn viewer_text(text: &str) {
     if !text.is_empty() {
-        send_input(&type_script(text), Some(text));
+        let _ = box_exec(&type_script(text), Some(text), false);
     }
     touch();
 }
@@ -1876,65 +1673,8 @@ mod tests {
         assert!(box_script("computer_use_scroll", &serde_json::json!({"direction": "sideways"}), s).is_err());
     }
 
-    #[test]
-    fn viewer_input_is_one_line() {
-        // A click: its script on one line, no stdin.
-        assert_eq!(input_line("xdotool mousemove 1 2 sleep 0.03 click --repeat 1 --delay 80 1", None), "{ xdotool mousemove 1 2 sleep 0.03 click --repeat 1 --delay 80 1; } </dev/null\n");
-        // Typed text rides as base64 (quotes, newlines, Chinese can't break the line).
-        let l = input_line(&type_script("a'b\nc"), Some("a'b\nc"));
-        assert!(l.starts_with("printf %s YSdiCmM= | base64 -d | { xdotool type") && l.ends_with("; }\n") && l.matches('\n').count() == 1, "{l}");
-    }
-
-    /// Uses the real Docker: `cargo test --release -- --ignored box_live_is_quick --nocapture`.
-    /// A viewer gets a picture at once, the next one soon after the screen changes, several a
-    /// second while it keeps changing, and the box stops taking pictures when nobody watches.
-    #[test]
-    #[ignore]
-    fn box_live_is_quick() {
-        ensure_running().expect("box starts");
-        let t = Instant::now();
-        let (seq, jpeg) = live_frame(0, Duration::from_secs(5)).expect("a first picture");
-        println!("first picture: {} ms, {} KB", t.elapsed().as_millis(), jpeg.len() / 1024);
-        assert!(jpeg.starts_with(&[0xff, 0xd8]));
-        // A still screen: the same picture after the wait. (An open web page may animate; then
-        // a new one comes, which is right too.)
-        let t = Instant::now();
-        let (same, _) = live_frame(seq, Duration::from_millis(1500)).unwrap();
-        println!("after {} ms: {}", t.elapsed().as_millis(), if same == seq { "same picture" } else { "a newer one (something moves)" });
-        // Input (through the lasting input shell, already open) moves the pointer: a new
-        // picture soon.
-        viewer_move(600, 300);
-        let (seq, _) = live_frame(seq, Duration::from_secs(5)).unwrap();
-        std::thread::sleep(Duration::from_millis(600));
-        let t = Instant::now();
-        viewer_move(700, 300);
-        let (next, _) = live_frame(seq, Duration::from_secs(5)).unwrap();
-        let react = t.elapsed();
-        println!("input → new picture: {} ms", react.as_millis());
-        assert!(next > seq && react < Duration::from_millis(600), "{react:?}");
-        // A screen that keeps changing: pictures a second.
-        let _ = box_exec("(setsid xterm -geometry 60x20+40+40 -e sh -c 'while :; do date +%s.%N; done' >/dev/null 2>&1 &)", None, false);
-        std::thread::sleep(Duration::from_millis(1200));
-        let (mut n, _) = live_frame(0, Duration::from_secs(2)).unwrap();
-        let (start, t) = (n, Instant::now());
-        while t.elapsed() < Duration::from_secs(3) {
-            n = live_frame(n, Duration::from_secs(2)).unwrap().0;
-        }
-        let fps = (n - start) as f64 / t.elapsed().as_secs_f64();
-        println!("changing screen: {fps:.1} pictures a second");
-        // (pkill straight from docker exec: inside `sh -c` it matched its own shell first.)
-        let _ = dk(&["exec", container(), "pkill", "-f", "date +%s.%N"]);
-        assert!(fps >= 4.0, "{fps}");
-        // Nobody asks: it stops, in the box too.
-        std::thread::sleep(LIVE_IDLE + Duration::from_secs(2));
-        assert!(!live().0.lock().unwrap().running);
-        let left = dk(&["exec", container(), "sh", "-c", "ps -eo args | grep -c '[a]nybot-live' || true"]).unwrap_or_default();
-        assert_eq!(left.trim(), "0", "picture loops left in the box");
-    }
-
     /// Uses the real Docker: `cargo test --release -- --ignored box_screen_is_exact --nocapture`.
-    /// The bot's clicks land where its picture says, a phone's tap where the phone's picture says,
+    /// The bot's clicks land where its picture says, the same spot twice doesn't hang, typing (ASCII, Chinese, a key by name) reaches the app,
-    /// the same spot twice doesn't hang, typing (ASCII, Chinese, a key by name) reaches the app,
     /// and every step comes back with the screen after it.
     #[test]
     #[ignore]
@@ -1960,9 +1700,6 @@ mod tests {
         println!("two clicks with pictures: {} ms", t.elapsed().as_millis());
         assert!(t.elapsed() < Duration::from_secs(6));
         assert_eq!(pointer(), ((40.0 * w as f64 / fw as f64).round() as i64, (40.0 * h as f64 / fh as f64).round() as i64));
-        // A phone's tap is in the full-size frame it was sent.
-        viewer_click(321, 123, "left", 1);
-        assert_eq!(pointer(), (321, 123));
         // Typing reaches the app in front: a fresh terminal, ASCII then Chinese, a key by name, return.
         let _ = dk(&["exec", "-u", "bot", container(), "rm", "-f", "/home/bot/exact.txt"]);
         let (txt, img) = computer_run("computer_use_open_app", &serde_json::json!({"app": "終端機"})).unwrap();

src/computer.rs

@@ -536,13 +536,29 @@ fn open_app(app: &str) -> Result<String, String> {
     if app.is_empty() {
         return Err("請給要打開的 App 名稱".into());
     }
-    crate::desktop::launch(app).or_else(|e| match app_named(app) {
+    launch(app).or_else(|e| match app_named(app) {
-        Some(real) if real != app => crate::desktop::launch(&real),
+        Some(real) if real != app => launch(&real),
         _ => Err(e),
     })?;
     Ok(format!("已打開「{app}」，放在最前面。"))
 }
 
+/// Open an app or bring it forward. macOS: LaunchServices (`open -a`), which needs no
+/// permission (agent-desktop's launch waited on the window through accessibility and failed
+/// without 輔助使用 now and then); elsewhere agent-desktop's.
+fn launch(app: &str) -> Result<(), String> {
+    #[cfg(target_os = "macos")]
+    {
+        let out = std::process::Command::new("open").args(["-a", app]).output().map_err(|e| e.to_string())?;
+        if out.status.success() {
+            return Ok(());
+        }
+        Err(format!("找不到「{app}」這個 App"))
+    }
+    #[cfg(not(target_os = "macos"))]
+    crate::desktop::launch(app)
+}
+
 /// Bring the app a control belongs to in front, so a click at its spot lands in it (and not in
 /// a window lying over it). Nothing to do when it is in front already.
 fn bring_forward(app: &str) {
@@ -555,7 +571,7 @@ fn bring_forward(app: &str) {
     if app.is_empty() || front.eq_ignore_ascii_case(app) {
         return;
     }
-    if crate::desktop::launch(app).is_ok() {
+    if launch(app).is_ok() {
         std::thread::sleep(std::time::Duration::from_millis(300));
     }
 }

src/desktop.rs

@@ -259,7 +259,8 @@ pub fn read_app(app: &str) -> Result<(String, Vec<String>), String> {
     Ok((app_name, lines))
 }
 
-/// Open an app, or bring it to the front when it runs.
+/// Open an app, or bring it to the front when it runs (macOS uses LaunchServices instead).
+#[cfg(not(target_os = "macos"))]
 pub fn launch(app: &str) -> Result<(), String> {
     run("launch", &serde_json::json!({"app": app})).map(|_| ())
 }

src/main.rs

@@ -47,6 +47,7 @@ mod tools;
 mod triggers;
 mod vault;
 mod vault_store;
+mod vnc;
 mod voice;
 mod tunnel;
 mod web;

src/remote.rs

@@ -322,6 +322,8 @@ pub fn handle(req: Req, via_relay: bool) -> Resp {
     let path: Vec<&str> = parts.iter().map(|s| *s).collect();
     match (req.method.as_str(), path.as_slice()) {
         ("GET", []) | ("GET", ["index.html"]) => return page(),
+        ("GET", ["screen"]) => return Resp::bytes("text/html; charset=utf-8", SCREEN_PAGE.as_bytes().to_vec()).with_header("Content-Security-Policy", "default-src 'self'; img-src 'self' blob: data:; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline'; connect-src 'self' ws: wss:"),
+        ("GET", ["novnc.js"]) => return Resp::bytes("text/javascript; charset=utf-8", NOVNC.as_bytes().to_vec()).with_header("Cache-Control", "public, max-age=86400"),
         ("GET", ["manifest.webmanifest"]) => return Resp::bytes("application/manifest+json", MANIFEST.as_bytes().to_vec()),
         ("GET", ["icon.svg"]) => return Resp::bytes("image/svg+xml", ICON.as_bytes().to_vec()),
         ("GET", [name]) if ICON_PNGS.iter().any(|(n, _)| n == name) => return Resp::bytes("image/png", ICON_PNGS.iter().find(|(n, _)| n == name).map(|(_, b)| b.to_vec()).unwrap_or_default()),
@@ -415,7 +417,7 @@ pub fn handle(req: Req, via_relay: bool) -> Resp {
             hub().push(cmd);
             Resp::json(&json!({"ok": true}))
         }
-        ("GET", ["api", "screen", which]) => screen(which, &s, req.query.get("after").and_then(|a| a.parse().ok())),
+        ("GET", ["api", "screen", which]) => screen(which, &s),
         ("POST", ["api", "screen", which]) => screen_input(which, &req.json(), &s),
         ("POST", ["api", "device", "run"]) => device_run(&req.json(), &s, via_relay, req.header("x-anybot-from")),
         ("GET", ["api", "sync"]) => {
@@ -435,24 +437,16 @@ pub fn handle(req: Req, via_relay: bool) -> Resp {
             Err(e) => Resp::error(503, &e),
         },
         ("POST", ["api", "sync", "upload", id]) => upload(id.parse().unwrap_or(0), &req),
-        // A screen of this brain watches (and may take over) the bots' own computer, which lives
+        // A screen of this brain shows the bots' own computer, which lives here: a small picture
-        // here: its picture on that screen used to be its own computer's, not this one's.
+        // (people watch it live through VNC, see vnc.rs).
-        ("GET", ["api", "sync", "screen"]) => {
+        ("GET", ["api", "sync", "screen"]) => match crate::cloudbox::thumbnail(320) {
-            if req.query.get("thumb").map(|t| t == "1").unwrap_or(false) {
+            Some(img) => match crate::computer::encode_jpeg(&image::DynamicImage::ImageRgba8(img).to_rgb8()) {
-                match crate::cloudbox::thumbnail(320) {
+                Some(j) => Resp::bytes("image/jpeg", j),
-                    Some(img) => {
+                None => Resp::error(500, "畫面編碼失敗"),
-                        let rgb = image::DynamicImage::ImageRgba8(img).to_rgb8();
+            },
-                        match crate::computer::encode_jpeg(&rgb) {
+            None => Resp::error(404, "還沒有畫面"),
-                            Some(j) => Resp::bytes("image/jpeg", j),
+        },
-                            None => Resp::error(500, "畫面編碼失敗"),
+        // 接管 / 我完成了 from a screen: the bot's steps wait (or not) for the person.
-                        }
-                    }
-                    None => Resp::error(404, "還沒有畫面"),
-                }
-            } else {
-                screen("box", &s, req.query.get("after").and_then(|a| a.parse().ok()))
-            }
-        }
         ("POST", ["api", "sync", "screen"]) => {
             let v = req.json();
             match v["kind"].as_str().unwrap_or("") {
@@ -460,25 +454,11 @@ pub fn handle(req: Req, via_relay: bool) -> Resp {
                     crate::cloudbox::HELD_BY_PERSON.store(v["on"].as_bool().unwrap_or(false), Ordering::SeqCst);
                     Resp::json(&json!({"ok": true}))
                 }
-                "viewer" => {
+                "wake" => match crate::cloudbox::ensure_running() {
-                    crate::cloudbox::set_viewer_open(v["on"].as_bool().unwrap_or(false));
+                    Ok(()) => Resp::json(&json!({"ok": true})),
-                    Resp::json(&json!({"ok": true}))
+                    Err(e) => Resp::error(500, &e),
-                }
+                },
-                // The person's own hands on the picture (frame pixels, like this computer's own
+                _ => Resp::error(400, "不認得的操作"),
-                // viewer; not the bot's tools, which stop while the person holds it).
-                "click" | "move" | "scroll" | "key" | "text" => {
-                    crate::cloudbox::touch();
-                    let (x, y) = (v["x"].as_i64().unwrap_or(0), v["y"].as_i64().unwrap_or(0));
-                    match v["kind"].as_str().unwrap_or("") {
-                        "click" => crate::cloudbox::viewer_click(x, y, if v["button"].as_str() == Some("right") { "right" } else { "left" }, v["count"].as_u64().unwrap_or(1).clamp(1, 3) as u32),
-                        "move" => crate::cloudbox::viewer_move(x, y),
-                        "scroll" => crate::cloudbox::viewer_scroll(x, y, v["down"].as_bool().unwrap_or(true), v["steps"].as_u64().unwrap_or(1).clamp(1, 20) as u32),
-                        "key" => crate::cloudbox::viewer_key(v["key"].as_str().unwrap_or("")),
-                        _ => crate::cloudbox::viewer_text(v["text"].as_str().unwrap_or("")),
-                    }
-                    Resp::json(&json!({"ok": true}))
-                }
-                _ => screen_input("box", &v, &s),
             }
         }
         ("POST", ["api", "sync", "models"]) => match hub().ask_sync_for(SyncKind::Models(req.json()["provider"].as_str().unwrap_or("").to_string()), 45) {
@@ -606,27 +586,10 @@ fn upload(bot: u64, req: &Req) -> Resp {
 }
 
 /// A frame of the bot's computer ("box") or this Mac ("mac") as JPEG.
-/// How long a viewer's request for a newer picture waits for the screen to change (well under
+/// This Mac's screen for the phone (when allowed). The bots' computer is watched through VNC
-/// the account server's forward timeout).
+/// (the page /screen, vnc.rs).
-const LIVE_WAIT_MS: u64 = 8_000;
+fn screen(which: &str, s: &Settings) -> Resp {
-
-/// A screen's picture for a viewer. The box: its live picture, as it was taken (no decoding and
-/// encoding again here); with `after` (the number of the picture the viewer has) it answers the
-/// moment the screen changes, or with the same picture after a few seconds. X-Seq is its number,
-/// X-Size the desktop's size (what the viewer's taps are in).
-fn screen(which: &str, s: &Settings, after: Option<u64>) -> Resp {
     match which {
-        "box" => {
-            crate::cloudbox::touch();
-            let wait = Duration::from_millis(if after.is_some() { LIVE_WAIT_MS } else { 3_000 });
-            match crate::cloudbox::live_frame(after.unwrap_or(0), wait) {
-                Some((seq, jpeg)) => {
-                    let (w, h) = crate::cloudbox::screen_size().unwrap_or_default();
-                    Resp::bytes("image/jpeg", jpeg.to_vec()).with_header("X-Size", &format!("{w}x{h}")).with_header("X-Seq", &seq.to_string())
-                }
-                None => Resp::error(503, "Bot 的電腦在休息，按「叫醒」"),
-            }
-        }
         "mac" if s.mac_from_phone => {
             let (text, ok, img) = crate::computer::run_by("computer_use_screenshot", "{}", "手機");
             match (ok, img) {
@@ -644,41 +607,15 @@ fn screen(which: &str, s: &Settings, after: Option<u64>) -> Resp {
     }
 }
 
-/// The phone taps, types or scrolls on a screen, as the person, in the pixels of the picture it
+/// The phone taps, types or scrolls on this Mac's screen, as the person, in the pixels of the
-/// was sent: the box's frame is its desktop at full size, this computer's is the bot-sized
+/// picture it was sent (shot_size). The bots' computer takes the person's input through VNC.
-/// picture (shot_size). It used to go through the bot's tools, which mapped the tap with the
-/// bot's last screenshot (a different size: taps landed a quarter off, or failed with "take a
-/// screenshot first") and refused it while the person held the screen.
 fn screen_input(which: &str, v: &Value, s: &Settings) -> Resp {
-    let kind = v["kind"].as_str().unwrap_or("");
-    if which == "box" && kind == "wake" {
-        return match crate::cloudbox::ensure_running() {
-            Ok(()) => Resp::json(&json!({"ok": true})),
-            Err(e) => Resp::error(500, &e),
-        };
-    }
     let (x, y) = (v["x"].as_f64().unwrap_or(0.0), v["y"].as_f64().unwrap_or(0.0));
     let down = v["direction"].as_str().unwrap_or("down") != "up";
     let amount = v["amount"].as_i64().unwrap_or(3).clamp(1, 20);
     match which {
-        "box" => {
-            if !matches!(kind, "click" | "type" | "key" | "scroll") {
-                return Resp::error(400, "不認得的操作");
-            }
-            if let Err(e) = crate::cloudbox::ensure_running() {
-                return Resp::error(500, &e);
-            }
-            let (xi, yi) = (x.round() as i64, y.round() as i64);
-            match kind {
-                "click" => crate::cloudbox::viewer_click(xi, yi, if v["button"].as_str() == Some("right") { "right" } else { "left" }, v["count"].as_u64().unwrap_or(1) as u32),
-                "type" => crate::cloudbox::viewer_text(v["text"].as_str().unwrap_or("")),
-                "key" => crate::cloudbox::viewer_key(v["key"].as_str().unwrap_or("")),
-                _ => crate::cloudbox::viewer_scroll(xi, yi, down, amount as u32),
-            }
-            Resp::json(&json!({"ok": true}))
-        }
         "mac" if s.mac_from_phone => {
-            let (name, args) = match kind {
+            let (name, args) = match v["kind"].as_str().unwrap_or("") {
                 "click" => ("computer_use_click", json!({"x": x, "y": y, "button": v["button"].as_str().unwrap_or("left"), "count": v["count"].as_u64().unwrap_or(1)})),
                 "type" => ("computer_use_typing", json!({"value": v["text"]})),
                 "key" => ("computer_use_press_key", json!({"key": v["key"]})),
@@ -690,7 +627,8 @@ fn screen_input(which: &str, v: &Value, s: &Settings) -> Resp {
                 Err(e) => Resp::error(500, &e),
             }
         }
-        _ => Resp::error(403, "這台 Mac 沒有開放從手機操作"),
+        "mac" => Resp::error(403, "這台 Mac 沒有開放從手機操作"),
+        _ => Resp::error(404, "沒有這個畫面"),
     }
 }
 
@@ -868,10 +806,69 @@ pub fn set_lan(on: bool) -> Result<u16, String> {
         .ok_or("找不到可用的連接埠")?;
     let port = listener.local_addr().map_err(|e| e.to_string())?.port();
     *LAN_BOUND.lock().unwrap() = Some(port);
-    std::thread::spawn(move || httpd::serve(listener, Arc::new(|r| handle(r, false)), Arc::new(move || LAN_GEN.load(Ordering::SeqCst) == gen)));
+    std::thread::spawn(move || httpd::serve_upgrade(listener, Arc::new(|r| handle(r, false)), Some(Arc::new(lan_vnc)), Arc::new(move || LAN_GEN.load(Ordering::SeqCst) == gen)));
     Ok(port)
 }
 
+/// The phone page on the local network watching the bots' computer: /api/vnc, signed in with
+/// the phone's password like every other request here; the VNC stream goes straight to the box.
+fn lan_vnc(req: Req, mut c: std::net::TcpStream) {
+    use std::io::Write;
+    use tungstenite::{protocol::Role, Message, WebSocket};
+    if req.path != "/api/vnc" {
+        return httpd::write_response(&mut c, &Resp::error(404, "沒有這個網址"));
+    }
+    if devices_ok(&req.bearer()).is_none() {
+        return httpd::write_response(&mut c, &Resp::error(401, "請先登入"));
+    }
+    let key = req.header("sec-websocket-key").to_string();
+    let accept = tungstenite::handshake::derive_accept_key(key.as_bytes());
+    if key.is_empty() || c.write_all(format!("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: {accept}\r\n\r\n").as_bytes()).is_err() {
+        return;
+    }
+    let _ = c.set_nodelay(true);
+    let _ = c.set_read_timeout(Some(Duration::from_millis(2)));
+    let mut ws = WebSocket::from_raw_socket(c, Role::Server, None);
+    let (tx, rx) = std::sync::mpsc::channel::<Vec<u8>>();
+    let send = move |b: &[u8]| tx.send(b.to_vec()).is_ok();
+    let Ok(mut session) = crate::vnc::open(send.clone()) else { return };
+    if !send(crate::vnc::GREETING) {
+        return;
+    }
+    'live: loop {
+        match ws.read() {
+            Ok(Message::Binary(b)) => {
+                if !session.feed(&b, &send) {
+                    break;
+                }
+            }
+            Ok(Message::Text(t)) => crate::vnc::person_typed(&t),
+            Ok(Message::Close(_)) => break,
+            Ok(_) => {}
+            Err(tungstenite::Error::Io(e)) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => {}
+            Err(_) => break,
+        }
+        loop {
+            match rx.try_recv() {
+                // An empty send: the box's side ended.
+                Ok(b) if b.is_empty() => break 'live,
+                Ok(b) => {
+                    if ws.write(Message::Binary(b.into())).is_err() {
+                        break 'live;
+                    }
+                }
+                Err(_) => break,
+            }
+        }
+        match ws.flush() {
+            Err(tungstenite::Error::Io(e)) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => {}
+            Err(_) => break,
+            Ok(()) => {}
+        }
+    }
+    let _ = ws.close(None);
+}
+
 /// Addresses a phone can open (local network and Tailscale).
 pub fn addresses(port: u16) -> Vec<String> {
     // macOS: ifconfig ("inet 192.168.1.5 netmask…"); Linux: `ip -4 -o addr` ("… inet 192.168.1.5/24 …").
@@ -1212,16 +1209,11 @@ fn to_brain(s: &Settings, method: &str, rest: &str, query: &[(&str, String)], bo
 }
 
 fn to_brain_raw(s: &Settings, method: &str, rest: &str, query: &[(&str, String)], body: &[u8], secs: u64) -> Result<(u16, Vec<u8>), String> {
-    to_brain_full(s, method, rest, query, body, secs).map(|(st, b, _)| (st, b))
-}
-
-/// Same, with the answer's headers.
-fn to_brain_full(s: &Settings, method: &str, rest: &str, query: &[(&str, String)], body: &[u8], secs: u64) -> Result<(u16, Vec<u8>, Vec<(String, String)>), String> {
     if s.brain_id.is_empty() {
         return Err("沒有選大腦".into());
     }
     if let Some(r) = crate::tunnel::call(&s.brain_id, method, &format!("/api/sync{rest}"), query, body, secs) {
-        return r.map(|a| (a.status, a.body, a.headers));
+        return r.map(|a| (a.status, a.body));
     }
     let q: Vec<String> = query.iter().map(|(k, v)| format!("{k}={}", url_escape(v))).collect();
     let url = brain_url(s, &format!("{rest}{}{}", if q.is_empty() { "" } else { "?" }, q.join("&")))?;
@@ -1229,9 +1221,8 @@ fn to_brain_full(s: &Settings, method: &str, rest: &str, query: &[(&str, String)
     let r = if method == "GET" { agent(secs).get(&url).header("Authorization", &auth).call() } else { agent(secs).post(&url).header("Authorization", &auth).header("Content-Type", "application/octet-stream").send(body) };
     let mut resp = r.map_err(|e| format!("連不到帳號伺服器：{}", why_unreachable(&e)))?;
     let st = resp.status().as_u16();
-    let headers = resp.headers().iter().filter_map(|(k, v)| Some((k.as_str().to_string(), v.to_str().ok()?.to_string()))).collect();
     let bytes = resp.body_mut().with_config().limit(200 * 1024 * 1024).read_to_vec().map_err(|_| "和大腦的連線到一半斷掉了".to_string())?;
-    Ok((st, bytes, headers))
+    Ok((st, bytes))
 }
 
 /// What changed at the brain since `since` (waits up to LONG_POLL_SECS when nothing has).
@@ -1259,15 +1250,21 @@ pub fn sync_screen(s: &Settings, thumb: bool) -> Result<Vec<u8>, String> {
     Ok(bytes)
 }
 
-/// The brain's bots' computer, live: a picture newer than number `after` (or the same one when
+/// A link for this account's browser to watch the brain's computer (/screen on the account
-/// the screen didn't change for a few seconds), with its number.
+/// server; one-time, it works for ten minutes).
-pub fn sync_live(s: &Settings, after: u64) -> Result<(u64, Vec<u8>), String> {
+pub fn screen_ticket(s: &Settings) -> Result<String, String> {
-    let (st, bytes, headers) = to_brain_full(s, "GET", "/screen", &[("after", after.to_string())], &[], LIVE_WAIT_MS / 1000 + 15)?;
+    if s.relay_url.is_empty() || s.brain_id.is_empty() {
-    if !(200..300).contains(&st) {
+        return Err("這台沒有連到帳號的大腦".into());
-        return Err(why_refused(st, &serde_json::from_slice(&bytes).unwrap_or(Value::Null), "大腦"));
     }
-    let seq = headers.iter().find(|(k, _)| k.eq_ignore_ascii_case("x-seq")).and_then(|(_, v)| v.parse().ok()).unwrap_or(0);
+    let url = format!("{}/relay/screen-ticket", s.relay_url.trim_end_matches('/'));
-    Ok((seq, bytes))
+    let mut r = agent(15)
+        .post(&url)
+        .header("Authorization", &format!("Bearer {}", s.host_token))
+        .header("Content-Type", "application/json")
+        .send(json!({"host": s.brain_id}).to_string().as_bytes())
+        .map_err(|e| format!("連不到帳號伺服器：{}", why_unreachable(&e)))?;
+    let v: Value = r.body_mut().read_json().map_err(|_| "帳號伺服器的回答看不懂".to_string())?;
+    v["ticket"].as_str().map(String::from).ok_or_else(|| "帳號伺服器沒有給看畫面的連結（它可能是舊版，請更新）".into())
 }
 
 /// A file the brain keeps (an attachment shown in a chat), as bytes.
@@ -1526,6 +1523,9 @@ pub fn devices_online(s: &Settings) -> Vec<(String, String, bool)> {
 // ---------- the phone page ----------
 
 const PAGE: &str = include_str!("../mobile/index.html");
+/// The bots' computer, watched and used in a browser (noVNC; see vnc.rs).
+const SCREEN_PAGE: &str = include_str!("../mobile/screen.html");
+const NOVNC: &str = include_str!("../mobile/novnc.js");
 pub const MANIFEST: &str = r##"{"name":"AnyBot","short_name":"AnyBot","start_url":"./","display":"standalone","background_color":"#121212","theme_color":"#121212","icons":[{"src":"icon.svg","sizes":"any","type":"image/svg+xml"},{"src":"icon-192.png","sizes":"192x192","type":"image/png","purpose":"any maskable"},{"src":"icon-512.png","sizes":"512x512","type":"image/png","purpose":"any maskable"}]}"##;
 /// The app's icon, the same drawing everywhere (icons/anybot.svg is the Mac/Linux one).
 pub const ICON: &str = include_str!("../mobile/icon.svg");

src/tunnel.rs

@@ -26,7 +26,6 @@ const CUT: &str = "和帳號伺服器的連線斷了，正在重新連線";
 pub struct Answer {
     pub status: u16,
     pub body: Vec<u8>,
-    pub headers: Vec<(String, String)>,
 }
 
 type Calls = Arc<Mutex<HashMap<String, Sender<Result<Answer, String>>>>>;
@@ -229,8 +228,15 @@ pub fn run(url: &str, token: &str, alive: &dyn Fn() -> bool, status: &dyn Fn(&st
     true
 }
 
+/// How long a turn of the connection's loop waits for something to read before sending what is
+/// queued: short while things are moving (a live screen), longer when quiet. A fixed 40 ms
+/// added up to 40 ms to every message sent, on each side.
+const READ_WAIT_BUSY: Duration = Duration::from_millis(2);
+const READ_WAIT_QUIET: Duration = Duration::from_millis(40);
+
 fn keep(mut ws: WebSocket<Stream>, serial: u64, alive: &dyn Fn() -> bool) {
-    let _ = ws.get_ref().tcp().set_read_timeout(Some(Duration::from_millis(40)));
+    let _ = ws.get_ref().tcp().set_read_timeout(Some(READ_WAIT_QUIET));
+    let (mut moved, mut busy) = (Instant::now() - Duration::from_secs(60), false);
     let (tx, rx) = channel::<Vec<u8>>();
     let calls: Calls = Arc::default();
     *link().lock().unwrap() = Some(Link { tx: tx.clone(), calls: calls.clone(), serial });
@@ -238,9 +244,15 @@ fn keep(mut ws: WebSocket<Stream>, serial: u64, alive: &dyn Fn() -> bool) {
     let _ = tx.send(wire::pack(&json!({"t": "hello", "brain": s.brain_id, "hands": s.device_link}), &[]));
     let (mut heard, mut pinged) = (Instant::now(), Instant::now());
     'live: while alive() {
+        let now_busy = moved.elapsed() < Duration::from_secs(2);
+        if now_busy != busy {
+            busy = now_busy;
+            let _ = ws.get_ref().tcp().set_read_timeout(Some(if busy { READ_WAIT_BUSY } else { READ_WAIT_QUIET }));
+        }
         match ws.read() {
             Ok(Message::Binary(b)) => {
                 heard = Instant::now();
+                moved = heard;
                 if let Some((head, body)) = wire::unpack(&b) {
                     // The relay says this machine was unregistered (signed out): stop for good.
                     if head["t"] == "bye" {
@@ -255,6 +267,7 @@ fn keep(mut ws: WebSocket<Stream>, serial: u64, alive: &dyn Fn() -> bool) {
             Err(_) => break,
         }
         while let Ok(f) = rx.try_recv() {
+            moved = Instant::now();
             if ws.write(Message::Binary(f.into())).is_err() {
                 break 'live;
             }
@@ -280,14 +293,72 @@ fn keep(mut ws: WebSocket<Stream>, serial: u64, alive: &dyn Fn() -> bool) {
         *l = None;
     }
     drop(l);
+    streams().lock().unwrap().clear();
     // Whoever waits on an answer hears now, not after a timeout.
     for (_, w) in calls.lock().unwrap().drain() {
         let _ = w.send(Err(CUT.into()));
     }
 }
 
+/// People watching the bots' computer through the account server: one VNC stream each, by the
+/// relay's stream id (vopen / vdata / vtext / vclose).
+fn streams() -> &'static Mutex<HashMap<String, crate::vnc::Session>> {
+    static S: OnceLock<Mutex<HashMap<String, crate::vnc::Session>>> = OnceLock::new();
+    S.get_or_init(Default::default)
+}
+
+fn on_stream(kind: &str, sid: String, body: Vec<u8>, tx: &Sender<Vec<u8>>) {
+    let send = {
+        let (tx, sid) = (tx.clone(), sid.clone());
+        move |b: &[u8]| tx.send(wire::pack(&json!({"t": "vdata", "sid": sid}), b)).is_ok()
+    };
+    let close = {
+        let (tx, sid) = (tx.clone(), sid.clone());
+        move |why: &str| {
+            streams().lock().unwrap().remove(&sid);
+            let _ = tx.send(wire::pack(&json!({"t": "vclose", "sid": sid, "why": why}), &[]));
+        }
+    };
+    match kind {
+        "vopen" => {
+            std::thread::spawn(move || {
+                let out = {
+                    let (send, close) = (send.clone(), close.clone());
+                    move |b: &[u8]| if b.is_empty() { close("這台電腦的畫面關了"); false } else { send(b) }
+                };
+                match crate::vnc::open(out) {
+                    Ok(s) => {
+                        streams().lock().unwrap().insert(sid, s);
+                        // Only now: the viewer answers this at once.
+                        if !send(crate::vnc::GREETING) {
+                            close("");
+                        }
+                    }
+                    Err(e) => close(&e),
+                }
+            });
+        }
+        "vdata" => {
+            let mut all = streams().lock().unwrap();
+            let ok = all.get_mut(&sid).map(|s| s.feed(&body, &send)).unwrap_or(false);
+            drop(all);
+            if !ok {
+                close("這台電腦的畫面關了");
+            }
+        }
+        "vtext" => crate::vnc::person_typed(&String::from_utf8_lossy(&body)),
+        _ => {
+            streams().lock().unwrap().remove(&sid);
+        }
+    }
+}
+
 fn on_frame(head: Value, body: Vec<u8>, tx: &Sender<Vec<u8>>, calls: &Calls) {
-    match head["t"].as_str().unwrap_or("") {
+    let t = head["t"].as_str().unwrap_or("");
+    if matches!(t, "vopen" | "vdata" | "vtext" | "vclose") {
+        return on_stream(t, head["sid"].as_str().unwrap_or("").to_string(), body, tx);
+    }
+    match t {
         "job" => {
             let tx = tx.clone();
             std::thread::spawn(move || {
@@ -310,8 +381,7 @@ fn on_frame(head: Value, body: Vec<u8>, tx: &Sender<Vec<u8>>, calls: &Calls) {
         "ret" => {
             if let Some(w) = calls.lock().unwrap().remove(head["id"].as_str().unwrap_or("")) {
                 let body = if head["z"].as_bool().unwrap_or(false) { unzip(&body) } else { Ok(body) };
-                let headers = head["headers"].as_array().into_iter().flatten().filter_map(|x| Some((x[0].as_str()?.to_string(), x[1].as_str()?.to_string()))).collect();
+                let _ = w.send(body.map(|body| Answer { status: head["status"].as_u64().unwrap_or(502) as u16, body }));
-                let _ = w.send(body.map(|body| Answer { status: head["status"].as_u64().unwrap_or(502) as u16, body, headers }));
             }
         }
         _ => {}

src/vnc.rs

@@ -0,0 +1,263 @@
+// People watching and helping on the bots' computer: VNC, carried to them as a stream.
+//
+// The box already runs a VNC server (x11vnc, inside the container, with a password). A person's
+// viewer (noVNC in a browser: the phone, or any computer through the account server) speaks VNC
+// to this proxy, which reaches the box's server through one `docker exec` (a few lines of
+// Python in the box joining its stdin/stdout to 127.0.0.1:5900). VNC sends only what changed on
+// the screen and every mouse move and key as it happens, so watching is live and helping is
+// like sitting at it — unlike pictures sent one by one.
+//
+// The proxy signs in to the box with its password itself and offers the viewer "no password",
+// so the password never leaves this AnyBot; whoever reaches the proxy was already let in (an
+// account session or one-time ticket at the account server, or the phone's password here).
+// While the person clicks or types, the bot's own steps wait (HELD_BY_PERSON), and go on a
+// little after they stop (person_active).
+
+use std::io::{Read, Write};
+use std::process::{Child, ChildStdin};
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Mutex;
+use std::time::{Duration, Instant};
+
+/// The bot waits this long after the person's last click or key before it goes on.
+const HELD_AFTER_INPUT: Duration = Duration::from_secs(8);
+
+/// Joins the box's VNC server to stdin/stdout, and ends when either side does (a loop left in
+/// the box would hold the connection open).
+const BRIDGE: &str = r#"import os,select,socket
+s=socket.create_connection(('127.0.0.1',5900))
+while True:
+ r=select.select([s,0],[],[])[0]
+ if s in r:
+  d=s.recv(65536)
+  if not d: break
+  os.write(1,d)
+ if 0 in r:
+  d=os.read(0,65536)
+  if not d: break
+  s.sendall(d)
+"#;
+
+/// Viewers connected now (the box stays awake while there is one).
+static OPEN: AtomicUsize = AtomicUsize::new(0);
+/// When the person last clicked or typed through a viewer.
+static LAST_INPUT: Mutex<Option<Instant>> = Mutex::new(None);
+
+/// VNC authentication: the 16-byte challenge encrypted with DES, the key being the password's
+/// first 8 bytes with each byte's bits reversed (RFC 6143 §7.2.2, as every VNC server does it).
+pub fn vnc_response(password: &str, challenge: &[u8; 16]) -> [u8; 16] {
+    use des::cipher::{BlockEncrypt, KeyInit};
+    let mut key = [0u8; 8];
+    for (k, b) in key.iter_mut().zip(password.bytes()) {
+        *k = b.reverse_bits();
+    }
+    let c = des::Des::new_from_slice(&key).expect("8-byte key");
+    let mut out = *challenge;
+    for block in out.chunks_mut(8) {
+        c.encrypt_block(des::cipher::generic_array::GenericArray::from_mut_slice(block));
+    }
+    out
+}
+
+/// This proxy's first words to a viewer, as a VNC server: version 3.8 (then "no password"; see
+/// Session::feed).
+pub const GREETING: &[u8] = b"RFB 003.008\n";
+
+/// One person's viewer joined to the box.
+pub struct Session {
+    child: Child,
+    stdin: ChildStdin,
+    /// The viewer's side of the handshake still to read: its version (12 bytes) and its choice
+    /// of security type (1 byte).
+    greeting: usize,
+    /// Viewer → box messages not yet complete (to notice clicks and keys).
+    pending: Vec<u8>,
+    sniffing: bool,
+    /// The viewer's ClientInit (one byte) is still to come.
+    init: bool,
+}
+
+/// Open a viewer's way to the box: sign in to its VNC server, and start sending what the server
+/// says to `out` (from a thread of its own; `out` returns false when the viewer is gone, and
+/// gets an empty slice when the box's side ended). The caller then sends the viewer GREETING
+/// (once it is ready for the viewer's answer).
+pub fn open(out: impl Fn(&[u8]) -> bool + Send + 'static) -> Result<Session, String> {
+    crate::cloudbox::ensure_running()?;
+    let mut child = crate::cloudbox::exec_piped(&["python3", "-c", BRIDGE])?;
+    let mut stdin = child.stdin.take().ok_or("連不上這台電腦的畫面")?;
+    let mut stdout = child.stdout.take().ok_or("連不上這台電腦的畫面")?;
+    let fail = |child: &mut Child, why: &str| -> String {
+        let _ = child.kill();
+        let _ = child.wait();
+        format!("連不上這台電腦的畫面：{why}")
+    };
+    let mut read = |n: usize| -> Result<Vec<u8>, String> {
+        let mut b = vec![0u8; n];
+        stdout.read_exact(&mut b).map(|_| b).map_err(|_| "畫面伺服器沒有回應".to_string())
+    };
+    // The box's side, as a VNC client: version 3.8, then its password.
+    let signed_in = (|| -> Result<(), String> {
+        let v = read(12)?;
+        if !v.starts_with(b"RFB 003.") {
+            return Err("不是 VNC 伺服器".into());
+        }
+        stdin.write_all(b"RFB 003.008\n").map_err(|e| e.to_string())?;
+        let n = read(1)?[0] as usize;
+        if n == 0 {
+            return Err("畫面伺服器拒絕連線".into());
+        }
+        let types = read(n)?;
+        if types.contains(&2) {
+            stdin.write_all(&[2]).map_err(|e| e.to_string())?;
+            let c: [u8; 16] = read(16)?.try_into().map_err(|_| "認證失敗".to_string())?;
+            stdin.write_all(&vnc_response(&crate::cloudbox::box_vnc_password(), &c)).map_err(|e| e.to_string())?;
+        } else if types.contains(&1) {
+            stdin.write_all(&[1]).map_err(|e| e.to_string())?;
+        } else {
+            return Err("畫面伺服器要的登入方式不支援".into());
+        }
+        if read(4)? != [0, 0, 0, 0] {
+            return Err("畫面伺服器的密碼不對".into());
+        }
+        Ok(())
+    })();
+    if let Err(e) = signed_in {
+        return Err(fail(&mut child, &e));
+    }
+    OPEN.fetch_add(1, Ordering::SeqCst);
+    crate::cloudbox::set_viewer_open(true);
+    std::thread::spawn(move || {
+        let mut buf = vec![0u8; 64 * 1024];
+        loop {
+            match stdout.read(&mut buf) {
+                Ok(0) | Err(_) => break,
+                Ok(n) => {
+                    if !out(&buf[..n]) {
+                        return;
+                    }
+                }
+            }
+        }
+        // The box's side ended: an empty send says so.
+        out(&[]);
+    });
+    Ok(Session { child, stdin, greeting: 13, pending: Vec::new(), sniffing: true, init: true })
+}
+
+impl Session {
+    /// Bytes from the viewer. False when the box's side is gone.
+    pub fn feed(&mut self, mut b: &[u8], reply: impl Fn(&[u8]) -> bool) -> bool {
+        if self.greeting > 0 {
+            let take = self.greeting.min(b.len());
+            let before = self.greeting;
+            self.greeting -= take;
+            b = &b[take..];
+            // After the version (12 bytes): the one security type, "None".
+            if before > 1 && self.greeting <= 1 && !reply(&[1, 1]) {
+                return false;
+            }
+            // After the choice: OK.
+            if self.greeting == 0 && !reply(&[0, 0, 0, 0]) {
+                return false;
+            }
+            if b.is_empty() {
+                return true;
+            }
+        }
+        self.notice_input(b);
+        self.stdin.write_all(b).and_then(|_| self.stdin.flush()).is_ok()
+    }
+
+    /// Follow the viewer's messages (RFC 6143 §7.5) to see clicks and keys. Anything it doesn't
+    /// know stops the watching, never the forwarding.
+    fn notice_input(&mut self, b: &[u8]) {
+        if !self.sniffing {
+            return;
+        }
+        self.pending.extend_from_slice(b);
+        // The first byte after the handshake is ClientInit (the shared flag).
+        if self.init && !self.pending.is_empty() {
+            self.pending.remove(0);
+            self.init = false;
+        }
+        let mut at = 0;
+        loop {
+            let rest = &self.pending[at..];
+            let Some(&kind) = rest.first() else { break };
+            let len = match kind {
+                0 => 20,
+                2 if rest.len() >= 4 => 4 + 4 * u16::from_be_bytes([rest[2], rest[3]]) as usize,
+                3 => 10,
+                4 => 8,
+                5 => 6,
+                6 if rest.len() >= 8 => 8 + u32::from_be_bytes([rest[4], rest[5], rest[6], rest[7]]) as usize,
+                2 | 6 => break,
+                _ => {
+                    self.sniffing = false;
+                    self.pending.clear();
+                    return;
+                }
+            };
+            if rest.len() < len {
+                break;
+            }
+            match kind {
+                4 if rest[1] == 1 => person_acted(),
+                5 if rest[1] != 0 => person_acted(),
+                _ => {}
+            }
+            at += len;
+        }
+        self.pending.drain(..at);
+    }
+}
+
+impl Drop for Session {
+    fn drop(&mut self) {
+        let _ = self.child.kill();
+        let _ = self.child.wait();
+        if OPEN.fetch_sub(1, Ordering::SeqCst) == 1 {
+            crate::cloudbox::set_viewer_open(false);
+        }
+    }
+}
+
+/// The person clicked or typed: the bot's steps wait until a little after they stop.
+pub fn person_acted() {
+    *LAST_INPUT.lock().unwrap_or_else(|e| e.into_inner()) = Some(Instant::now());
+}
+
+/// The person is using the box through a viewer right now (input in the last few seconds).
+pub fn person_active() -> bool {
+    LAST_INPUT.lock().unwrap_or_else(|e| e.into_inner()).map(|t| t.elapsed() < HELD_AFTER_INPUT).unwrap_or(false)
+}
+
+/// Text the person typed in the viewer's own box (any language): pasted into the box.
+pub fn person_typed(text: &str) {
+    if !text.is_empty() {
+        person_acted();
+        crate::cloudbox::viewer_text(text);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_bridge_keeps_its_indentation() {
+        // Python needs its indentation (a Rust "\" line continuation would strip it).
+        assert!(BRIDGE.contains("\n r=select") && BRIDGE.contains("\n  if not d: break"));
+    }
+
+    #[test]
+    fn vnc_password_answer_matches_the_standard() {
+        // Password "password", challenge 00..0f: DES-ECB with the bit-reversed key 0e86ceceeef64e26
+        // (worked out with `openssl enc -des-ecb -nopad`).
+        let c: [u8; 16] = core::array::from_fn(|i| i as u8);
+        let want = [0xb8, 0x66, 0x92, 0x41, 0x25, 0xc8, 0xee, 0xbb, 0x9d, 0xeb, 0xc1, 0xdb, 0x61, 0xc5, 0x38, 0xe2];
+        assert_eq!(vnc_response("password", &c), want);
+        // Only the first 8 bytes of a password count.
+        assert_eq!(vnc_response("password-and-more", &c), want);
+    }
+}