Watching and using the bots' computer from afar: pictures as the screen changes, input through a lasting shell

- One lasting docker exec takes the desktop as a JPEG inside the box and sends a picture only
  when it changed (quicker while it changes, slower while still). Viewers ask for the picture
  after the one they have and get it at once. It was one exec per picture (a 1 MB PNG decoded
  and encoded again on the brain) and a fixed 0.7 s / 1 s pause: about one picture a second.
  Measured on the Mac's box: first picture 45 ms, ~90 KB (was ~1.1 MB), 9–19 pictures a second
  while it changes; viewers take at most ~8 a second over the network.
- The loop in the box ends the one before it, ends itself when a write fails ("0" lines while
  still) and is ended when nobody watched for 10 s (ending docker exec on this side left it
  running in the box).
- The person's clicks and keys go through one lasting shell in the box (text as base64), not a
  docker exec each.
- Answers over the lasting connection keep their headers (the relay dropped them): a picture's
  number reaches the app. JPEG/PNG/PDF/zip bodies are no longer deflated again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

mobile/index.html

@@ -740,24 +740,35 @@ function ownComputer() { const n = botName(S.open); return n ? n + " 的電腦"
 $("b-screen").onclick = $("b-screen2").onclick = () => { S.back = S.view; show("screen"); $("seg").children[0].textContent = ownComputer(); $("seg").children[1].style.display = S.mac ? "" : "none"; startFrames(); };
 $("b-back2").onclick = () => show(S.back === "chat" ? "chat" : "list");
 document.querySelectorAll("#seg button").forEach((b) => b.onclick = () => { document.querySelectorAll("#seg button").forEach((x) => x.classList.toggle("on", x === b)); S.screen = b.dataset.w; $("k-wake").style.display = S.screen === "box" ? "" : "none"; startFrames(); });
-let busyFrame = false, lastUrl = null;
+// The bots' computer sends a picture the moment its screen changes: ask for the one after the
-function startFrames() { stopFrames(); frame(); S.frameTimer = setInterval(frame, 1000); }
+// picture shown (X-Seq), again as soon as it arrives. (It was one picture a second, so a tap
-function stopFrames() { clearInterval(S.frameTimer); S.frameTimer = null; }
+// showed its result a second or two later.) This computer's own screen: a picture at a time.
-async function frame() {
+let lastUrl = null;
-  if (busyFrame || document.hidden) return; busyFrame = true;
+const nap = (ms) => new Promise((r) => setTimeout(r, ms));
+function startFrames() { stopFrames(); const run = S.frameRun = {}; frames(run); }
+function stopFrames() { S.frameRun = null; }
+async function frames(run) {
+  let seq = 0;
+  while (S.frameRun === run) {
+    if (document.hidden) { await nap(800); continue; }
     try {
-    const r = await api(base() + "api/screen/" + S.screen, {raw: true});
+      const live = S.screen === "box";
-    if (!r.ok) { const v = await r.json().catch(() => ({})); $("stage").innerHTML = `<div class="msg">${esc(v.error || "看不到畫面")}</div>`; $("screen-live").textContent = ""; }
+      const r = await api(base() + "api/screen/" + S.screen + (live ? "?after=" + seq : ""), {raw: true});
-    else {
+      if (S.frameRun !== run) return;
-      const sz = (r.headers.get("X-Size") || "0x0").split("x").map(Number); S.size = sz;
+      if (!r.ok) { const v = await r.json().catch(() => ({})); $("stage").innerHTML = `<div class="msg">${esc(v.error || "看不到畫面")}</div>`; $("screen-live").textContent = ""; await nap(1500); continue; }
+      const n = +(r.headers.get("X-Seq") || 0);
+      if (live && n && n === seq) continue;
+      seq = n;
+      S.size = (r.headers.get("X-Size") || "0x0").split("x").map(Number);
       const url = URL.createObjectURL(await r.blob());
       let img = $("stage").querySelector("img");
       if (!img) { $("stage").innerHTML = ""; img = document.createElement("img"); img.draggable = false; $("stage").appendChild(img); bindStage(img); }
       img.src = url; if (lastUrl) URL.revokeObjectURL(lastUrl); lastUrl = url;
       $("screen-live").innerHTML = `<span class="live"></span>即時`;
+      // At most about 8 pictures a second (an animated page would send all it can).
+      await nap(!live || !n ? 700 : 120);
+    } catch (e) { $("screen-live").textContent = ""; await nap(1500); }
   }
-  } catch (e) { $("screen-live").textContent = ""; }
-  busyFrame = false;
 }
 function point(img, e) {
   const r = img.getBoundingClientRect();
@@ -777,7 +788,7 @@ function act(img, e, button) {
   input2({kind: "click", x: p.x, y: p.y, button});
 }
 async function input2(body) {
-  try { await api(base() + "api/screen/" + S.screen, {json: body}); setTimeout(frame, 250); } catch (e) { toast(e.message); }
+  try { await api(base() + "api/screen/" + S.screen, {json: body}); } catch (e) { toast(e.message); }
 }
 document.querySelectorAll("[data-key]").forEach((b) => b.onclick = () => input2({kind: "key", key: b.dataset.key}));
 document.querySelectorAll("[data-scroll]").forEach((b) => b.onclick = () => input2({kind: "scroll", x: Math.round(S.size[0] / 2), y: Math.round(S.size[1] / 2), direction: b.dataset.scroll, amount: 5}));
@@ -861,7 +872,7 @@ if ("serviceWorker" in navigator) navigator.serviceWorker.addEventListener("mess
   if (d.host && d.host !== S.host && S.relay) { S.host = d.host; localStorage.setItem("anybot.host", S.host); S.version = 0; }
   if (d.bot) openChat(+d.bot);
 });
-document.addEventListener("visibilitychange", () => { if (!document.hidden && S.view === "screen") frame(); });
+document.addEventListener("visibilitychange", () => { if (!document.hidden && S.view === "screen") startFrames(); });
 hello();
 </script>
 </body>

relay/src/main.rs

@@ -469,7 +469,9 @@ impl Relay {
                 let (me, tx, user, method, z, from) = (self.clone(), tx.clone(), h.user.clone(), s("method"), head["z"].as_bool().unwrap_or(false), h.id.clone());
                 std::thread::spawn(move || {
                     let a = me.forward_raw(&to, &method, &path, &query, &body, z, (&user, role), true, &from);
-                    let _ = tx.send(wire::pack(&json!({"t": "ret", "id": id, "status": a.status, "ctype": a.ctype, "z": a.z}), &a.body));
+                    // The answer's headers too (a live picture's number, a screen's size).
+                    let headers: Vec<[&str; 2]> = a.headers.iter().map(|(k, v)| [k.as_str(), v.as_str()]).collect();
+                    let _ = tx.send(wire::pack(&json!({"t": "ret", "id": id, "status": a.status, "ctype": a.ctype, "z": a.z, "headers": headers}), &a.body));
                 });
             }
             "hello" => {

src/app/computers_ui.rs

@@ -7,10 +7,9 @@
 use super::*;
 use crate::cloudbox::{self, Rebuild, Run};
 
-/// How often the viewer asks the box for a new picture (estimated).
+/// While the viewer is open it redraws at least this often (new pictures ask for a redraw
+/// themselves as they arrive).
 const VIEWER_FRAME_MS: u64 = 450;
-/// The brain's computer comes over the account server: a picture about every 0.7 s.
-const REMOTE_FRAME_MS: u64 = 700;
 /// The floating bar stays this long after the bot's last mouse/keyboard action on this Mac.
 const LOCAL_BAR_SECS: u64 = 8;
 
@@ -1044,18 +1043,25 @@ impl App {
             let s_frames = s.clone();
             std::thread::spawn(move || {
                 let s = s_frames;
+                // Ask for the picture after the one shown: the brain answers when the screen
+                // changes (no fixed pause between pictures).
                 let mut seq = 0u64;
                 while !s2.load(Ordering::SeqCst) {
-                    match crate::remote::sync_screen(&s, false).ok().and_then(|b| image::load_from_memory(&b).ok()) {
+                    match crate::remote::sync_live(&s, seq) {
-                        Some(img) => {
+                        Ok((n, jpeg)) if n != seq || n == 0 => {
+                            if let Ok(img) = image::load_from_memory(&jpeg) {
                                 let img = img.to_rgba8();
-                            seq += 1;
                                 let size = [img.width() as usize, img.height() as usize];
-                            *slot.lock().unwrap() = Some((seq, egui::ColorImage::from_rgba_unmultiplied(size, img.as_raw())));
+                                *slot.lock().unwrap() = Some((n.max(seq + 1), egui::ColorImage::from_rgba_unmultiplied(size, img.as_raw())));
                                 c.request_repaint();
-                            std::thread::sleep(std::time::Duration::from_millis(REMOTE_FRAME_MS));
                             }
-                        None => {
+                            // At most about 8 a second over the network (an animated page
+                            // sends all it can); a brain too old to number them: one a second.
+                            std::thread::sleep(std::time::Duration::from_millis(if n == 0 { 1000 } else { 120 }));
+                            seq = n;
+                        }
+                        Ok(_) => {}
+                        Err(_) => {
                             std::thread::sleep(std::time::Duration::from_millis(1500));
                             c.request_repaint();
                         }
@@ -1089,19 +1095,23 @@ impl App {
         std::thread::spawn(move || {
             let mut seq = 0u64;
             while !s2.load(Ordering::SeqCst) {
-                let got = cloudbox::frame();
+                match cloudbox::live_frame(seq, std::time::Duration::from_secs(5)) {
-                if let Some(img) = got {
+                    Some((n, jpeg)) if n != seq => {
-                    seq += 1;
+                        if let Ok(img) = image::load_from_memory(&jpeg) {
+                            let img = img.to_rgba8();
                             let size = [img.width() as usize, img.height() as usize];
-                    let ci = egui::ColorImage::from_rgba_unmultiplied(size, img.as_raw());
+                            *slot.lock().unwrap() = Some((n, egui::ColorImage::from_rgba_unmultiplied(size, img.as_raw())));
-                    *slot.lock().unwrap() = Some((seq, ci));
                             c.request_repaint();
-                    std::thread::sleep(std::time::Duration::from_millis(VIEWER_FRAME_MS));
+                        }
-                } else {
+                        seq = n;
+                    }
+                    Some(_) => {}
+                    None => {
                         std::thread::sleep(std::time::Duration::from_millis(1000));
                         c.request_repaint();
                     }
                 }
+            }
         });
         let (tx, rx) = channel::<Input>();
         self.cu.input_tx = Some(tx);

src/cloudbox.rs

@@ -1188,13 +1188,162 @@ pub fn thumbnail(width: u32) -> Option<image::RgbaImage> {
     image::open(&path).ok().map(|i| i.to_rgba8())
 }
 
-/// A frame for the viewer (no waking: returns None when the box isn't running).
+/// The screen now, for a small picture (no waking: None when the box isn't running): the live
+/// picture when someone is watching, else one taken now.
 pub fn frame() -> Option<image::RgbaImage> {
     if !run_state().is_running() {
         return None;
     }
-    let png = dk_timeout(&["exec", container(), "import", "-window", "root", "-display", DISPLAY, "-quality", "1", "png:-"], 5_000).ok()?;
+    let jpeg = match live_latest() {
-    image::load_from_memory(&png).ok().map(|i| i.to_rgba8())
+        Some(j) => j,
+        None => std::sync::Arc::new(dk_timeout(&["exec", container(), "import", "-window", "root", "-display", DISPLAY, "-quality", "80", "jpeg:-"], 5_000).ok()?),
+    };
+    image::load_from_memory(&jpeg).ok().map(|i| i.to_rgba8())
+}
+
+// ---------- the live picture (people watching) ----------
+// One lasting `docker exec` takes the desktop as a JPEG inside the box (ImageMagick, in C) a few
+// times a second and sends a picture only when it changed; a viewer asks for "a picture newer
+// than mine" and gets it the moment there is one. It used to be one exec per picture (a 1 MB
+// PNG, decoded and encoded again as a JPEG here) and a fixed pause after each: about one
+// picture a second, so a click showed its result a second or two later. It stops when nobody
+// has asked for a while.
+
+/// JPEG quality of the live picture (text stays readable; 60–90 KB for 1280×800).
+const LIVE_QUALITY: u8 = 70;
+/// Pause between looks at the screen inside the box: short while it is changing (a click's
+/// result, scrolling), longer while it stays still.
+const LIVE_GAP_BUSY: &str = "0.04";
+const LIVE_GAP_STILL: &str = "0.12";
+/// Stop taking pictures when no viewer asked for this long.
+const LIVE_IDLE: Duration = Duration::from_secs(10);
+
+struct Live {
+    seq: u64,
+    jpeg: std::sync::Arc<Vec<u8>>,
+    at: Instant,
+    asked: Instant,
+    running: bool,
+}
+
+fn live() -> &'static (Mutex<Live>, std::sync::Condvar) {
+    static L: OnceLock<(Mutex<Live>, std::sync::Condvar)> = OnceLock::new();
+    L.get_or_init(|| (Mutex::new(Live { seq: 0, jpeg: Default::default(), at: Instant::now(), asked: Instant::now(), running: false }), std::sync::Condvar::new()))
+}
+
+/// The latest live picture, when one was taken in the last few seconds.
+fn live_latest() -> Option<std::sync::Arc<Vec<u8>>> {
+    let l = live().0.lock().unwrap_or_else(|e| e.into_inner());
+    (l.running && l.seq > 0 && l.at.elapsed() < Duration::from_secs(3)).then(|| l.jpeg.clone())
+}
+
+/// A picture of the desktop newer than `after` (0: any), waiting up to `wait` for the screen to
+/// change; the same one again when it didn't. (number, JPEG); None when the box isn't running.
+pub fn live_frame(after: u64, wait: Duration) -> Option<(u64, std::sync::Arc<Vec<u8>>)> {
+    if !run_state().is_running() {
+        return None;
+    }
+    let (m, cv) = live();
+    let mut l = m.lock().unwrap_or_else(|e| e.into_inner());
+    l.asked = Instant::now();
+    if !l.running {
+        l.running = true;
+        std::thread::spawn(live_loop);
+    }
+    let until = Instant::now() + wait;
+    while l.seq == 0 || l.seq <= after {
+        let left = until.saturating_duration_since(Instant::now());
+        if left.is_zero() {
+            break;
+        }
+        l = cv.wait_timeout(l, left).map(|r| r.0).unwrap_or_else(|e| e.into_inner().0);
+    }
+    (l.seq > 0).then(|| (l.seq, l.jpeg.clone()))
+}
+
+/// The box's side: a picture whenever the screen changed, as "<bytes>\n<jpeg>", and "0\n" now
+/// and then while it stays still. Ending the docker exec on this side doesn't end the loop in
+/// the box (a still screen never writes, so it never noticed): it ends the one before it when
+/// it starts, and itself when a write fails (the "0" lines make sure one does).
+fn live_script() -> String {
+    format!(
+        "[ -f /tmp/anybot-live.pid ] && kill $(cat /tmp/anybot-live.pid) 2>/dev/null; echo $$ > /tmp/anybot-live.pid; f=/tmp/anybot-live-$$.jpg; trap 'rm -f $f' EXIT; prev=; still=0; \
+         while :; do import -window root -display {DISPLAY} -quality {LIVE_QUALITY} jpeg:$f 2>/dev/null || exit 3; h=$(cksum < $f); \
+         if [ \"$h\" != \"$prev\" ]; then prev=$h; still=0; {{ wc -c < $f && cat $f; }} || exit 0; sleep {LIVE_GAP_BUSY}; \
+         else still=$((still+1)); if [ $still -ge 15 ]; then still=0; echo 0 || exit 0; fi; sleep {LIVE_GAP_STILL}; fi; done"
+    )
+}
+
+/// End the picture loop in the box (see live_script).
+fn live_stop_in_box() {
+    let _ = dk_timeout(&["exec", container(), "sh", "-c", "[ -f /tmp/anybot-live.pid ] && kill $(cat /tmp/anybot-live.pid) 2>/dev/null; rm -f /tmp/anybot-live.pid; true"], 5_000);
+}
+
+fn live_loop() {
+    use std::io::{BufRead, Read};
+    let (m, cv) = live();
+    let stop = |m: &Mutex<Live>| m.lock().unwrap_or_else(|e| e.into_inner()).running = false;
+    let Ok(mut child) = docker().args(["exec", "-u", "bot", container(), "sh", "-c", &live_script()]).stdout(Stdio::piped()).stderr(Stdio::null()).spawn() else {
+        return stop(m);
+    };
+    let Some(out) = child.stdout.take() else {
+        let _ = child.kill();
+        return stop(m);
+    };
+    // Reading blocks while the screen stays the same; the watcher ends it when nobody asks.
+    let done = std::sync::Arc::new(AtomicBool::new(false));
+    let d2 = done.clone();
+    let reader = std::thread::spawn(move || {
+        let mut r = std::io::BufReader::new(out);
+        let mut line = String::new();
+        while !d2.load(Ordering::SeqCst) {
+            line.clear();
+            if r.read_line(&mut line).unwrap_or(0) == 0 {
+                break;
+            }
+            let Ok(n) = line.trim().parse::<usize>() else { break };
+            // "0": still there, nothing new.
+            if n == 0 {
+                continue;
+            }
+            let mut jpeg = vec![0u8; n];
+            if r.read_exact(&mut jpeg).is_err() {
+                break;
+            }
+            if !jpeg.starts_with(&[0xff, 0xd8]) {
+                continue;
+            }
+            let mut l = m.lock().unwrap_or_else(|e| e.into_inner());
+            l.seq += 1;
+            l.jpeg = std::sync::Arc::new(jpeg);
+            l.at = Instant::now();
+            cv.notify_all();
+        }
+        d2.store(true, Ordering::SeqCst);
+    });
+    loop {
+        std::thread::sleep(Duration::from_millis(500));
+        let idle = m.lock().unwrap_or_else(|e| e.into_inner()).asked.elapsed() > LIVE_IDLE;
+        if idle || done.load(Ordering::SeqCst) || !run_state().is_running() {
+            break;
+        }
+    }
+    done.store(true, Ordering::SeqCst);
+    let _ = child.kill();
+    let _ = child.wait();
+    if run_state().is_running() {
+        live_stop_in_box();
+    }
+    let _ = reader.join();
+    let mut l = m.lock().unwrap_or_else(|e| e.into_inner());
+    l.running = false;
+    // Viewers waiting on a picture that won't come get the last one now.
+    cv.notify_all();
+}
+
+/// The desktop's size (for a viewer mapping its taps), when known.
+pub fn screen_size() -> Option<(u32, u32)> {
+    box_size().ok()
 }
 
 /// The desktop's size in pixels (asked once per box).
@@ -1475,11 +1624,53 @@ pub fn computer_run(name: &str, v: &Value) -> Result<(String, Option<String>), S
 }
 
 // ---------- the viewer's own input (when the person takes control) ----------
-// Same scripts, at the desktop's own pixels (the viewer shows it full size), no picture back.
+// Same scripts, at the desktop's own pixels (the viewer shows it full size), no picture back,
+// through one lasting shell in the box (a click or a key no longer starts a docker exec of its
+// own: 50–200 ms each, more with a Docker host elsewhere). Typed text rides along as base64.
+
+fn input_shell() -> &'static Mutex<Option<std::process::Child>> {
+    static S: OnceLock<Mutex<Option<std::process::Child>>> = OnceLock::new();
+    S.get_or_init(|| Mutex::new(None))
+}
+
+/// One line for the lasting shell: the step's script, its text (if any) fed on its stdin.
+pub fn input_line(script: &str, stdin: Option<&str>) -> String {
+    let script = script.replace('\n', " ");
+    match stdin {
+        Some(t) => format!("printf %s {} | base64 -d | {{ {script}; }}\n", base64::engine::general_purpose::STANDARD.encode(t)),
+        None => format!("{{ {script}; }} </dev/null\n"),
+    }
+}
+
+fn send_input(script: &str, stdin: Option<&str>) {
+    use std::io::Write;
+    let line = input_line(script, stdin);
+    let mut sh = input_shell().lock().unwrap_or_else(|e| e.into_inner());
+    for _ in 0..2 {
+        let alive = sh.as_mut().map(|c| matches!(c.try_wait(), Ok(None))).unwrap_or(false);
+        if !alive {
+            *sh = docker()
+                .args(["exec", "-i", "-u", "bot", "-e", "DISPLAY=:1", container(), "sh", "-c", "while IFS= read -r l; do eval \"$l\" >/dev/null 2>&1; done"])
+                .stdin(Stdio::piped())
+                .stdout(Stdio::null())
+                .stderr(Stdio::null())
+                .spawn()
+                .ok();
+        }
+        let Some(c) = sh.as_mut() else { return };
+        if c.stdin.as_mut().map(|i| i.write_all(line.as_bytes()).and_then(|_| i.flush()).is_ok()).unwrap_or(false) {
+            return;
+        }
+        // The box restarted under it: start a new one and try once more.
+        let _ = c.kill();
+        let _ = c.wait();
+        *sh = None;
+    }
+}
 
 fn viewer(name: &str, v: Value) {
     if let Ok((script, stdin, _)) = box_script(name, &v, (1.0, 1.0)) {
-        let _ = box_exec(&script, stdin.as_deref(), false);
+        send_input(&script, stdin.as_deref());
     }
     touch();
 }
@@ -1504,13 +1695,13 @@ pub fn viewer_key(combo: &str) {
     }
     // X key names pass straight through (they are what the viewers send).
     let x = combo_to_x(c).unwrap_or_else(|_| c.to_string());
-    let _ = box_exec(&format!("xdotool key --clearmodifiers {x}"), None, false);
+    send_input(&format!("xdotool key --clearmodifiers {x}"), None);
     touch();
 }
 
 pub fn viewer_text(text: &str) {
     if !text.is_empty() {
-        let _ = box_exec(&type_script(text), Some(text), false);
+        send_input(&type_script(text), Some(text));
     }
     touch();
 }
@@ -1685,6 +1876,62 @@ mod tests {
         assert!(box_script("computer_use_scroll", &serde_json::json!({"direction": "sideways"}), s).is_err());
     }
 
+    #[test]
+    fn viewer_input_is_one_line() {
+        // A click: its script on one line, no stdin.
+        assert_eq!(input_line("xdotool mousemove 1 2 sleep 0.03 click --repeat 1 --delay 80 1", None), "{ xdotool mousemove 1 2 sleep 0.03 click --repeat 1 --delay 80 1; } </dev/null\n");
+        // Typed text rides as base64 (quotes, newlines, Chinese can't break the line).
+        let l = input_line(&type_script("a'b\nc"), Some("a'b\nc"));
+        assert!(l.starts_with("printf %s YSdiCmM= | base64 -d | { xdotool type") && l.ends_with("; }\n") && l.matches('\n').count() == 1, "{l}");
+    }
+
+    /// Uses the real Docker: `cargo test --release -- --ignored box_live_is_quick --nocapture`.
+    /// A viewer gets a picture at once, the next one soon after the screen changes, several a
+    /// second while it keeps changing, and the box stops taking pictures when nobody watches.
+    #[test]
+    #[ignore]
+    fn box_live_is_quick() {
+        ensure_running().expect("box starts");
+        let t = Instant::now();
+        let (seq, jpeg) = live_frame(0, Duration::from_secs(5)).expect("a first picture");
+        println!("first picture: {} ms, {} KB", t.elapsed().as_millis(), jpeg.len() / 1024);
+        assert!(jpeg.starts_with(&[0xff, 0xd8]));
+        // A still screen: the same picture after the wait. (An open web page may animate; then
+        // a new one comes, which is right too.)
+        let t = Instant::now();
+        let (same, _) = live_frame(seq, Duration::from_millis(1500)).unwrap();
+        println!("after {} ms: {}", t.elapsed().as_millis(), if same == seq { "same picture" } else { "a newer one (something moves)" });
+        // Input (through the lasting input shell, already open) moves the pointer: a new
+        // picture soon.
+        viewer_move(600, 300);
+        let (seq, _) = live_frame(seq, Duration::from_secs(5)).unwrap();
+        std::thread::sleep(Duration::from_millis(600));
+        let t = Instant::now();
+        viewer_move(700, 300);
+        let (next, _) = live_frame(seq, Duration::from_secs(5)).unwrap();
+        let react = t.elapsed();
+        println!("input → new picture: {} ms", react.as_millis());
+        assert!(next > seq && react < Duration::from_millis(600), "{react:?}");
+        // A screen that keeps changing: pictures a second.
+        let _ = box_exec("(setsid xterm -geometry 60x20+40+40 -e sh -c 'while :; do date +%s.%N; done' >/dev/null 2>&1 &)", None, false);
+        std::thread::sleep(Duration::from_millis(1200));
+        let (mut n, _) = live_frame(0, Duration::from_secs(2)).unwrap();
+        let (start, t) = (n, Instant::now());
+        while t.elapsed() < Duration::from_secs(3) {
+            n = live_frame(n, Duration::from_secs(2)).unwrap().0;
+        }
+        let fps = (n - start) as f64 / t.elapsed().as_secs_f64();
+        println!("changing screen: {fps:.1} pictures a second");
+        // (pkill straight from docker exec: inside `sh -c` it matched its own shell first.)
+        let _ = dk(&["exec", container(), "pkill", "-f", "date +%s.%N"]);
+        assert!(fps >= 4.0, "{fps}");
+        // Nobody asks: it stops, in the box too.
+        std::thread::sleep(LIVE_IDLE + Duration::from_secs(2));
+        assert!(!live().0.lock().unwrap().running);
+        let left = dk(&["exec", container(), "sh", "-c", "ps -eo args | grep -c '[a]nybot-live' || true"]).unwrap_or_default();
+        assert_eq!(left.trim(), "0", "picture loops left in the box");
+    }
+
     /// Uses the real Docker: `cargo test --release -- --ignored box_screen_is_exact --nocapture`.
     /// The bot's clicks land where its picture says, a phone's tap where the phone's picture says,
     /// the same spot twice doesn't hang, typing (ASCII, Chinese, a key by name) reaches the app,

src/remote.rs

@@ -415,7 +415,7 @@ pub fn handle(req: Req, via_relay: bool) -> Resp {
             hub().push(cmd);
             Resp::json(&json!({"ok": true}))
         }
-        ("GET", ["api", "screen", which]) => screen(which, &s),
+        ("GET", ["api", "screen", which]) => screen(which, &s, req.query.get("after").and_then(|a| a.parse().ok())),
         ("POST", ["api", "screen", which]) => screen_input(which, &req.json(), &s),
         ("POST", ["api", "device", "run"]) => device_run(&req.json(), &s, via_relay, req.header("x-anybot-from")),
         ("GET", ["api", "sync"]) => {
@@ -450,7 +450,7 @@ pub fn handle(req: Req, via_relay: bool) -> Resp {
                     None => Resp::error(404, "還沒有畫面"),
                 }
             } else {
-                screen("box", &s)
+                screen("box", &s, req.query.get("after").and_then(|a| a.parse().ok()))
             }
         }
         ("POST", ["api", "sync", "screen"]) => {
@@ -606,17 +606,23 @@ fn upload(bot: u64, req: &Req) -> Resp {
 }
 
 /// A frame of the bot's computer ("box") or this Mac ("mac") as JPEG.
-fn screen(which: &str, s: &Settings) -> Resp {
+/// How long a viewer's request for a newer picture waits for the screen to change (well under
+/// the account server's forward timeout).
+const LIVE_WAIT_MS: u64 = 8_000;
+
+/// A screen's picture for a viewer. The box: its live picture, as it was taken (no decoding and
+/// encoding again here); with `after` (the number of the picture the viewer has) it answers the
+/// moment the screen changes, or with the same picture after a few seconds. X-Seq is its number,
+/// X-Size the desktop's size (what the viewer's taps are in).
+fn screen(which: &str, s: &Settings, after: Option<u64>) -> Resp {
     match which {
         "box" => {
             crate::cloudbox::touch();
-            match crate::cloudbox::frame() {
+            let wait = Duration::from_millis(if after.is_some() { LIVE_WAIT_MS } else { 3_000 });
-                Some(img) => {
+            match crate::cloudbox::live_frame(after.unwrap_or(0), wait) {
-                    let rgb = image::DynamicImage::ImageRgba8(img).to_rgb8();
+                Some((seq, jpeg)) => {
-                    match crate::computer::encode_jpeg(&rgb) {
+                    let (w, h) = crate::cloudbox::screen_size().unwrap_or_default();
-                        Some(j) => Resp::bytes("image/jpeg", j).with_header("X-Size", &format!("{}x{}", rgb.width(), rgb.height())),
+                    Resp::bytes("image/jpeg", jpeg.to_vec()).with_header("X-Size", &format!("{w}x{h}")).with_header("X-Seq", &seq.to_string())
-                        None => Resp::error(500, "畫面編碼失敗"),
-                    }
                 }
                 None => Resp::error(503, "Bot 的電腦在休息，按「叫醒」"),
             }
@@ -1206,11 +1212,16 @@ fn to_brain(s: &Settings, method: &str, rest: &str, query: &[(&str, String)], bo
 }
 
 fn to_brain_raw(s: &Settings, method: &str, rest: &str, query: &[(&str, String)], body: &[u8], secs: u64) -> Result<(u16, Vec<u8>), String> {
+    to_brain_full(s, method, rest, query, body, secs).map(|(st, b, _)| (st, b))
+}
+
+/// Same, with the answer's headers.
+fn to_brain_full(s: &Settings, method: &str, rest: &str, query: &[(&str, String)], body: &[u8], secs: u64) -> Result<(u16, Vec<u8>, Vec<(String, String)>), String> {
     if s.brain_id.is_empty() {
         return Err("沒有選大腦".into());
     }
     if let Some(r) = crate::tunnel::call(&s.brain_id, method, &format!("/api/sync{rest}"), query, body, secs) {
-        return r.map(|a| (a.status, a.body));
+        return r.map(|a| (a.status, a.body, a.headers));
     }
     let q: Vec<String> = query.iter().map(|(k, v)| format!("{k}={}", url_escape(v))).collect();
     let url = brain_url(s, &format!("{rest}{}{}", if q.is_empty() { "" } else { "?" }, q.join("&")))?;
@@ -1218,8 +1229,9 @@ fn to_brain_raw(s: &Settings, method: &str, rest: &str, query: &[(&str, String)]
     let r = if method == "GET" { agent(secs).get(&url).header("Authorization", &auth).call() } else { agent(secs).post(&url).header("Authorization", &auth).header("Content-Type", "application/octet-stream").send(body) };
     let mut resp = r.map_err(|e| format!("連不到帳號伺服器：{}", why_unreachable(&e)))?;
     let st = resp.status().as_u16();
+    let headers = resp.headers().iter().filter_map(|(k, v)| Some((k.as_str().to_string(), v.to_str().ok()?.to_string()))).collect();
     let bytes = resp.body_mut().with_config().limit(200 * 1024 * 1024).read_to_vec().map_err(|_| "和大腦的連線到一半斷掉了".to_string())?;
-    Ok((st, bytes))
+    Ok((st, bytes, headers))
 }
 
 /// What changed at the brain since `since` (waits up to LONG_POLL_SECS when nothing has).
@@ -1247,6 +1259,17 @@ pub fn sync_screen(s: &Settings, thumb: bool) -> Result<Vec<u8>, String> {
     Ok(bytes)
 }
 
+/// The brain's bots' computer, live: a picture newer than number `after` (or the same one when
+/// the screen didn't change for a few seconds), with its number.
+pub fn sync_live(s: &Settings, after: u64) -> Result<(u64, Vec<u8>), String> {
+    let (st, bytes, headers) = to_brain_full(s, "GET", "/screen", &[("after", after.to_string())], &[], LIVE_WAIT_MS / 1000 + 15)?;
+    if !(200..300).contains(&st) {
+        return Err(why_refused(st, &serde_json::from_slice(&bytes).unwrap_or(Value::Null), "大腦"));
+    }
+    let seq = headers.iter().find(|(k, _)| k.eq_ignore_ascii_case("x-seq")).and_then(|(_, v)| v.parse().ok()).unwrap_or(0);
+    Ok((seq, bytes))
+}
+
 /// A file the brain keeps (an attachment shown in a chat), as bytes.
 pub fn sync_file(s: &Settings, path: &str) -> Result<Vec<u8>, String> {
     let (st, bytes) = to_brain_raw(s, "GET", "/file", &[("path", path.to_string())], &[], 120)?;

src/tunnel.rs

@@ -26,6 +26,7 @@ const CUT: &str = "和帳號伺服器的連線斷了，正在重新連線";
 pub struct Answer {
     pub status: u16,
     pub body: Vec<u8>,
+    pub headers: Vec<(String, String)>,
 }
 
 type Calls = Arc<Mutex<HashMap<String, Sender<Result<Answer, String>>>>>;
@@ -51,7 +52,10 @@ pub fn zip(b: &[u8]) -> Vec<u8> {
 /// The body to send and whether it is compressed: only when that saves a good part of it
 /// (pictures and PDFs are compressed already).
 fn maybe_zip(b: Vec<u8>) -> (Vec<u8>, bool) {
-    if b.len() <= ZIP_OVER {
+    // JPEG, PNG, GIF, WebP, PDF, zip/docx, gzip: compressed already, not worth a try (the live
+    // picture is a JPEG several times a second).
+    const PACKED: [&[u8]; 7] = [b"\xff\xd8\xff", b"\x89PNG", b"GIF8", b"RIFF", b"%PDF", b"PK\x03\x04", b"\x1f\x8b"];
+    if b.len() <= ZIP_OVER || PACKED.iter().any(|m| b.starts_with(m)) {
         return (b, false);
     }
     let z = zip(&b);
@@ -306,7 +310,8 @@ fn on_frame(head: Value, body: Vec<u8>, tx: &Sender<Vec<u8>>, calls: &Calls) {
         "ret" => {
             if let Some(w) = calls.lock().unwrap().remove(head["id"].as_str().unwrap_or("")) {
                 let body = if head["z"].as_bool().unwrap_or(false) { unzip(&body) } else { Ok(body) };
-                let _ = w.send(body.map(|body| Answer { status: head["status"].as_u64().unwrap_or(502) as u16, body }));
+                let headers = head["headers"].as_array().into_iter().flatten().filter_map(|x| Some((x[0].as_str()?.to_string(), x[1].as_str()?.to_string()))).collect();
+                let _ = w.send(body.map(|body| Answer { status: head["status"].as_u64().unwrap_or(502) as u16, body, headers }));
             }
         }
         _ => {}