From 8ec802ca3826be8d60e3d0bb5ab57eb69dd33b4d Mon Sep 17 00:00:00 2001 From: daniel wang Date: Tue, 8 Sep 2026 01:14:53 +0000 Subject: [PATCH] fix job web --- Makefile | 26 ++- apps/web/src/App.tsx | 103 +++++++--- apps/web/src/computer.css | 17 +- apps/web/src/handoff.ts | 57 ++++++ apps/web/src/locales/en.ts | 1 + apps/web/src/locales/zh-TW.ts | 2 +- apps/web/vnc.html | 32 +++- crates/api/src/computer.rs | 13 +- crates/control/src/a11y.rs | 6 +- crates/control/src/cdp.rs | 6 +- crates/control/src/cua/browser.rs | 21 +- crates/control/src/cua/client.rs | 174 +++++++++++++++-- crates/control/src/cua/mod.rs | 115 ++++++++--- crates/control/src/cua/native.rs | 44 ++++- crates/control/src/cua/translate.rs | 102 +++++++--- crates/control/src/lease.rs | 10 - crates/control/src/legacy.rs | 14 +- crates/control/src/observe.rs | 41 ++++ crates/control/src/x11.rs | 21 -- crates/sandbox/src/docker.rs | 43 ++++- docs/cua-compatibility.md | 67 ++++++- docs/cua-schemas/0.23.2/schema-drag.txt | 147 ++++++++++++++ .../cua-schemas/0.23.2/schema-end_session.txt | 17 ++ .../0.23.2/schema-start_session.txt | 50 +++++ docs/development.md | 24 +++ image/computer/Dockerfile | 60 +++--- image/computer/fluxbox/menu | 4 - image/computer/fluxbox/style | 92 --------- image/supervisor/Dockerfile | 4 +- scripts/build-computer-image.sh | 6 - scripts/build-image.sh | 136 +++++++++++++ scripts/cua-smoke-test.sh | 12 ++ tests/frontend.test.mjs | 179 +++++++++++++++++- 33 files changed, 1323 insertions(+), 323 deletions(-) create mode 100644 apps/web/src/handoff.ts create mode 100644 docs/cua-schemas/0.23.2/schema-drag.txt create mode 100644 docs/cua-schemas/0.23.2/schema-end_session.txt create mode 100644 docs/cua-schemas/0.23.2/schema-start_session.txt delete mode 100644 image/computer/fluxbox/menu delete mode 100644 image/computer/fluxbox/style delete mode 100755 scripts/build-computer-image.sh create mode 100755 scripts/build-image.sh diff --git a/Makefile b/Makefile index dd19507..43ec085 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,14 @@ COMPOSE ?= docker compose COMPUTER_IMAGE ?= lazyboy/computer:local WEB_DIR ?= apps/web DATA_DIR ?= ./data +BUILDX_BUILDER ?= lazyboy +# PUSH=1 publishes the manifest list instead of writing an OCI archive. +MULTI_FLAGS ?= +$(if $(filter 1,$(PUSH)),$(eval MULTI_FLAGS := --push)) .PHONY: help env env-force \ up logs ps health down purge \ - computer postgres postgres-down pg-collation \ + computer computer-multi images-multi postgres postgres-down pg-collation \ cua-smoke \ build build-api build-supervisor build-controld \ fmt fmt-check clippy lint audit test clean \ @@ -32,6 +36,8 @@ help: ## Show this help @echo "" @echo " Individual pieces:" @echo " make computer Build the heavy Debian desktop image (lazyboy/computer:local)" + @echo " make computer-multi Cross-build the desktop image for amd64 + arm64 (PUSH=1 to publish)" + @echo " make images-multi Cross-build desktop + api + supervisor for amd64 + arm64" @echo " make cua-smoke Run Cua Driver smoke test in a disposable desktop container" @echo " make postgres Start only postgres (127.0.0.1:5434) and wait for ready" @echo " make postgres-down Stop postgres" @@ -93,7 +99,23 @@ purge: ## Stop containers and delete the postgres data volume # --- Individual pieces ----------------------------------------------------- computer: ## Build the Debian desktop image used to spawn bot computers - docker build -f image/computer/Dockerfile -t $(COMPUTER_IMAGE) . + ./scripts/build-image.sh --tag $(COMPUTER_IMAGE) + +# Cross-builds every supported CPU architecture into one manifest list. Needs a +# docker-container builder + QEMU binfmt; both are bootstrapped by the script. +# PUSH=1 publishes to a registry, otherwise an OCI archive is written. +computer-multi: ## Cross-build the desktop image for all CPU architectures + ./scripts/build-image.sh --file image/computer/Dockerfile --multi $(MULTI_FLAGS) + +# The api and supervisor images carry per-architecture binaries as well (ONNX +# Runtime, node, the distroless libc), so they get the same treatment as the +# desktop image instead of only ever existing for the build host. +images-multi: ## Cross-build every shipped image for all CPU architectures + @for dockerfile in image/computer/Dockerfile image/supervisor/Dockerfile \ + image/api/Dockerfile; do \ + echo "== $$dockerfile"; \ + ./scripts/build-image.sh --file "$$dockerfile" --multi $(MULTI_FLAGS) || exit 1; \ + done cua-smoke: computer ## Run the Cua Driver smoke test inside a disposable desktop container ./scripts/cua-smoke-test.sh --docker --image $(COMPUTER_IMAGE) diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index 3447b9d..e53754b 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -16,6 +16,7 @@ import visibility2 from "react-useanimations/lib/visibility2"; import searchToX from "react-useanimations/lib/searchToX"; import { api, ApiError } from "./api"; import { createCoalescer, subscribeToSession } from "./live"; +import { HANDOFF_MS, VEIL_FADE_MS, handoffRemaining, keepScreenUrl, nextVeil, viewOnlyFor, viewerPath, type Veil } from "./handoff"; import { Avatar, AvatarLookProvider, AvatarStack, BLOBATAR_BACKGROUNDS, BLOBATAR_EXPRESSIONS, BLOBATAR_SHAPES, DEFAULT_LOOK, persistBlobatarShape, readAvatarLooks, resolveBlobatarShape, writeAvatarLook, type AvatarBackground, type AvatarExpression, type AvatarLook } from "./avatar"; import { dateLocale, getLocale, listJoin, setLocale, t, useLocale, type MessageKey } from "./i18n"; import type { AvatarShape, Bot, ComputerMode, ComputerStatus, FileSkill, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, MessageFile, ModelProviderId, Playbook, PlaybookInput, PlaybookStep, Room, RoomMember, Session, TaughtSkill, VoiceSettings, WorkspaceSettings } from "./types"; @@ -86,6 +87,7 @@ function localizeError(message:string){ if(message==="示範進行中:先按「完成示範」或「取消」,再送訊息。")return t("teachInProgress"); if(message==="AI 回應逾時(150 秒)")return t("aiTimeout"); if(message==="run is not retryable")return t("errorRetryFailed"); + if(message==="Stop the task first")return t("takeoverBusy"); return message; } function hudLabel(computer:ComputerStatus,connecting:boolean,handingOff:boolean){ @@ -160,6 +162,11 @@ export function App(){ const desktopFrameRef=useRef(null); const holderRef=useRef(computer.controlHolder); const paneBotRef=useRef(null); const skipHandoffRef=useRef(true); const [desktopReady,setDesktopReady]=useState(false); const [handingOff,setHandingOff]=useState(false); + // The holder the server has not agreed to yet, when the current handoff + // started, and a guard so a fast double click cannot fight itself over the + // mouse. All three exist so the click, not the round trip, owns the answer. + const expectedHolderRef=useRef(null); const handoffAtRef=useRef(0); const controlBusyRef=useRef(false); const [controlBusy,setControlBusy]=useState(false); + const [veil,setVeil]=useState({label:null,leaving:false}); const [pendingFiles,setPendingFiles]=useState([]); const messageEndRef=useRef(null); const sentHistoryRef=useRef([]); const historyIndexRef=useRef(null); const historyDraftRef=useRef(""); @@ -213,7 +220,18 @@ export function App(){ const status=await api(`/api/computer/${computerBot}/status`);if(refreshSeq!==refreshSeqRef.current)return; const [nextMessages,screen,nextSkills]=await Promise.all([messagesJob,status.state==="running"?api<{url:string|null}>(`/api/computer/${computerBot}/screen`).catch(()=>({url:null})):Promise.resolve({url:null}),skillsJob]); if(refreshSeq!==refreshSeqRef.current)return; - setBusyMembers(nextBusy);setComputer(status);setMessages(nextMessages);setSkills(nextSkills);setScreenUrl(status.botId===computerBot?screen.url:null) + setBusyMembers(nextBusy);setComputer(status);setMessages(nextMessages);setSkills(nextSkills); + // A missing url is not a dead desktop. While it boots or wakes the mounted + // frame keeps its VNC session instead of blacking out and reconnecting. + setScreenUrl(current=>status.botId===computerBot?keepScreenUrl(current,screen.url,status.state):null); + // Agreement with the server is what ends a handoff. The beat has a floor so + // the swap is still visible on a LAN, and the ceiling timer still covers a + // reply that never lands. + if(expectedHolderRef.current&&status.controlHolder===expectedHolderRef.current){ + expectedHolderRef.current=null; + const wait=handoffRemaining(handoffAtRef.current,Date.now()); + if(wait>0)window.setTimeout(()=>setHandingOff(false),wait);else setHandingOff(false); + } },[activeId,activeRoomId,activeSessionId]); useEffect(()=>{loadBots().catch(e=>{if(e instanceof ApiError&&e.status===401)setAuthRequired(true);else setError(localizeError(e.message))})},[loadBots]); useEffect(()=>{if(!voiceSettings?.enabled)setCallOpen(false)},[voiceSettings?.enabled]); @@ -245,9 +263,9 @@ export function App(){ useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard?.writeText(text).then(()=>setClipboardStatus(t("clipboardSynced"))).catch(()=>setClipboardStatus(t("clipboardSyncBlocked")))}if(event.data.type==="lazyboy-copy-request"&&computer.controlHolder==="user")void copySelection();if(event.data.type==="lazyboy-paste-text"&&typeof event.data.text==="string")pasteText(event.data.text);if(event.data.type==="lazyboy-mobile-key"&&typeof event.data.key==="string"&&/^(?:(?:ctrl|alt)\+)?(?:Return|BackSpace|Tab|Escape|Left|Right|Up|Down|[acvz])$/.test(event.data.key))queueDesktopInput({kind:"key",key:event.data.key});if(event.data.type==="lazyboy-paste-request"&&computer.controlHolder==="user")setClipboardOpen(true);if(event.data.type==="lazyboy-desktop-ready")setDesktopReady(true);if(event.data.type==="lazyboy-desktop-lost")setDesktopReady(false)};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)}); useEffect(()=>{setDesktopReady(false)},[screenUrl,paneBotId]); useEffect(()=>{if(skipHandoffRef.current){skipHandoffRef.current=false;holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId;return}if((holderRef.current!==computer.controlHolder||paneBotRef.current!==paneBotId)&&computer.state==="running")setHandingOff(true);holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId},[computer.controlHolder,paneBotId,computer.state]); - useEffect(()=>{if(!handingOff)return;const timer=setTimeout(()=>setHandingOff(false),1600);return()=>clearTimeout(timer)},[handingOff]); - useEffect(()=>{const frame=desktopFrameRef.current;if(!frame?.contentWindow||!screenUrl)return;frame.contentWindow.postMessage({type:"lazyboy-view-only",viewOnly:computer.controlHolder!=="user"},location.origin)},[computer.controlHolder,screenUrl,desktopReady]); - useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||event.data?.type!=="lazyboy-request-control"||!paneBotId)return;void action(()=>api(`/api/computer/${paneBotId}/takeover`,{method:"POST",body:"{}"}))};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)},[paneBotId]); + useEffect(()=>{if(!handingOff)return;handoffAtRef.current=Date.now();const timer=setTimeout(()=>{expectedHolderRef.current=null;setHandingOff(false)},HANDOFF_MS);return()=>clearTimeout(timer)},[handingOff]); + useEffect(()=>{const frame=desktopFrameRef.current;if(!frame?.contentWindow||!screenUrl)return;frame.contentWindow.postMessage({type:"lazyboy-view-only",viewOnly:viewOnlyFor(computer.controlHolder)},location.origin)},[computer.controlHolder,screenUrl,desktopReady]); + useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||event.data?.type!=="lazyboy-request-control"||!paneBotId)return;void setControl("user")};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)},[paneBotId]); useEffect(()=>{const close=(event:MouseEvent)=>{const target=event.target;if(target instanceof Element&&target.closest(".create-menu-wrap,.account-wrap,.session-picker,.context-menu,.plus-menu-wrap"))return;setContext(null);setRoomContext(null);setSessionMenuOpen(false);setAccountOpen(false);setCreateMenuOpen(false);setPlusOpen(false)};window.addEventListener("click",close);return()=>window.removeEventListener("click",close)},[]); useEffect(()=>{const onKey=(event:KeyboardEvent)=>{if(event.key!=="Escape")return;setAccountOpen(false);setAccountDialog(null);setCreateMenuOpen(false);setPlusOpen(false);setTeachOpen(false);setEditingSkillId(null);setSessionMenuOpen(false);setContext(null);setRoomContext(null)};window.addEventListener("keydown",onKey);return()=>window.removeEventListener("keydown",onKey)},[]); useEffect(()=>{setWorkspaceName(name=>isDefaultWorkspaceName(name)?t("localWorkspace"):name)},[locale]); @@ -260,10 +278,10 @@ export function App(){ async function openLoginScreen(){ const id=paneBot?.id||active?.id;if(!id)return; setRightPart("computer");setRightCollapsed(false);setComputerOpen(true); - await action(async()=>{ - try{await api(`/api/computer/${id}/boot`,{method:"POST",body:"{}"})}catch{/* already up */} - await api(`/api/computer/${id}/takeover`,{method:"POST",body:"{}"}).catch(()=>{}); - }); + await action(async()=>{try{await api(`/api/computer/${id}/boot`,{method:"POST",body:"{}"})}catch{/* already up */}}); + // The mouse is the point of the login screen, so it comes over on the + // optimistic handoff rather than a second round trip behind a spinner. + await setControl("user",id); } async function reloadSchedules(){if(!paneBotId)return;setSchedules(await api(`/api/bots/${paneBotId}/schedules`))} async function saveScheduleDraft(){ @@ -336,18 +354,52 @@ export function App(){ async function logout(){setAccountOpen(false);await api("/api/session",{method:"DELETE",body:"{}"}).catch(()=>{});setBots([]);setRooms([]);setMcpServers([]);setActiveId(null);setActiveRoomId(null);setAuthRequired(true)} async function changeComputer(operation:"boot"|"restart"|"stop"){ const botId=paneBotId;if(!botId)return; - const previous=computer;setScreenUrl(null);setDesktopReady(false); + const previous=computer; + setDesktopReady(false); + // Stopping drops the pixels; starting keeps them. The viewer url never + // changes, so the frame mounts straight away and noVNC dials in the moment + // X answers, instead of booting only after the reply has already arrived. + if(operation==="stop")setScreenUrl(null);else setScreenUrl(current=>current||viewerPath(botId)); setComputer(current=>({...current,state:operation==="stop"?current.state:operation==="boot"&¤t.state==="suspended"?"suspended":"booting"})); - try{const status=await api(`/api/computer/${botId}/${operation}`,{method:"POST",body:"{}",signal:AbortSignal.timeout(120_000)});if(currentPaneRef.current===botId)setComputer(status)} + try{ + const status=await api(`/api/computer/${botId}/${operation}`,{method:"POST",body:"{}",signal:AbortSignal.timeout(120_000)}); + if(currentPaneRef.current!==botId)return; + setComputer(status); + // Anything but running and the warm frame is a lie: drop it so the panel + // shows what is really there instead of a desktop that will never return. + if(status.state!=="running")setScreenUrl(null); + } catch(error){ const status=await api(`/api/computer/${botId}/status`,{signal:AbortSignal.timeout(5_000)}).catch(()=>previous); - if(currentPaneRef.current===botId)setComputer(status); + if(currentPaneRef.current===botId){setComputer(status);if(status.state!=="running")setScreenUrl(null)} throw error; } } const startBoot=()=>changeComputer("boot"); const stopComputer=()=>changeComputer("stop"); const restartComputer=()=>changeComputer("restart"); + /** Tell the viewer, this instant, whether human input counts. The lease is + * advisory inside the container, so this is the gate that actually moves the + * mouse, and nobody should wait on a round trip to be able to click. */ + function pushViewOnly(viewOnly:boolean){desktopFrameRef.current?.contentWindow?.postMessage({type:"lazyboy-view-only",viewOnly},location.origin)} + /** Take or release the screen. The button, the badge, the veil and the mouse + * all move on the local copy; the server only has to agree with what is + * already on screen, and if it refuses the read-back puts the mouse back. */ + async function setControl(holder:ComputerStatus["controlHolder"],botId:string|null=paneBotId){ + if(!botId||controlBusyRef.current)return; + controlBusyRef.current=true;setControlBusy(true); + expectedHolderRef.current=holder; + setComputer(current=>({...current,controlHolder:holder,takeoverRequested:holder==="user"?false:current.takeoverRequested})); + setHandingOff(true); + pushViewOnly(viewOnlyFor(holder)); + try{await api(`/api/computer/${botId}/${holder==="user"?"takeover":"release"}`,{method:"POST",body:"{}"})} + catch(error){setError(localizeError(error instanceof Error?error.message:t("operationFailed")))} + finally{ + // Read the truth back once: a refused takeover has to give the mouse up + // again, and the status answer is who holds it now. + await refresh().catch(()=>{});expectedHolderRef.current=null;controlBusyRef.current=false;setControlBusy(false); + } + } useEffect(() => { const viewport = window.visualViewport; const update = () => { @@ -360,8 +412,13 @@ export function App(){ }, []); const connecting=computer.state==="running"&&Boolean(screenUrl)&&!desktopReady; const overlayLabel=hudLabel(computer,connecting,handingOff); + // The veil fades in with its label and fades back out through the same + // mascot, so a fast handoff reads as a finished gesture rather than a frame + // that was dropped. Status blips never get to strobe it. + useEffect(()=>{setVeil(current=>nextVeil(overlayLabel,current))},[overlayLabel]); + useEffect(()=>{if(!veil.leaving)return;const timer=setTimeout(()=>setVeil({label:null,leaving:false}),VEIL_FADE_MS);return()=>clearTimeout(timer)},[veil.leaving]); const frame=screenUrl?