feat/cua-driver-poc #7
26
Makefile
26
Makefile
|
|
@ -8,10 +8,14 @@ COMPOSE ?= docker compose
|
||||||
COMPUTER_IMAGE ?= lazyboy/computer:local
|
COMPUTER_IMAGE ?= lazyboy/computer:local
|
||||||
WEB_DIR ?= apps/web
|
WEB_DIR ?= apps/web
|
||||||
DATA_DIR ?= ./data
|
DATA_DIR ?= ./data
|
||||||
|
BUILDX_BUILDER ?= lazyboy
|
||||||
|
# PUSH=1 publishes the manifest list instead of writing an OCI archive.
|
||||||
|
MULTI_FLAGS ?=
|
||||||
|
$(if $(filter 1,$(PUSH)),$(eval MULTI_FLAGS := --push))
|
||||||
|
|
||||||
.PHONY: help env env-force \
|
.PHONY: help env env-force \
|
||||||
up logs ps health down purge \
|
up logs ps health down purge \
|
||||||
computer postgres postgres-down pg-collation \
|
computer computer-multi images-multi postgres postgres-down pg-collation \
|
||||||
cua-smoke \
|
cua-smoke \
|
||||||
build build-api build-supervisor build-controld \
|
build build-api build-supervisor build-controld \
|
||||||
fmt fmt-check clippy lint audit test clean \
|
fmt fmt-check clippy lint audit test clean \
|
||||||
|
|
@ -32,6 +36,8 @@ help: ## Show this help
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo " Individual pieces:"
|
@echo " Individual pieces:"
|
||||||
@echo " make computer Build the heavy Debian desktop image (lazyboy/computer:local)"
|
@echo " make computer Build the heavy Debian desktop image (lazyboy/computer:local)"
|
||||||
|
@echo " make computer-multi Cross-build the desktop image for amd64 + arm64 (PUSH=1 to publish)"
|
||||||
|
@echo " make images-multi Cross-build desktop + api + supervisor for amd64 + arm64"
|
||||||
@echo " make cua-smoke Run Cua Driver smoke test in a disposable desktop container"
|
@echo " make cua-smoke Run Cua Driver smoke test in a disposable desktop container"
|
||||||
@echo " make postgres Start only postgres (127.0.0.1:5434) and wait for ready"
|
@echo " make postgres Start only postgres (127.0.0.1:5434) and wait for ready"
|
||||||
@echo " make postgres-down Stop postgres"
|
@echo " make postgres-down Stop postgres"
|
||||||
|
|
@ -93,7 +99,23 @@ purge: ## Stop containers and delete the postgres data volume
|
||||||
# --- Individual pieces -----------------------------------------------------
|
# --- Individual pieces -----------------------------------------------------
|
||||||
|
|
||||||
computer: ## Build the Debian desktop image used to spawn bot computers
|
computer: ## Build the Debian desktop image used to spawn bot computers
|
||||||
docker build -f image/computer/Dockerfile -t $(COMPUTER_IMAGE) .
|
./scripts/build-image.sh --tag $(COMPUTER_IMAGE)
|
||||||
|
|
||||||
|
# Cross-builds every supported CPU architecture into one manifest list. Needs a
|
||||||
|
# docker-container builder + QEMU binfmt; both are bootstrapped by the script.
|
||||||
|
# PUSH=1 publishes to a registry, otherwise an OCI archive is written.
|
||||||
|
computer-multi: ## Cross-build the desktop image for all CPU architectures
|
||||||
|
./scripts/build-image.sh --file image/computer/Dockerfile --multi $(MULTI_FLAGS)
|
||||||
|
|
||||||
|
# The api and supervisor images carry per-architecture binaries as well (ONNX
|
||||||
|
# Runtime, node, the distroless libc), so they get the same treatment as the
|
||||||
|
# desktop image instead of only ever existing for the build host.
|
||||||
|
images-multi: ## Cross-build every shipped image for all CPU architectures
|
||||||
|
@for dockerfile in image/computer/Dockerfile image/supervisor/Dockerfile \
|
||||||
|
image/api/Dockerfile; do \
|
||||||
|
echo "== $$dockerfile"; \
|
||||||
|
./scripts/build-image.sh --file "$$dockerfile" --multi $(MULTI_FLAGS) || exit 1; \
|
||||||
|
done
|
||||||
|
|
||||||
cua-smoke: computer ## Run the Cua Driver smoke test inside a disposable desktop container
|
cua-smoke: computer ## Run the Cua Driver smoke test inside a disposable desktop container
|
||||||
./scripts/cua-smoke-test.sh --docker --image $(COMPUTER_IMAGE)
|
./scripts/cua-smoke-test.sh --docker --image $(COMPUTER_IMAGE)
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -28,9 +28,13 @@
|
||||||
.computer-part .preview .desktop-frame{display:block;aspect-ratio:16 / 10}
|
.computer-part .preview .desktop-frame{display:block;aspect-ratio:16 / 10}
|
||||||
.overlay-screen .overlay-desktop{position:relative;width:min(100%,1440px);height:100%}
|
.overlay-screen .overlay-desktop{position:relative;width:min(100%,1440px);height:100%}
|
||||||
.overlay-screen .overlay-desktop>.desktop-frame,.overlay-screen .overlay-desktop>.empty-computer{width:100%;height:100%;border:1px solid var(--border);border-radius:14px;overflow:hidden;background:var(--main)}
|
.overlay-screen .overlay-desktop>.desktop-frame,.overlay-screen .overlay-desktop>.empty-computer{width:100%;height:100%;border:1px solid var(--border);border-radius:14px;overflow:hidden;background:var(--main)}
|
||||||
.computer-hud{position:absolute;top:10px;left:10px;z-index:3;display:grid;justify-items:center;gap:6px;padding:10px 12px 8px;border-radius:16px;background:rgba(10,10,12,.72);backdrop-filter:blur(8px);pointer-events:none;box-shadow:0 10px 30px rgba(0,0,0,.35)}
|
/* The waiting veil sits over the remote pixels, not beside them: while the
|
||||||
.computer-hud .avatar.blobatar.thinking{animation:computer-hud-bob 1.6s ease-in-out infinite,avatar-rainbow-glow 1.8s linear infinite}
|
desktop is booting, waking, or changing hands, the signal mascot breathes
|
||||||
.computer-hud-label{font-size:12px;letter-spacing:.02em;line-height:1.3;white-space:nowrap;background:linear-gradient(90deg,#7a8088 0%,#7a8088 28%,#fff 50%,#7a8088 72%,#7a8088 100%);background-size:220% 100%;-webkit-background-clip:text;background-clip:text;color:transparent;animation:working-shimmer 1.35s linear infinite}
|
under its amber orbit and the label shimmers. It never takes a click, so the
|
||||||
|
mouse is live the instant the veil lifts, and it fades rather than blinks. */
|
||||||
|
.computer-hud{position:absolute;inset:0;z-index:3;display:grid;align-content:center;justify-items:center;gap:14px;padding:10px 12px 8px;border-radius:inherit;background:radial-gradient(ellipse at center,#142722e8,#101012ed);backdrop-filter:blur(8px);pointer-events:none;opacity:1;transition:opacity .22s ease}
|
||||||
|
.computer-hud.is-leaving{opacity:0}
|
||||||
|
.computer-hud-label{font-size:12px;letter-spacing:.02em;line-height:1.5;white-space:normal;text-align:center;max-width:90%;color:#c9ddd5}
|
||||||
.empty-computer.is-waiting{min-height:100%;background:var(--inset)}
|
.empty-computer.is-waiting{min-height:100%;background:var(--inset)}
|
||||||
.computer-actions{display:flex;align-items:center;justify-content:flex-end;gap:8px;min-width:0;flex-wrap:wrap}
|
.computer-actions{display:flex;align-items:center;justify-content:flex-end;gap:8px;min-width:0;flex-wrap:wrap}
|
||||||
.computer-actions .restart-computer{font-size:12px;padding-inline:10px;color:var(--muted)}
|
.computer-actions .restart-computer{font-size:12px;padding-inline:10px;color:var(--muted)}
|
||||||
|
|
@ -47,12 +51,10 @@
|
||||||
.control-bar{flex-wrap:wrap}
|
.control-bar{flex-wrap:wrap}
|
||||||
.control-bar>.computer-actions{flex:1 1 180px}
|
.control-bar>.computer-actions{flex:1 1 180px}
|
||||||
/* Independent monitor mascot, with mint halo and amber orbit. */
|
/* Independent monitor mascot, with mint halo and amber orbit. */
|
||||||
.computer-hud{inset:0;align-content:center;border-radius:inherit;gap:14px;background:radial-gradient(ellipse at center,#142722e8,#101012ed);box-shadow:none}
|
|
||||||
.computer-signal{position:relative;display:grid;place-items:center;width:64px;height:64px;border:1px solid #65dcb34d;border-radius:24px;animation:monitor-breathe 2.8s ease-in-out infinite}
|
.computer-signal{position:relative;display:grid;place-items:center;width:64px;height:64px;border:1px solid #65dcb34d;border-radius:24px;animation:monitor-breathe 2.8s ease-in-out infinite}
|
||||||
.computer-signal::before{content:"";position:absolute;inset:-8px;border:1px solid #65dcb326;border-top-color:#efbe72;border-radius:50%;animation:monitor-orbit 4s linear infinite}
|
.computer-signal::before{content:"";position:absolute;inset:-8px;border:1px solid #65dcb326;border-top-color:#efbe72;border-radius:50%;animation:monitor-orbit 4s linear infinite}
|
||||||
.computer-signal-face{display:flex;align-items:center;justify-content:center;gap:12px;width:46px;height:38px;border-radius:15px;background:#85dfbb;color:#143429;transform:rotate(-6deg)}
|
.computer-signal-face{display:flex;align-items:center;justify-content:center;gap:12px;width:46px;height:38px;border-radius:15px;background:#85dfbb;color:#143429;transform:rotate(-6deg)}
|
||||||
.computer-signal-face i{width:5px;height:11px;border-radius:5px;background:currentColor;animation:monitor-blink 4.6s ease-in-out infinite}
|
.computer-signal-face i{width:5px;height:11px;border-radius:5px;background:currentColor;animation:monitor-blink 4.6s ease-in-out infinite}
|
||||||
.computer-hud-label{color:#c9ddd5;background:none;animation:none;white-space:normal;text-align:center;max-width:90%;line-height:1.5}
|
|
||||||
|
|
||||||
.clipboard-status{margin:0;min-height:1.4em;font-size:12px;line-height:1.4;color:var(--muted)}
|
.clipboard-status{margin:0;min-height:1.4em;font-size:12px;line-height:1.4;color:var(--muted)}
|
||||||
|
|
||||||
|
|
@ -69,10 +71,9 @@
|
||||||
.computer-power-menu small{display:block;padding:8px 10px 6px;border-top:1px solid var(--line);color:var(--muted);font-size:11px;line-height:1.6}
|
.computer-power-menu small{display:block;padding:8px 10px 6px;border-top:1px solid var(--line);color:var(--muted);font-size:11px;line-height:1.6}
|
||||||
.computer-overlay .computer-power-menu{top:calc(100% + 8px);bottom:auto}
|
.computer-overlay .computer-power-menu{top:calc(100% + 8px);bottom:auto}
|
||||||
|
|
||||||
/* HUD belongs outside the remote pixels, including during connection/handoff. */
|
/* The overlay tracks the visible viewport so the veil and the controls stay
|
||||||
|
above the keyboard; the veil itself belongs to the desktop box below. */
|
||||||
.computer-overlay{top:var(--visible-top,0px);bottom:auto;height:var(--visible-height,100dvh)}
|
.computer-overlay{top:var(--visible-top,0px);bottom:auto;height:var(--visible-height,100dvh)}
|
||||||
.computer-overlay>header{flex-shrink:0}
|
.computer-overlay>header{flex-shrink:0}
|
||||||
.computer-hud{position:static;inset:auto;display:block;padding:7px 12px;border-radius:0;background:var(--surface);flex:none}
|
|
||||||
.computer-hud .computer-signal{display:none}
|
|
||||||
.overlay-error{position:static;transform:none;flex:none;margin:0 8px 8px}
|
.overlay-error{position:static;transform:none;flex:none;margin:0 8px 8px}
|
||||||
@media(max-width:700px){.overlay-screen{padding:4px}.computer-overlay>header{min-height:44px;gap:4px}.computer-overlay>header>div{gap:5px}.computer-overlay>header strong{font-size:12px}}
|
@media(max-width:700px){.overlay-screen{padding:4px}.computer-overlay>header{min-height:44px;gap:4px}.computer-overlay>header>div{gap:5px}.computer-overlay>header strong{font-size:12px}}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,57 @@
|
||||||
|
// Everything that decides when the screen veil shows, and when the viewer may
|
||||||
|
// be thrown away. Both jobs used to be a fixed timer plus "null the url", which
|
||||||
|
// is exactly what made starting and handing over feel like a stall: the mascot
|
||||||
|
// popped, the desktop went black, and noVNC booted from scratch.
|
||||||
|
|
||||||
|
import type { ComputerState } from "./types";
|
||||||
|
|
||||||
|
/** The viewer url is the proxy path for a bot and never carries a nonce, so the
|
||||||
|
* frame can mount while the desktop is still coming up and let noVNC dial into
|
||||||
|
* it on retry instead of waiting for a status round trip. */
|
||||||
|
export function viewerPath(botId: string): string {
|
||||||
|
return `/view/${encodeURIComponent(botId)}/vnc.html`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Only a computer that is really gone may drop the pixels. While it boots,
|
||||||
|
* wakes, or a poll blinks, the mounted frame keeps its VNC session; remounting
|
||||||
|
* costs a page load, a handshake, and a black flash for no information. */
|
||||||
|
export function keepScreenUrl(current: string | null, next: string | null, state: ComputerState): string | null {
|
||||||
|
if (next) return next;
|
||||||
|
if (state === "stopped" || state === "error") return null;
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A handoff should read as one deliberate beat, not as a wait. This is the
|
||||||
|
* ceiling, used only when the server never confirms; agreement on the new
|
||||||
|
* holder usually releases the veil far sooner. */
|
||||||
|
export const HANDOFF_MS = 900;
|
||||||
|
/** The floor. Fast as the reply may land, the swap is worth one visible beat,
|
||||||
|
* otherwise the mascot strobes instead of gesturing. */
|
||||||
|
export const HANDOFF_MIN_MS = 320;
|
||||||
|
/** Kept in step with the veil's CSS fade so it never vanishes mid-animation. */
|
||||||
|
export const VEIL_FADE_MS = 220;
|
||||||
|
|
||||||
|
/** How much longer the veil has to stay after the server agreed on a holder:
|
||||||
|
* zero once the beat has been served, and never more than the ceiling timer,
|
||||||
|
* which is what a lost reply falls back on. */
|
||||||
|
export function handoffRemaining(startedAt: number, now: number): number {
|
||||||
|
const held = now - startedAt;
|
||||||
|
if (held >= HANDOFF_MS) return 0;
|
||||||
|
return Math.max(HANDOFF_MIN_MS - held, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Veil { label: string | null; leaving: boolean }
|
||||||
|
|
||||||
|
/** Fade in with a label, fade out once the label clears, and stay out when
|
||||||
|
* there was never anything to show. Pure, so a status blip can be tested. */
|
||||||
|
export function nextVeil(label: string | null, current: Veil): Veil {
|
||||||
|
if (label) return current.label === label && !current.leaving ? current : { label, leaving: false };
|
||||||
|
if (!current.label) return current;
|
||||||
|
return current.leaving ? current : { label: current.label, leaving: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The lease is advisory inside the container, so the browser gate is what
|
||||||
|
* actually moves the mouse: whoever holds control may send input right now. */
|
||||||
|
export function viewOnlyFor(holder: "none" | "bot" | "user"): boolean {
|
||||||
|
return holder !== "user";
|
||||||
|
}
|
||||||
|
|
@ -253,6 +253,7 @@ export const en: { [K in keyof typeof zhTW]: string } = {
|
||||||
computerPower: "Computer menu",
|
computerPower: "Computer menu",
|
||||||
sharedPowerHint: "This is a shared computer. The action affects every agent using it.",
|
sharedPowerHint: "This is a shared computer. The action affects every agent using it.",
|
||||||
stopAndTakeOver: "Stop and take over",
|
stopAndTakeOver: "Stop and take over",
|
||||||
|
takeoverBusy: "Stop the task first, then take the mouse.",
|
||||||
hudBooting: "Computer starting…",
|
hudBooting: "Computer starting…",
|
||||||
hudWaking: "Waking…",
|
hudWaking: "Waking…",
|
||||||
hudConnecting: "Connecting…",
|
hudConnecting: "Connecting…",
|
||||||
|
|
|
||||||
|
|
@ -64,7 +64,7 @@ export const zhTW = {
|
||||||
mcpNoResults: "找不到符合的 MCP。換個關鍵字,或改用自訂接入。", mcpCustom: "自訂接入", mcpBackToList: "回到列表",
|
mcpNoResults: "找不到符合的 MCP。換個關鍵字,或改用自訂接入。", mcpCustom: "自訂接入", mcpBackToList: "回到列表",
|
||||||
mcpConnectNamed: "接入 {name}", mcpKeyHint: "這個 MCP 需要憑證才能連。",
|
mcpConnectNamed: "接入 {name}", mcpKeyHint: "這個 MCP 需要憑證才能連。",
|
||||||
noMcp: "還沒有 MCP。點上面的「選擇 MCP」從市集接入。", toolsCount: "{count} 個工具", disabled: "已關閉", disconnected: "未連線", noTools: "沒有可用工具", reconnect: "重新連線", disable: "停用", enable: "啟用",
|
noMcp: "還沒有 MCP。點上面的「選擇 MCP」從市集接入。", toolsCount: "{count} 個工具", disabled: "已關閉", disconnected: "未連線", noTools: "沒有可用工具", reconnect: "重新連線", disable: "停用", enable: "啟用",
|
||||||
preparingDesktop: "正在準備 Agent 的獨立桌面…", computerPreviewHint: "開啟電腦後,畫面會顯示在這裡。", bootingProgress: "啟動中…", restartComputer: "重啟", shutDownComputer: "關閉電腦", computerPower: "電腦選單", sharedPowerHint: "此為共用電腦,操作會影響使用它的所有 Agent。", stopAndTakeOver: "停止並接管",
|
preparingDesktop: "正在準備 Agent 的獨立桌面…", computerPreviewHint: "開啟電腦後,畫面會顯示在這裡。", bootingProgress: "啟動中…", restartComputer: "重啟", shutDownComputer: "關閉電腦", computerPower: "電腦選單", sharedPowerHint: "此為共用電腦,操作會影響使用它的所有 Agent。", stopAndTakeOver: "停止並接管", takeoverBusy: "請先停止任務,再接手滑鼠。",
|
||||||
hudBooting: "電腦啟動中…", hudWaking: "喚醒中…", hudConnecting: "連線中…", hudHandoff: "換手中…",
|
hudBooting: "電腦啟動中…", hudWaking: "喚醒中…", hudConnecting: "連線中…", hudHandoff: "換手中…",
|
||||||
pasteToRemoteComputer: "貼到遠端電腦", pasteRemoteHelp: "把外面的文字貼在這裡,再送進 VNC。這個方式在區網 HTTP 也能使用。", pasteTextPlaceholder: "在此貼上文字…", pasteIntoVnc: "貼入 VNC",
|
pasteToRemoteComputer: "貼到遠端電腦", pasteRemoteHelp: "把外面的文字貼在這裡,再送進 VNC。這個方式在區網 HTTP 也能使用。", pasteTextPlaceholder: "在此貼上文字…", pasteIntoVnc: "貼入 VNC",
|
||||||
botNamePlaceholder: "例如:研究助理", sharedComputerHint: "與其他機器人共用環境", privateComputerHint: "全新的獨立 Docker", create: "建立",
|
botNamePlaceholder: "例如:研究助理", sharedComputerHint: "與其他機器人共用環境", privateComputerHint: "全新的獨立 Docker", create: "建立",
|
||||||
|
|
|
||||||
|
|
@ -85,6 +85,20 @@
|
||||||
|
|
||||||
let rfb = null;
|
let rfb = null;
|
||||||
let reconnectTimer = null;
|
let reconnectTimer = null;
|
||||||
|
// A desktop that has never answered on this page is still coming up, and
|
||||||
|
// x11vnc opens its port a heartbeat after the last attempt failed. Holding
|
||||||
|
// every retry at the calm cadence leaves up to half a second of veil on top
|
||||||
|
// of a desktop that is already there, so the first window of attempts comes
|
||||||
|
// back eager. Once a session has been seen the calm cadence returns: a
|
||||||
|
// genuinely dropped connection should not be hammered.
|
||||||
|
const CALM_RETRY_MS = 1500;
|
||||||
|
const EAGER_RETRY_MS = 350;
|
||||||
|
const EAGER_ATTEMPTS = 40;
|
||||||
|
let everConnected = false;
|
||||||
|
let eagerAttempts = 0;
|
||||||
|
const retryDelay = () => (!everConnected && eagerAttempts < EAGER_ATTEMPTS)
|
||||||
|
? (eagerAttempts += 1, EAGER_RETRY_MS)
|
||||||
|
: CALM_RETRY_MS;
|
||||||
const parentOrigin = window.location.origin;
|
const parentOrigin = window.location.origin;
|
||||||
const keyboard = document.getElementById("mobile-keyboard");
|
const keyboard = document.getElementById("mobile-keyboard");
|
||||||
const sentinel = "\u200b";
|
const sentinel = "\u200b";
|
||||||
|
|
@ -347,12 +361,21 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The host flips this gate whenever control changes hands, including
|
||||||
|
// while the frame is still dialling or between retries. Keep the last
|
||||||
|
// thing it asked for and hand it to every new RFB, so a handoff never
|
||||||
|
// depends on which side of a reconnect the message happened to land.
|
||||||
|
let wantedViewOnly = null;
|
||||||
function applyViewOnly(value) {
|
function applyViewOnly(value) {
|
||||||
|
wantedViewOnly = Boolean(value);
|
||||||
if (!rfb) return;
|
if (!rfb) return;
|
||||||
if (value) { releaseDrag(); gesture = null; modifier = null; if (shortcuts) shortcuts.hidden = true; }
|
if (wantedViewOnly) { releaseDrag(); gesture = null; modifier = null; if (shortcuts) shortcuts.hidden = true; }
|
||||||
rfb.viewOnly = Boolean(value);
|
rfb.viewOnly = wantedViewOnly;
|
||||||
updatePointerControls();
|
updatePointerControls();
|
||||||
if (rfb.viewOnly) { keyboard.blur(); resetKeyboard(); }
|
if (rfb.viewOnly) { keyboard.blur(); resetKeyboard(); }
|
||||||
|
// Taking over has to be usable, not just painted: focus so the very
|
||||||
|
// first keystroke after the veil lands on the desktop.
|
||||||
|
else { try { rfb.focus(); } catch (_) {} }
|
||||||
}
|
}
|
||||||
|
|
||||||
function connect() {
|
function connect() {
|
||||||
|
|
@ -368,10 +391,13 @@
|
||||||
pinTaskbar();
|
pinTaskbar();
|
||||||
updatePointerControls();
|
updatePointerControls();
|
||||||
rfb.addEventListener("connect", () => {
|
rfb.addEventListener("connect", () => {
|
||||||
|
everConnected = true;
|
||||||
|
eagerAttempts = 0;
|
||||||
setStatus("");
|
setStatus("");
|
||||||
pinTaskbar();
|
pinTaskbar();
|
||||||
updatePointerControls();
|
updatePointerControls();
|
||||||
try { rfb.focus(); } catch (_) {}
|
try { rfb.focus(); } catch (_) {}
|
||||||
|
if (wantedViewOnly !== null) applyViewOnly(wantedViewOnly);
|
||||||
window.parent.postMessage({ type: "lazyboy-desktop-ready" }, parentOrigin);
|
window.parent.postMessage({ type: "lazyboy-desktop-ready" }, parentOrigin);
|
||||||
});
|
});
|
||||||
rfb.addEventListener("disconnect", (event) => {
|
rfb.addEventListener("disconnect", (event) => {
|
||||||
|
|
@ -382,7 +408,7 @@
|
||||||
const clean = event && event.detail && event.detail.clean;
|
const clean = event && event.detail && event.detail.clean;
|
||||||
setStatus(clean ? "Disconnected — retrying" : "Desktop connection lost — retrying");
|
setStatus(clean ? "Disconnected — retrying" : "Desktop connection lost — retrying");
|
||||||
if (reconnectTimer) clearTimeout(reconnectTimer);
|
if (reconnectTimer) clearTimeout(reconnectTimer);
|
||||||
reconnectTimer = setTimeout(connect, 1500);
|
reconnectTimer = setTimeout(connect, retryDelay());
|
||||||
});
|
});
|
||||||
rfb.addEventListener("clipboard", (event) => {
|
rfb.addEventListener("clipboard", (event) => {
|
||||||
const text = event && event.detail ? event.detail.text : "";
|
const text = event && event.detail ? event.detail.text : "";
|
||||||
|
|
|
||||||
|
|
@ -3,8 +3,8 @@ use std::time::Duration;
|
||||||
|
|
||||||
use chrono::{TimeDelta, Utc};
|
use chrono::{TimeDelta, Utc};
|
||||||
use lazyboy_contracts::{
|
use lazyboy_contracts::{
|
||||||
BrowserProfileMode, ComputerCapabilities, ComputerMode, ComputerState, ComputerStatus,
|
BrowserProfileMode, ComputerCapabilities, ComputerMode, ComputerStatus, ControlHolder,
|
||||||
ControlHolder, DEFAULT_SCREEN_HEIGHT, DEFAULT_SCREEN_WIDTH,
|
DEFAULT_SCREEN_HEIGHT, DEFAULT_SCREEN_WIDTH,
|
||||||
};
|
};
|
||||||
use lazyboy_control::{
|
use lazyboy_control::{
|
||||||
AdapterContext, CommandRequest, EnsureScreenRequest, ProvisionRequest, admit_gui,
|
AdapterContext, CommandRequest, EnsureScreenRequest, ProvisionRequest, admit_gui,
|
||||||
|
|
@ -998,11 +998,6 @@ pub async fn heartbeat(state: &AppState, actor: &Actor, bot_id: &str) -> Result<
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn user_has_control(computer: &ComputerRow, bot_id: &str) -> bool {
|
|
||||||
user_has_screen_control(computer, None, bot_id)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn user_has_screen_control(
|
pub fn user_has_screen_control(
|
||||||
computer: &ComputerRow,
|
computer: &ComputerRow,
|
||||||
screen: Option<&ScreenRow>,
|
screen: Option<&ScreenRow>,
|
||||||
|
|
@ -1232,7 +1227,3 @@ struct ActiveRunRow {
|
||||||
thread_id: String,
|
thread_id: String,
|
||||||
step: Option<String>,
|
step: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn _keep_state(state: ComputerState, holder: ControlHolder, mode: BrowserProfileMode) {
|
|
||||||
let _ = (state, holder, mode);
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
use lazyboy_contracts::UiElement;
|
use lazyboy_contracts::UiElement;
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
use crate::normalize_display;
|
||||||
use crate::x11::{is_browser_title, parse_ui_elements};
|
use crate::x11::{is_browser_title, parse_ui_elements};
|
||||||
use crate::{PRIMARY_DISPLAY, normalize_display};
|
|
||||||
|
|
||||||
const A11Y_PY: &str = include_str!("a11y.py");
|
const A11Y_PY: &str = include_str!("a11y.py");
|
||||||
|
|
||||||
|
|
@ -30,10 +30,6 @@ pub fn a11y_command_on(display: &str, request: &Value) -> Vec<String> {
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn a11y_command(request: &Value) -> Vec<String> {
|
|
||||||
a11y_command_on(PRIMARY_DISPLAY, request)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn parse_a11y_page(raw: &str) -> A11yPage {
|
pub fn parse_a11y_page(raw: &str) -> A11yPage {
|
||||||
let value: Value = serde_json::from_str(raw.trim()).unwrap_or(Value::Null);
|
let value: Value = serde_json::from_str(raw.trim()).unwrap_or(Value::Null);
|
||||||
let ok = value.get("ok").and_then(Value::as_bool) == Some(true);
|
let ok = value.get("ok").and_then(Value::as_bool) == Some(true);
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,8 @@ use lazyboy_contracts::UiElement;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
use crate::normalize_display;
|
||||||
use crate::x11::parse_ui_elements;
|
use crate::x11::parse_ui_elements;
|
||||||
use crate::{PRIMARY_DISPLAY, normalize_display};
|
|
||||||
|
|
||||||
const CDP_PY: &str = include_str!("cdp.py");
|
const CDP_PY: &str = include_str!("cdp.py");
|
||||||
|
|
||||||
|
|
@ -39,10 +39,6 @@ pub fn cdp_command_on(display: &str, profile: Option<&str>, request: &Value) ->
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn cdp_command(request: &Value) -> Vec<String> {
|
|
||||||
cdp_command_on(PRIMARY_DISPLAY, None, request)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Same as `cdp_command_on` but the JSON body is meant to arrive on stdin
|
/// Same as `cdp_command_on` but the JSON body is meant to arrive on stdin
|
||||||
/// so secrets never appear on the process argv.
|
/// so secrets never appear on the process argv.
|
||||||
pub fn cdp_stdin_command_on(display: &str, profile: Option<&str>) -> Vec<String> {
|
pub fn cdp_stdin_command_on(display: &str, profile: Option<&str>) -> Vec<String> {
|
||||||
|
|
|
||||||
|
|
@ -8,8 +8,6 @@ use crate::controller::ControlError;
|
||||||
use crate::process::spawn_detached;
|
use crate::process::spawn_detached;
|
||||||
use crate::{BrowserRequest, CdpPage, launch_argv_on};
|
use crate::{BrowserRequest, CdpPage, launch_argv_on};
|
||||||
|
|
||||||
pub const SESSION: &str = "lazyboy";
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct BrowserBind {
|
pub struct BrowserBind {
|
||||||
pub pid: u64,
|
pub pid: u64,
|
||||||
|
|
@ -253,12 +251,7 @@ async fn attach(
|
||||||
window: &ListedWindow,
|
window: &ListedWindow,
|
||||||
) -> Result<BrowserBind, ControlError> {
|
) -> Result<BrowserBind, ControlError> {
|
||||||
client
|
client
|
||||||
.call(
|
.call(display, "start_session", &json!({}), &[])
|
||||||
display,
|
|
||||||
"start_session",
|
|
||||||
&json!({ "session": SESSION }),
|
|
||||||
&[],
|
|
||||||
)
|
|
||||||
.await?;
|
.await?;
|
||||||
let pid = window.pid;
|
let pid = window.pid;
|
||||||
let window_id = window.id;
|
let window_id = window.id;
|
||||||
|
|
@ -269,7 +262,6 @@ async fn attach(
|
||||||
&json!({
|
&json!({
|
||||||
"pid": pid,
|
"pid": pid,
|
||||||
"window_id": window_id,
|
"window_id": window_id,
|
||||||
"session": SESSION,
|
|
||||||
"strategy": { "kind": "existing_profile" },
|
"strategy": { "kind": "existing_profile" },
|
||||||
"allow_launch": false,
|
"allow_launch": false,
|
||||||
}),
|
}),
|
||||||
|
|
@ -289,7 +281,6 @@ async fn attach(
|
||||||
&json!({
|
&json!({
|
||||||
"pid": pid,
|
"pid": pid,
|
||||||
"window_id": window_id,
|
"window_id": window_id,
|
||||||
"session": SESSION,
|
|
||||||
"include_screenshot": false,
|
"include_screenshot": false,
|
||||||
}),
|
}),
|
||||||
&[],
|
&[],
|
||||||
|
|
@ -317,7 +308,6 @@ pub async fn snapshot(
|
||||||
&json!({
|
&json!({
|
||||||
"target_id": bind.target_id,
|
"target_id": bind.target_id,
|
||||||
"tab_id": bind.tab_id,
|
"tab_id": bind.tab_id,
|
||||||
"session": SESSION,
|
|
||||||
"snapshot_format": "semantic_v2",
|
"snapshot_format": "semantic_v2",
|
||||||
"include_screenshot": false,
|
"include_screenshot": false,
|
||||||
}),
|
}),
|
||||||
|
|
@ -376,7 +366,6 @@ pub async fn run(
|
||||||
&json!({
|
&json!({
|
||||||
"target_id": attached.target_id,
|
"target_id": attached.target_id,
|
||||||
"tab_id": attached.tab_id,
|
"tab_id": attached.tab_id,
|
||||||
"session": SESSION,
|
|
||||||
"url": url,
|
"url": url,
|
||||||
}),
|
}),
|
||||||
&[],
|
&[],
|
||||||
|
|
@ -397,7 +386,6 @@ pub async fn run(
|
||||||
"key": key,
|
"key": key,
|
||||||
"pid": attached.pid,
|
"pid": attached.pid,
|
||||||
"window_id": attached.window_id,
|
"window_id": attached.window_id,
|
||||||
"session": SESSION,
|
|
||||||
}),
|
}),
|
||||||
&[],
|
&[],
|
||||||
)
|
)
|
||||||
|
|
@ -422,7 +410,7 @@ async fn click(
|
||||||
.as_deref()
|
.as_deref()
|
||||||
.ok_or_else(|| ControlError::InvalidAction("browser click needs a selector".into()))?;
|
.ok_or_else(|| ControlError::InvalidAction("browser click needs a selector".into()))?;
|
||||||
let page = bind.page.as_ref().ok_or(ControlError::StaleReference)?;
|
let page = bind.page.as_ref().ok_or(ControlError::StaleReference)?;
|
||||||
let Some(r#ref) = find_ref(&page, selector) else {
|
let Some(r#ref) = find_ref(page, selector) else {
|
||||||
return Ok(CdpPage {
|
return Ok(CdpPage {
|
||||||
ok: false,
|
ok: false,
|
||||||
error: Some(
|
error: Some(
|
||||||
|
|
@ -444,7 +432,6 @@ async fn click(
|
||||||
&json!({
|
&json!({
|
||||||
"target_id": bind.target_id,
|
"target_id": bind.target_id,
|
||||||
"tab_id": bind.tab_id,
|
"tab_id": bind.tab_id,
|
||||||
"session": SESSION,
|
|
||||||
"ref": r#ref,
|
"ref": r#ref,
|
||||||
"input_route": "dom_event",
|
"input_route": "dom_event",
|
||||||
}),
|
}),
|
||||||
|
|
@ -465,7 +452,7 @@ async fn type_into(
|
||||||
tracing::info!(backend = "cua", tool = "browser_type", length = text.len());
|
tracing::info!(backend = "cua", tool = "browser_type", length = text.len());
|
||||||
if let Some(selector) = request.selector.as_deref() {
|
if let Some(selector) = request.selector.as_deref() {
|
||||||
let page = bind.page.as_ref().ok_or(ControlError::StaleReference)?;
|
let page = bind.page.as_ref().ok_or(ControlError::StaleReference)?;
|
||||||
let Some(r#ref) = find_ref(&page, selector) else {
|
let Some(r#ref) = find_ref(page, selector) else {
|
||||||
return Ok(CdpPage {
|
return Ok(CdpPage {
|
||||||
ok: false,
|
ok: false,
|
||||||
error: Some("target field is unavailable; no text inserted".into()),
|
error: Some("target field is unavailable; no text inserted".into()),
|
||||||
|
|
@ -484,7 +471,6 @@ async fn type_into(
|
||||||
&json!({
|
&json!({
|
||||||
"target_id": bind.target_id,
|
"target_id": bind.target_id,
|
||||||
"tab_id": bind.tab_id,
|
"tab_id": bind.tab_id,
|
||||||
"session": SESSION,
|
|
||||||
"ref": r#ref,
|
"ref": r#ref,
|
||||||
"text": text,
|
"text": text,
|
||||||
"replace": true,
|
"replace": true,
|
||||||
|
|
@ -501,7 +487,6 @@ async fn type_into(
|
||||||
"text": text,
|
"text": text,
|
||||||
"pid": bind.pid,
|
"pid": bind.pid,
|
||||||
"window_id": bind.window_id,
|
"window_id": bind.window_id,
|
||||||
"session": SESSION,
|
|
||||||
}),
|
}),
|
||||||
&[],
|
&[],
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ use std::path::PathBuf;
|
||||||
use std::process::Stdio;
|
use std::process::Stdio;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use serde_json::Value;
|
use serde_json::{Value, json};
|
||||||
use tokio::io::AsyncWriteExt;
|
use tokio::io::AsyncWriteExt;
|
||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
|
|
||||||
|
|
@ -36,6 +36,16 @@ impl CuaClient {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Every call runs in its own CLI process, so the driver would otherwise
|
||||||
|
/// give each one an implicit session that dies with the process. Trajectory
|
||||||
|
/// recording, snapshots, and browser binds only line up under one label.
|
||||||
|
pub fn session_for_display(display: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"lazyboy-{}",
|
||||||
|
normalize_display(display).trim_start_matches(':')
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn dbus_file(display: &str) -> PathBuf {
|
pub fn dbus_file(display: &str) -> PathBuf {
|
||||||
let number = normalize_display(display)
|
let number = normalize_display(display)
|
||||||
.trim_start_matches(':')
|
.trim_start_matches(':')
|
||||||
|
|
@ -79,6 +89,33 @@ impl CuaClient {
|
||||||
payload: &Value,
|
payload: &Value,
|
||||||
extra: &[&str],
|
extra: &[&str],
|
||||||
) -> Result<Value, ControlError> {
|
) -> Result<Value, ControlError> {
|
||||||
|
let mut body = with_session_label(screen, payload);
|
||||||
|
let mut escalated = false;
|
||||||
|
loop {
|
||||||
|
let outcome = self.attempt(screen, tool, &body, extra).await?;
|
||||||
|
if !escalated
|
||||||
|
&& outcome.error.is_some()
|
||||||
|
&& let Some(mode) = recommended_delivery(&outcome.value)
|
||||||
|
&& let Some(map) = body.as_object_mut()
|
||||||
|
{
|
||||||
|
map.insert("delivery_mode".to_string(), json!(mode));
|
||||||
|
escalated = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return match outcome.error {
|
||||||
|
Some(error) => Err(error),
|
||||||
|
None => Ok(outcome.value),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn attempt(
|
||||||
|
&self,
|
||||||
|
screen: &str,
|
||||||
|
tool: &str,
|
||||||
|
payload: &Value,
|
||||||
|
extra: &[&str],
|
||||||
|
) -> Result<Outcome, ControlError> {
|
||||||
let socket = Self::socket_for_display(screen);
|
let socket = Self::socket_for_display(screen);
|
||||||
if !socket.exists() {
|
if !socket.exists() {
|
||||||
return Err(ControlError::DriverUnavailable);
|
return Err(ControlError::DriverUnavailable);
|
||||||
|
|
@ -102,14 +139,12 @@ impl CuaClient {
|
||||||
let output = bounded_input_output(&mut command, payload).await?;
|
let output = bounded_input_output(&mut command, payload).await?;
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||||
let combined = format!("{stdout}\n{stderr}");
|
let (value, error) = decode_stdout(
|
||||||
let value =
|
&stdout,
|
||||||
parse_jsonish(&stdout).unwrap_or_else(|| Value::String(stdout.trim().to_string()));
|
&stderr,
|
||||||
let error = if !output.status.success() || stdout.trim_start().starts_with('❌') {
|
output.status.success(),
|
||||||
Some(classify_cua_failure(&combined))
|
classify_cua_failure,
|
||||||
} else {
|
);
|
||||||
response_error(&value)
|
|
||||||
};
|
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
backend = "cua",
|
backend = "cua",
|
||||||
tool,
|
tool,
|
||||||
|
|
@ -117,12 +152,54 @@ impl CuaClient {
|
||||||
duration_ms = started.elapsed().as_millis() as u64,
|
duration_ms = started.elapsed().as_millis() as u64,
|
||||||
success = error.is_none()
|
success = error.is_none()
|
||||||
);
|
);
|
||||||
if let Some(error) = error {
|
Ok(Outcome { value, error })
|
||||||
return Err(error);
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Ok(value)
|
struct Outcome {
|
||||||
|
value: Value,
|
||||||
|
error: Option<ControlError>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Refusals and prose diagnostics can arrive with exit code 0, so only a JSON
|
||||||
|
/// object counts as proof that the driver ran the tool.
|
||||||
|
fn decode_stdout(
|
||||||
|
stdout: &str,
|
||||||
|
stderr: &str,
|
||||||
|
success: bool,
|
||||||
|
classify: impl Fn(&str) -> ControlError,
|
||||||
|
) -> (Value, Option<ControlError>) {
|
||||||
|
let combined = format!("{stdout}\n{stderr}");
|
||||||
|
let trimmed = stdout.trim();
|
||||||
|
let empty = Value::Null;
|
||||||
|
let decoded = (!trimmed.is_empty())
|
||||||
|
.then(|| parse_jsonish(trimmed))
|
||||||
|
.flatten();
|
||||||
|
let error = if !success || trimmed.starts_with('\u{274c}') || decoded.is_none() {
|
||||||
|
Some(classify(&combined))
|
||||||
|
} else {
|
||||||
|
response_error(decoded.as_ref().unwrap_or(&empty))
|
||||||
|
};
|
||||||
|
(decoded.unwrap_or(empty), error)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The documented contract for `background_unavailable` is one retry with the
|
||||||
|
/// delivery mode named in the structured escalation.
|
||||||
|
fn recommended_delivery(value: &Value) -> Option<String> {
|
||||||
|
["/escalation/recommended", "/error/escalation/recommended"]
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|pointer| value.pointer(pointer).and_then(Value::as_str))
|
||||||
|
.find(|mode| matches!(*mode, "foreground" | "background"))
|
||||||
|
.map(str::to_string)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn with_session_label(display: &str, payload: &Value) -> Value {
|
||||||
|
let mut body = payload.clone();
|
||||||
|
if let Some(map) = body.as_object_mut() {
|
||||||
|
map.entry("session")
|
||||||
|
.or_insert_with(|| json!(CuaClient::session_for_display(display)));
|
||||||
}
|
}
|
||||||
|
body
|
||||||
}
|
}
|
||||||
|
|
||||||
fn response_error(value: &Value) -> Option<ControlError> {
|
fn response_error(value: &Value) -> Option<ControlError> {
|
||||||
|
|
@ -166,7 +243,15 @@ async fn bounded_input_output(
|
||||||
Ok::<(), std::io::Error>(())
|
Ok::<(), std::io::Error>(())
|
||||||
};
|
};
|
||||||
let (written, output) = tokio::join!(write, child.wait_with_output());
|
let (written, output) = tokio::join!(write, child.wait_with_output());
|
||||||
written.map_err(|_| ControlError::DriverUnhealthy)?;
|
// A driver that exits before reading stdin (unknown tool, rejected
|
||||||
|
// arguments) closes the pipe, so a broken pipe is expected and the exit
|
||||||
|
// status plus stderr hold the real reason. Losing them here would
|
||||||
|
// downgrade every fast refusal to a generic driver failure.
|
||||||
|
if let Err(error) = written
|
||||||
|
&& error.kind() != std::io::ErrorKind::BrokenPipe
|
||||||
|
{
|
||||||
|
return Err(ControlError::DriverUnhealthy);
|
||||||
|
}
|
||||||
output.map_err(|_| ControlError::DriverUnhealthy)
|
output.map_err(|_| ControlError::DriverUnhealthy)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
|
|
@ -291,6 +376,44 @@ mod tests {
|
||||||
assert!(response_error(&serde_json::json!({"outline": "❌ payment declined"})).is_none());
|
assert!(response_error(&serde_json::json!({"outline": "❌ payment declined"})).is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn only_a_json_object_proves_the_tool_ran() {
|
||||||
|
let classify = |text: &str| classify_cua_failure(text);
|
||||||
|
let (value, error) = decode_stdout(r#"{"width":1280}"#, "", true, classify);
|
||||||
|
assert!(error.is_none());
|
||||||
|
assert_eq!(value["width"], 1280);
|
||||||
|
|
||||||
|
// Prose with a zero exit code used to be reported as success.
|
||||||
|
let (value, error) = decode_stdout("no window matched", "", true, classify);
|
||||||
|
assert!(matches!(error, Some(ControlError::DriverUnhealthy)));
|
||||||
|
assert!(value.is_null());
|
||||||
|
|
||||||
|
let (_, error) = decode_stdout("\u{274c} unsupported tool", "", true, classify);
|
||||||
|
assert!(matches!(error, Some(ControlError::Unsupported)));
|
||||||
|
|
||||||
|
let (_, error) = decode_stdout(r#"{"code":"background_unavailable"}"#, "", true, classify);
|
||||||
|
assert!(error.is_some());
|
||||||
|
|
||||||
|
// A crash with empty stdout must never look like an empty success.
|
||||||
|
let (_, error) = decode_stdout("", "signal: 11", false, classify);
|
||||||
|
assert!(error.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn escalation_is_read_from_the_documented_pointers() {
|
||||||
|
assert_eq!(
|
||||||
|
recommended_delivery(&json!({"escalation": {"recommended": "foreground"}})).as_deref(),
|
||||||
|
Some("foreground")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
recommended_delivery(&json!({"error": {"escalation": {"recommended": "background"}}}))
|
||||||
|
.as_deref(),
|
||||||
|
Some("background")
|
||||||
|
);
|
||||||
|
assert!(recommended_delivery(&json!({"escalation": {"recommended": "reboot"}})).is_none());
|
||||||
|
assert!(recommended_delivery(&json!({"ok": true})).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn piped_json_reaches_eof_without_argv_exposure() {
|
async fn piped_json_reaches_eof_without_argv_exposure() {
|
||||||
let mut command = Command::new("sh");
|
let mut command = Command::new("sh");
|
||||||
|
|
@ -319,6 +442,27 @@ mod tests {
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn driver_that_never_reads_stdin_still_reports_its_exit() {
|
||||||
|
// Bigger than the pipe buffer: the child never reads, so the write can
|
||||||
|
// only fail with EPIPE and must not swallow the driver's own error.
|
||||||
|
let payload = json!({ "blob": "x".repeat(1 << 20) });
|
||||||
|
let mut command = Command::new("sh");
|
||||||
|
command.args(["-c", "echo invalid_action_target >&2; exit 1"]);
|
||||||
|
let output = tokio::time::timeout(
|
||||||
|
Duration::from_secs(10),
|
||||||
|
bounded_input_output(&mut command, &payload),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("bounded")
|
||||||
|
.expect("exit status survives a broken stdin pipe");
|
||||||
|
assert!(!output.status.success());
|
||||||
|
assert!(
|
||||||
|
String::from_utf8_lossy(&output.stderr).contains("invalid_action_target"),
|
||||||
|
"driver stderr must reach the classifier"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn unknown_driver_failure_does_not_echo_secret() {
|
fn unknown_driver_failure_does_not_echo_secret() {
|
||||||
let error = classify_cua_failure("failed typing secret-password");
|
let error = classify_cua_failure("failed typing secret-password");
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ mod translate;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
|
@ -22,12 +23,29 @@ use crate::controller::{
|
||||||
use crate::process::spawn_detached;
|
use crate::process::spawn_detached;
|
||||||
use crate::{
|
use crate::{
|
||||||
ActionRequest, ActionResult, BrowserRequest, CdpPage, RecordingRequest, RecordingResult,
|
ActionRequest, ActionResult, BrowserRequest, CdpPage, RecordingRequest, RecordingResult,
|
||||||
RecordingSession, action_pause_ms, normalize_display, observation_from_png,
|
RecordingSession, action_pause_ms, image_dimensions, normalize_display, observation_from_png,
|
||||||
observation_with_elements, teach_trajectory_dir,
|
observation_with_elements, teach_trajectory_dir,
|
||||||
};
|
};
|
||||||
use client::first_array_of_objects;
|
use client::first_array_of_objects;
|
||||||
|
|
||||||
pub use client::CuaClient;
|
pub use client::CuaClient;
|
||||||
|
|
||||||
|
/// Last resort when neither the screenshot nor the driver reports a mode.
|
||||||
|
const FALLBACK_SCREEN: (u32, u32) = (1280, 800);
|
||||||
|
|
||||||
|
/// `image/computer/Dockerfile` pins `CUA_DRIVER_RS_VERSION`; only that tool
|
||||||
|
/// surface is guaranteed. Patch releases stay compatible, a new minor does not.
|
||||||
|
const PINNED_DRIVER: (u32, u32) = (0, 23);
|
||||||
|
|
||||||
|
/// `cua-driver --version` prints `cua-driver 0.23.2`, and some builds append a
|
||||||
|
/// target suffix (`cua-driver 0.23.2 (x86_64-linux)`), so only the leading
|
||||||
|
/// numeric version is trusted.
|
||||||
|
fn driver_release(version: &str) -> Option<(u32, u32)> {
|
||||||
|
let digits = version.find(|character: char| character.is_ascii_digit())?;
|
||||||
|
let mut parts = version[digits..].split('.');
|
||||||
|
Some((parts.next()?.parse().ok()?, parts.next()?.parse().ok()?))
|
||||||
|
}
|
||||||
|
|
||||||
pub use translate::{TranslatedAction, translate_action};
|
pub use translate::{TranslatedAction, translate_action};
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
|
|
@ -36,6 +54,7 @@ pub struct CuaController {
|
||||||
screens: tokio::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
|
screens: tokio::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
|
||||||
native: tokio::sync::Mutex<HashMap<String, HashMap<String, native::NativeTarget>>>,
|
native: tokio::sync::Mutex<HashMap<String, HashMap<String, native::NativeTarget>>>,
|
||||||
browser: tokio::sync::Mutex<HashMap<String, browser::BrowserBind>>,
|
browser: tokio::sync::Mutex<HashMap<String, browser::BrowserBind>>,
|
||||||
|
snapshots: AtomicU64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
|
|
@ -52,7 +71,9 @@ impl ComputerController for CuaController {
|
||||||
.await;
|
.await;
|
||||||
match report {
|
match report {
|
||||||
Ok(report) => {
|
Ok(report) => {
|
||||||
let compatible = version.as_deref() == Some("cua-driver 0.23.2");
|
let compatible = version
|
||||||
|
.as_deref()
|
||||||
|
.is_some_and(|text| driver_release(text) == Some(PINNED_DRIVER));
|
||||||
let healthy =
|
let healthy =
|
||||||
compatible && report.get("overall").and_then(Value::as_str) == Some("ok");
|
compatible && report.get("overall").and_then(Value::as_str) == Some("ok");
|
||||||
let mut details: Vec<String> = report
|
let mut details: Vec<String> = report
|
||||||
|
|
@ -64,7 +85,12 @@ impl ComputerController for CuaController {
|
||||||
.filter_map(|check| check["message"].as_str().map(str::to_string))
|
.filter_map(|check| check["message"].as_str().map(str::to_string))
|
||||||
.collect();
|
.collect();
|
||||||
if !compatible {
|
if !compatible {
|
||||||
details.push("expected pinned Cua Driver 0.23.2".into());
|
details.push(format!(
|
||||||
|
"Cua Driver {} is not the pinned {}.{} series",
|
||||||
|
version.as_deref().unwrap_or("unknown"),
|
||||||
|
PINNED_DRIVER.0,
|
||||||
|
PINNED_DRIVER.1
|
||||||
|
));
|
||||||
}
|
}
|
||||||
Ok(ControllerHealth {
|
Ok(ControllerHealth {
|
||||||
backend: "cua".into(),
|
backend: "cua".into(),
|
||||||
|
|
@ -292,15 +318,6 @@ impl CuaController {
|
||||||
completed += 5;
|
completed += 5;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if matches!(
|
|
||||||
action,
|
|
||||||
ComputerAction::Pointer {
|
|
||||||
pointer_type: PointerType::Down | PointerType::Up,
|
|
||||||
..
|
|
||||||
}
|
|
||||||
) {
|
|
||||||
return Err(ControlError::Unsupported);
|
|
||||||
}
|
|
||||||
if let ComputerAction::Ref {
|
if let ComputerAction::Ref {
|
||||||
verb,
|
verb,
|
||||||
target,
|
target,
|
||||||
|
|
@ -362,7 +379,10 @@ impl CuaController {
|
||||||
if png.is_empty() {
|
if png.is_empty() {
|
||||||
return Err(ControlError::Internal("screenshot failed".into()));
|
return Err(ControlError::Internal("screenshot failed".into()));
|
||||||
}
|
}
|
||||||
let (width, height) = png_dimensions(&png);
|
let (width, height) = match image_dimensions(&png) {
|
||||||
|
Some(dimensions) => dimensions,
|
||||||
|
None => self.screen_size(display).await,
|
||||||
|
};
|
||||||
let cursor = self.cursor(display).await;
|
let cursor = self.cursor(display).await;
|
||||||
let windows = self.windows(display).await.unwrap_or_default();
|
let windows = self.windows(display).await.unwrap_or_default();
|
||||||
let active = windows
|
let active = windows
|
||||||
|
|
@ -372,8 +392,15 @@ impl CuaController {
|
||||||
id: window.id.to_string(),
|
id: window.id.to_string(),
|
||||||
title: Some(window.title.clone()).filter(|title| !title.is_empty()),
|
title: Some(window.title.clone()).filter(|title| !title.is_empty()),
|
||||||
});
|
});
|
||||||
let (mut elements, targets) =
|
// Selectors only need to be short and unique: a process-local counter
|
||||||
native::observe(&self.client, display, &windows, &png_path.to_string_lossy()).await?;
|
// keeps stale handles from resolving without pasting a temp path into
|
||||||
|
// every identifier the Agent echoes back.
|
||||||
|
let snapshot = self.snapshots.fetch_add(1, Ordering::Relaxed);
|
||||||
|
let native::NativeObservation {
|
||||||
|
mut elements,
|
||||||
|
targets,
|
||||||
|
complete,
|
||||||
|
} = native::observe(&self.client, display, &windows, &format!("s{snapshot}")).await;
|
||||||
elements.extend(
|
elements.extend(
|
||||||
windows
|
windows
|
||||||
.into_iter()
|
.into_iter()
|
||||||
|
|
@ -402,10 +429,38 @@ impl CuaController {
|
||||||
observation_from_png(png, width, height, cursor, active),
|
observation_from_png(png, width, height, cursor, active),
|
||||||
elements,
|
elements,
|
||||||
);
|
);
|
||||||
observation.native_observation_complete = true;
|
observation.native_observation_complete = complete;
|
||||||
Ok(observation)
|
Ok(observation)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `scroll` on the desktop plane is aimed at a point and the action DSL
|
||||||
|
/// does not carry one, so aim at the pointer; the screen centre is the
|
||||||
|
/// next best guess when the pointer cannot be read.
|
||||||
|
async fn scroll_point(&self, display: &str) -> (u32, u32) {
|
||||||
|
if let Some(cursor) = self.cursor(display).await
|
||||||
|
&& cursor.x >= 0
|
||||||
|
&& cursor.y >= 0
|
||||||
|
{
|
||||||
|
return (cursor.x as u32, cursor.y as u32);
|
||||||
|
}
|
||||||
|
let (width, height) = self.screen_size(display).await;
|
||||||
|
(width / 2, height / 2)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn screen_size(&self, display: &str) -> (u32, u32) {
|
||||||
|
self.client
|
||||||
|
.call(display, "get_screen_size", &json!({}), &[])
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.and_then(|value| {
|
||||||
|
Some((
|
||||||
|
value.get("width").and_then(Value::as_u64)? as u32,
|
||||||
|
value.get("height").and_then(Value::as_u64)? as u32,
|
||||||
|
))
|
||||||
|
})
|
||||||
|
.unwrap_or(FALLBACK_SCREEN)
|
||||||
|
}
|
||||||
|
|
||||||
async fn cursor(&self, display: &str) -> Option<CursorPosition> {
|
async fn cursor(&self, display: &str) -> Option<CursorPosition> {
|
||||||
let value = self
|
let value = self
|
||||||
.client
|
.client
|
||||||
|
|
@ -449,6 +504,11 @@ impl CuaController {
|
||||||
}
|
}
|
||||||
TranslatedAction::FocusTitle { title } => self.focus_title(display, &title).await,
|
TranslatedAction::FocusTitle { title } => self.focus_title(display, &title).await,
|
||||||
TranslatedAction::Cua { tool, mut payload } => {
|
TranslatedAction::Cua { tool, mut payload } => {
|
||||||
|
if tool == "scroll" && payload.get("x").is_none() {
|
||||||
|
let (x, y) = self.scroll_point(display).await;
|
||||||
|
payload["x"] = json!(x);
|
||||||
|
payload["y"] = json!(y);
|
||||||
|
}
|
||||||
if tool == "drag" {
|
if tool == "drag" {
|
||||||
let x = payload["from_x"].as_f64().unwrap_or(0.0);
|
let x = payload["from_x"].as_f64().unwrap_or(0.0);
|
||||||
let y = payload["from_y"].as_f64().unwrap_or(0.0);
|
let y = payload["from_y"].as_f64().unwrap_or(0.0);
|
||||||
|
|
@ -662,12 +722,6 @@ fn number(value: &Value, key: &str) -> Option<u64> {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn png_dimensions(bytes: &[u8]) -> (u32, u32) {
|
|
||||||
image::load_from_memory(bytes)
|
|
||||||
.map(|image| (image.width(), image.height()))
|
|
||||||
.unwrap_or((1280, 800))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn observe_png_path(display: &str) -> PathBuf {
|
fn observe_png_path(display: &str) -> PathBuf {
|
||||||
let nanos = SystemTime::now()
|
let nanos = SystemTime::now()
|
||||||
.duration_since(UNIX_EPOCH)
|
.duration_since(UNIX_EPOCH)
|
||||||
|
|
@ -683,6 +737,23 @@ fn observe_png_path(display: &str) -> PathBuf {
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn driver_release_reads_the_pinned_minor_series() {
|
||||||
|
assert_eq!(driver_release("cua-driver 0.23.2"), Some((0, 23)));
|
||||||
|
assert_eq!(
|
||||||
|
driver_release("cua-driver 0.23.2 (x86_64-linux)"),
|
||||||
|
Some((0, 23))
|
||||||
|
);
|
||||||
|
assert_eq!(driver_release("cua-driver"), None);
|
||||||
|
assert_eq!(driver_release(""), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_new_minor_series_is_not_compatible() {
|
||||||
|
assert_ne!(driver_release("cua-driver 0.24.0"), Some(PINNED_DRIVER));
|
||||||
|
assert_eq!(driver_release("cua-driver 0.23.9"), Some(PINNED_DRIVER));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn normalized_drag_uses_one_driver_gesture() {
|
fn normalized_drag_uses_one_driver_gesture() {
|
||||||
let actions = crate::parse_computer_actions(
|
let actions = crate::parse_computer_actions(
|
||||||
|
|
|
||||||
|
|
@ -13,19 +13,30 @@ pub(super) struct NativeTarget {
|
||||||
token: String,
|
token: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub(super) struct NativeObservation {
|
||||||
|
pub(super) elements: Vec<UiElement>,
|
||||||
|
pub(super) targets: HashMap<String, NativeTarget>,
|
||||||
|
/// False when any window answered with a degraded tree, which means the
|
||||||
|
/// element list is partial and a second opinion is still worth taking.
|
||||||
|
pub(super) complete: bool,
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) async fn observe(
|
pub(super) async fn observe(
|
||||||
client: &CuaClient,
|
client: &CuaClient,
|
||||||
display: &str,
|
display: &str,
|
||||||
windows: &[ListedWindow],
|
windows: &[ListedWindow],
|
||||||
generation: &str,
|
snapshot_id: &str,
|
||||||
) -> Result<(Vec<UiElement>, HashMap<String, NativeTarget>), ControlError> {
|
) -> NativeObservation {
|
||||||
let mut elements = Vec::new();
|
let mut observed = NativeObservation {
|
||||||
let mut targets = HashMap::new();
|
complete: true,
|
||||||
|
..NativeObservation::default()
|
||||||
|
};
|
||||||
for window in windows
|
for window in windows
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|window| !window.app_name.to_ascii_lowercase().contains("chrom"))
|
.filter(|window| !window.app_name.to_ascii_lowercase().contains("chrom"))
|
||||||
{
|
{
|
||||||
let state = client
|
let state = match client
|
||||||
.call(
|
.call(
|
||||||
display,
|
display,
|
||||||
"get_window_state",
|
"get_window_state",
|
||||||
|
|
@ -35,7 +46,20 @@ pub(super) async fn observe(
|
||||||
}),
|
}),
|
||||||
&[],
|
&[],
|
||||||
)
|
)
|
||||||
.await?;
|
.await
|
||||||
|
{
|
||||||
|
Ok(state) => state,
|
||||||
|
Err(_) => {
|
||||||
|
observed.complete = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
// A degraded reply is window metadata plus a root node: usable for
|
||||||
|
// discovery, not as an element tree, and never as proof of coverage.
|
||||||
|
if state.get("degraded").and_then(Value::as_bool) == Some(true) {
|
||||||
|
observed.complete = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
for item in state
|
for item in state
|
||||||
.get("elements")
|
.get("elements")
|
||||||
.and_then(Value::as_array)
|
.and_then(Value::as_array)
|
||||||
|
|
@ -48,7 +72,7 @@ pub(super) async fn observe(
|
||||||
if item.get("enabled").and_then(Value::as_bool) == Some(false) {
|
if item.get("enabled").and_then(Value::as_bool) == Some(false) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let selector = format!("cua:{generation}:{}:{token}", window.id);
|
let selector = format!("cua:{snapshot_id}:{}:{token}", window.id);
|
||||||
let title = item.get("label").and_then(Value::as_str).unwrap_or("");
|
let title = item.get("label").and_then(Value::as_str).unwrap_or("");
|
||||||
let role = item.get("role").and_then(Value::as_str).unwrap_or("");
|
let role = item.get("role").and_then(Value::as_str).unwrap_or("");
|
||||||
let frame = item.get("frame").unwrap_or(&Value::Null);
|
let frame = item.get("frame").unwrap_or(&Value::Null);
|
||||||
|
|
@ -59,7 +83,7 @@ pub(super) async fn observe(
|
||||||
.unwrap_or(0.0)
|
.unwrap_or(0.0)
|
||||||
.clamp(0.0, u32::MAX as f64) as u32
|
.clamp(0.0, u32::MAX as f64) as u32
|
||||||
};
|
};
|
||||||
elements.push(UiElement {
|
observed.elements.push(UiElement {
|
||||||
id: 0,
|
id: 0,
|
||||||
title: if title.is_empty() {
|
title: if title.is_empty() {
|
||||||
role.into()
|
role.into()
|
||||||
|
|
@ -74,7 +98,7 @@ pub(super) async fn observe(
|
||||||
kind: Some("a11y".into()),
|
kind: Some("a11y".into()),
|
||||||
role: Some(role.into()),
|
role: Some(role.into()),
|
||||||
});
|
});
|
||||||
targets.insert(
|
observed.targets.insert(
|
||||||
selector,
|
selector,
|
||||||
NativeTarget {
|
NativeTarget {
|
||||||
pid: window.pid,
|
pid: window.pid,
|
||||||
|
|
@ -84,7 +108,7 @@ pub(super) async fn observe(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok((elements, targets))
|
observed
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn act(
|
pub(super) async fn act(
|
||||||
|
|
|
||||||
|
|
@ -36,7 +36,7 @@ pub fn translate_action(
|
||||||
y,
|
y,
|
||||||
pointer_type,
|
pointer_type,
|
||||||
button,
|
button,
|
||||||
} => Ok(translate_pointer(*x, *y, pointer_type, *button)),
|
} => translate_pointer(*x, *y, pointer_type, *button),
|
||||||
ComputerAction::Clipboard { text } => {
|
ComputerAction::Clipboard { text } => {
|
||||||
tracing::info!(backend = "cua", tool = "type_text", length = text.len());
|
tracing::info!(backend = "cua", tool = "type_text", length = text.len());
|
||||||
Ok(TranslatedAction::Cua {
|
Ok(TranslatedAction::Cua {
|
||||||
|
|
@ -44,7 +44,6 @@ pub fn translate_action(
|
||||||
payload: json!({
|
payload: json!({
|
||||||
"text": text,
|
"text": text,
|
||||||
"scope": "desktop",
|
"scope": "desktop",
|
||||||
"target": { "kind": "desktop", "display_id": "primary" },
|
|
||||||
}),
|
}),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -56,10 +55,9 @@ pub fn translate_action(
|
||||||
ScrollDirection::Up => "up",
|
ScrollDirection::Up => "up",
|
||||||
ScrollDirection::Down => "down",
|
ScrollDirection::Down => "down",
|
||||||
},
|
},
|
||||||
"amount": amount.unwrap_or(12),
|
"amount": amount.unwrap_or(12).clamp(1, 50),
|
||||||
"by": "line",
|
"by": "line",
|
||||||
"scope": "desktop",
|
"scope": "desktop",
|
||||||
"target": { "kind": "desktop", "display_id": "primary" },
|
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
|
|
@ -70,40 +68,33 @@ fn translate_pointer(
|
||||||
y: u32,
|
y: u32,
|
||||||
pointer_type: &PointerType,
|
pointer_type: &PointerType,
|
||||||
button: Option<PointerButton>,
|
button: Option<PointerButton>,
|
||||||
) -> TranslatedAction {
|
) -> Result<TranslatedAction, ControlError> {
|
||||||
let button = match button.unwrap_or(PointerButton::Left) {
|
let button = match button.unwrap_or(PointerButton::Left) {
|
||||||
PointerButton::Left => "left",
|
PointerButton::Left => "left",
|
||||||
PointerButton::Middle => "middle",
|
PointerButton::Middle => "middle",
|
||||||
PointerButton::Right => "right",
|
PointerButton::Right => "right",
|
||||||
};
|
};
|
||||||
match pointer_type {
|
match pointer_type {
|
||||||
PointerType::Move => TranslatedAction::Cua {
|
PointerType::Move => Ok(TranslatedAction::Cua {
|
||||||
tool: "move_cursor",
|
tool: "move_cursor",
|
||||||
payload: json!({
|
payload: json!({
|
||||||
"x": x,
|
"x": x,
|
||||||
"y": y,
|
"y": y,
|
||||||
"scope": "desktop",
|
"scope": "desktop",
|
||||||
"target": { "kind": "desktop", "display_id": "primary" },
|
|
||||||
}),
|
}),
|
||||||
},
|
}),
|
||||||
PointerType::Click => TranslatedAction::Cua {
|
PointerType::Click => Ok(TranslatedAction::Cua {
|
||||||
tool: "click",
|
tool: "click",
|
||||||
payload: json!({
|
payload: json!({
|
||||||
"x": x,
|
"x": x,
|
||||||
"y": y,
|
"y": y,
|
||||||
"button": button,
|
"button": button,
|
||||||
"scope": "desktop",
|
"scope": "desktop",
|
||||||
"target": { "kind": "desktop", "display_id": "primary" },
|
|
||||||
}),
|
}),
|
||||||
},
|
}),
|
||||||
PointerType::Down => TranslatedAction::Cua {
|
// A held-button gesture collapses into one `drag` call before it gets
|
||||||
tool: "mouse_button_down",
|
// here; the CLI has no lease that keeps a button down between calls.
|
||||||
payload: json!({ "x": x, "y": y, "button": button }),
|
PointerType::Down | PointerType::Up => Err(ControlError::Unsupported),
|
||||||
},
|
|
||||||
PointerType::Up => TranslatedAction::Cua {
|
|
||||||
tool: "mouse_button_up",
|
|
||||||
payload: json!({ "x": x, "y": y, "button": button }),
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -118,7 +109,6 @@ fn translate_key(key: &str, modifiers: Option<&[String]>) -> TranslatedAction {
|
||||||
payload: json!({
|
payload: json!({
|
||||||
"keys": keys,
|
"keys": keys,
|
||||||
"scope": "desktop",
|
"scope": "desktop",
|
||||||
"target": { "kind": "desktop", "display_id": "primary" },
|
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -127,7 +117,6 @@ fn translate_key(key: &str, modifiers: Option<&[String]>) -> TranslatedAction {
|
||||||
payload: json!({
|
payload: json!({
|
||||||
"key": key,
|
"key": key,
|
||||||
"scope": "desktop",
|
"scope": "desktop",
|
||||||
"target": { "kind": "desktop", "display_id": "primary" },
|
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
@ -166,7 +155,76 @@ mod tests {
|
||||||
assert_eq!(payload["x"], 40);
|
assert_eq!(payload["x"], 40);
|
||||||
assert_eq!(payload["y"], 80);
|
assert_eq!(payload["y"], 80);
|
||||||
assert_eq!(payload["scope"], "desktop");
|
assert_eq!(payload["scope"], "desktop");
|
||||||
assert_eq!(payload["target"]["display_id"], "primary");
|
// 0.23.2 rejects an explicit desktop target with invalid_action_target.
|
||||||
|
assert!(payload.get("target").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn desktop_actions_omit_the_rejected_target() {
|
||||||
|
for action in [
|
||||||
|
ComputerAction::Clipboard { text: "hi".into() },
|
||||||
|
ComputerAction::Key {
|
||||||
|
key: "return".into(),
|
||||||
|
modifiers: None,
|
||||||
|
},
|
||||||
|
ComputerAction::Key {
|
||||||
|
key: "c".into(),
|
||||||
|
modifiers: Some(vec!["ctrl".into()]),
|
||||||
|
},
|
||||||
|
ComputerAction::Pointer {
|
||||||
|
x: 1,
|
||||||
|
y: 2,
|
||||||
|
pointer_type: PointerType::Move,
|
||||||
|
button: None,
|
||||||
|
},
|
||||||
|
ComputerAction::Scroll {
|
||||||
|
direction: ScrollDirection::Down,
|
||||||
|
amount: None,
|
||||||
|
},
|
||||||
|
] {
|
||||||
|
let TranslatedAction::Cua { payload, .. } =
|
||||||
|
translate_action(&action, ":1", None).unwrap()
|
||||||
|
else {
|
||||||
|
panic!("every desktop action translates to a Cua tool");
|
||||||
|
};
|
||||||
|
assert!(payload.get("target").is_none(), "{action:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scroll_amount_stays_inside_the_driver_window() {
|
||||||
|
let scroll = |amount| match translate_action(
|
||||||
|
&ComputerAction::Scroll {
|
||||||
|
direction: ScrollDirection::Down,
|
||||||
|
amount: Some(amount),
|
||||||
|
},
|
||||||
|
":1",
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
{
|
||||||
|
TranslatedAction::Cua { payload, .. } => payload["amount"].as_i64().unwrap(),
|
||||||
|
other => panic!("expected scroll, got {other:?}"),
|
||||||
|
};
|
||||||
|
assert_eq!(scroll(0), 1);
|
||||||
|
assert_eq!(scroll(12), 12);
|
||||||
|
assert_eq!(scroll(200), 50);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn held_buttons_are_not_translated_one_by_one() {
|
||||||
|
for pointer_type in [PointerType::Down, PointerType::Up] {
|
||||||
|
let action = ComputerAction::Pointer {
|
||||||
|
x: 1,
|
||||||
|
y: 2,
|
||||||
|
pointer_type,
|
||||||
|
button: Some(PointerButton::Left),
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
translate_action(&action, ":1", None),
|
||||||
|
Err(ControlError::Unsupported)
|
||||||
|
));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,9 @@
|
||||||
use thiserror::Error;
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct ScreenLease {
|
pub struct ScreenLease {
|
||||||
pub owner_id: String,
|
pub owner_id: String,
|
||||||
pub fence: u32,
|
pub fence: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Error, Clone, PartialEq, Eq)]
|
|
||||||
#[error("computer is busy")]
|
|
||||||
pub struct ComputerBusyError;
|
|
||||||
|
|
||||||
pub fn screen_lease_id(run_id: &str, fence: u32) -> String {
|
pub fn screen_lease_id(run_id: &str, fence: u32) -> String {
|
||||||
format!("{run_id}:{fence}")
|
format!("{run_id}:{fence}")
|
||||||
}
|
}
|
||||||
|
|
@ -57,10 +51,6 @@ pub fn can_release_screen_lease(existing: Option<&str>, incoming: Option<&str>)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn next_fence(current: u32) -> u32 {
|
|
||||||
current.saturating_add(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use lazyboy_contracts::{ComputerAction, ComputerObservation, RefVerb};
|
use lazyboy_contracts::{
|
||||||
|
ComputerAction, ComputerObservation, DEFAULT_SCREEN_HEIGHT, DEFAULT_SCREEN_WIDTH, RefVerb,
|
||||||
|
};
|
||||||
use tokio::time::{Duration, sleep};
|
use tokio::time::{Duration, sleep};
|
||||||
|
|
||||||
use crate::controller::{
|
use crate::controller::{
|
||||||
|
|
@ -11,7 +13,7 @@ use crate::process::{
|
||||||
use crate::{
|
use crate::{
|
||||||
ActionRequest, ActionResult, BrowserRequest, CdpPage, RecordingRequest, RecordingResult,
|
ActionRequest, ActionResult, BrowserRequest, CdpPage, RecordingRequest, RecordingResult,
|
||||||
RecordingSession, a11y_command_on, action_pause_ms, cdp_command_on, cdp_record_command_on,
|
RecordingSession, a11y_command_on, action_pause_ms, cdp_command_on, cdp_record_command_on,
|
||||||
cdp_record_stop_command, devtools_port, launch_argv_on, observation_from_png,
|
cdp_record_stop_command, devtools_port, image_dimensions, launch_argv_on, observation_from_png,
|
||||||
observation_with_elements, open_argv_on, parse_a11y_page, parse_cdp_page, parse_pointer_state,
|
observation_with_elements, open_argv_on, parse_a11y_page, parse_cdp_page, parse_pointer_state,
|
||||||
parse_ui_elements, pointer_state_command_on, screenshot_command_on, teach_recorder_output,
|
parse_ui_elements, pointer_state_command_on, screenshot_command_on, teach_recorder_output,
|
||||||
teach_trajectory_dir, window_list_command_on, xdotool_argv_on,
|
teach_trajectory_dir, window_list_command_on, xdotool_argv_on,
|
||||||
|
|
@ -153,13 +155,17 @@ async fn legacy_browser(
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn observe_display(display: &str) -> Result<ComputerObservation, ControlError> {
|
pub async fn observe_display(display: &str) -> Result<ComputerObservation, ControlError> {
|
||||||
let png = capture_stdout(&screenshot_command_on(display))
|
let frame = capture_stdout(&screenshot_command_on(display))
|
||||||
.await
|
.await
|
||||||
.map_err(|_| ControlError::Internal("screenshot failed".into()))?;
|
.map_err(|_| ControlError::Internal("screenshot failed".into()))?;
|
||||||
|
// The capture is a JPEG of the root window, so the frame carries the real
|
||||||
|
// geometry; the constants are only a fallback for an undecodable frame.
|
||||||
|
let (width, height) =
|
||||||
|
image_dimensions(&frame).unwrap_or((DEFAULT_SCREEN_WIDTH, DEFAULT_SCREEN_HEIGHT));
|
||||||
let ((cursor, window), elements) =
|
let ((cursor, window), elements) =
|
||||||
tokio::join!(run_pointer_state(display), run_window_list(display));
|
tokio::join!(run_pointer_state(display), run_window_list(display));
|
||||||
Ok(observation_with_elements(
|
Ok(observation_with_elements(
|
||||||
observation_from_png(png, 1280, 800, cursor, window),
|
observation_from_png(frame, width, height, cursor, window),
|
||||||
elements,
|
elements,
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,20 @@ pub fn observation_from_png(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Pixel size of a captured frame, or `None` when the bytes are not decodable.
|
||||||
|
/// Each driver captures with a different codec (the legacy pipeline emits JPEG
|
||||||
|
/// from `xwd | convert`, the Cua driver writes PNG), so the dimensions have to
|
||||||
|
/// come from the frame itself rather than from a configured constant.
|
||||||
|
pub fn image_dimensions(bytes: &[u8]) -> Option<(u32, u32)> {
|
||||||
|
use image::ImageDecoder;
|
||||||
|
|
||||||
|
let reader = image::ImageReader::new(std::io::Cursor::new(bytes))
|
||||||
|
.with_guessed_format()
|
||||||
|
.ok()?;
|
||||||
|
let decoder = reader.into_decoder().ok()?;
|
||||||
|
Some(decoder.dimensions())
|
||||||
|
}
|
||||||
|
|
||||||
pub fn observation_to_control_json(observation: &ComputerObservation) -> Value {
|
pub fn observation_to_control_json(observation: &ComputerObservation) -> Value {
|
||||||
let mut body = json!({
|
let mut body = json!({
|
||||||
"png_base64": base64::engine::general_purpose::STANDARD.encode(&observation.image),
|
"png_base64": base64::engine::general_purpose::STANDARD.encode(&observation.image),
|
||||||
|
|
@ -109,6 +123,33 @@ pub fn signatures_similar(a: &[u8], b: &[u8]) -> bool {
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use image::codecs::jpeg::JpegEncoder;
|
||||||
|
use image::codecs::png::PngEncoder;
|
||||||
|
use image::{ExtendedColorType, ImageEncoder, Rgb, RgbImage};
|
||||||
|
use std::io::Cursor;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dimensions_come_from_the_frame_for_both_codecs() {
|
||||||
|
let frame = RgbImage::from_pixel(96, 48, Rgb([10, 20, 30]));
|
||||||
|
|
||||||
|
let mut png = Cursor::new(Vec::new());
|
||||||
|
PngEncoder::new(&mut png)
|
||||||
|
.write_image(frame.as_raw(), 96, 48, ExtendedColorType::Rgb8)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(image_dimensions(&png.into_inner()), Some((96, 48)));
|
||||||
|
|
||||||
|
// The legacy driver captures JPEG, so a hardcoded size would drift the
|
||||||
|
// moment the Xvfb geometry changes.
|
||||||
|
let mut jpeg = Cursor::new(Vec::new());
|
||||||
|
JpegEncoder::new_with_quality(&mut jpeg, 60)
|
||||||
|
.encode(frame.as_raw(), 96, 48, ExtendedColorType::Rgb8)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(image_dimensions(&jpeg.into_inner()), Some((96, 48)));
|
||||||
|
|
||||||
|
// A truncated capture must not invent a size.
|
||||||
|
assert_eq!(image_dimensions(&[0xFF, 0xD8, 0xFF]), None);
|
||||||
|
assert_eq!(image_dimensions(&[]), None);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn identical_bytes_share_a_frame_id() {
|
fn identical_bytes_share_a_frame_id() {
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,6 @@ pub fn is_browser_title(title: &str) -> bool {
|
||||||
|
|
||||||
use crate::screen::{PRIMARY_DISPLAY, normalize_display};
|
use crate::screen::{PRIMARY_DISPLAY, normalize_display};
|
||||||
|
|
||||||
pub const DISPLAY: &str = PRIMARY_DISPLAY;
|
|
||||||
pub const HOME: &str = "/home/lazyboy";
|
pub const HOME: &str = "/home/lazyboy";
|
||||||
|
|
||||||
fn display_env(display: &str) -> String {
|
fn display_env(display: &str) -> String {
|
||||||
|
|
@ -170,10 +169,6 @@ pub fn action_pause_ms(action: &ComputerAction) -> u64 {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn pointer_state_command() -> Vec<String> {
|
|
||||||
pointer_state_command_on(PRIMARY_DISPLAY)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn pointer_state_command_on(display: &str) -> Vec<String> {
|
pub fn pointer_state_command_on(display: &str) -> Vec<String> {
|
||||||
vec![
|
vec![
|
||||||
"env".into(),
|
"env".into(),
|
||||||
|
|
@ -200,10 +195,6 @@ print(json.dumps({"x": int(vals.get("X") or 0), "y": int(vals.get("Y") or 0), "i
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn window_list_command() -> Vec<String> {
|
|
||||||
window_list_command_on(PRIMARY_DISPLAY)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn window_list_command_on(display: &str) -> Vec<String> {
|
pub fn window_list_command_on(display: &str) -> Vec<String> {
|
||||||
vec![
|
vec![
|
||||||
"env".into(),
|
"env".into(),
|
||||||
|
|
@ -309,10 +300,6 @@ pub fn parse_pointer_state(
|
||||||
(cursor, window)
|
(cursor, window)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn open_argv(path: &str) -> Vec<String> {
|
|
||||||
open_argv_on(PRIMARY_DISPLAY, None, path)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn open_argv_on(display: &str, profile: Option<&str>, path: &str) -> Vec<String> {
|
pub fn open_argv_on(display: &str, profile: Option<&str>, path: &str) -> Vec<String> {
|
||||||
if path.starts_with("http://") || path.starts_with("https://") {
|
if path.starts_with("http://") || path.starts_with("https://") {
|
||||||
return browser_argv(display, profile, Some(path));
|
return browser_argv(display, profile, Some(path));
|
||||||
|
|
@ -325,10 +312,6 @@ pub fn open_argv_on(display: &str, profile: Option<&str>, path: &str) -> Vec<Str
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn launch_argv(application: &str, uri: Option<&str>) -> Option<Vec<String>> {
|
|
||||||
launch_argv_on(PRIMARY_DISPLAY, None, application, uri)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn launch_argv_on(
|
pub fn launch_argv_on(
|
||||||
display: &str,
|
display: &str,
|
||||||
profile: Option<&str>,
|
profile: Option<&str>,
|
||||||
|
|
@ -371,10 +354,6 @@ fn browser_argv(display: &str, profile: Option<&str>, uri: Option<&str>) -> Vec<
|
||||||
argv
|
argv
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn screenshot_command() -> Vec<String> {
|
|
||||||
screenshot_command_on(PRIMARY_DISPLAY)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn screenshot_command_on(display: &str) -> Vec<String> {
|
pub fn screenshot_command_on(display: &str) -> Vec<String> {
|
||||||
vec![
|
vec![
|
||||||
"bash".into(),
|
"bash".into(),
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,16 @@
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use base64::Engine;
|
use base64::Engine;
|
||||||
use lazyboy_contracts::ComputerCapabilities;
|
use lazyboy_contracts::ComputerCapabilities;
|
||||||
use lazyboy_contracts::{ActiveWindow, ComputerObservation, CursorPosition, SandboxKind};
|
use lazyboy_contracts::{
|
||||||
|
ActiveWindow, ComputerObservation, CursorPosition, DEFAULT_SCREEN_HEIGHT, DEFAULT_SCREEN_WIDTH,
|
||||||
|
SandboxKind,
|
||||||
|
};
|
||||||
use lazyboy_control::{
|
use lazyboy_control::{
|
||||||
ActionRequest, ActionResult, AdapterContext, BrowserRequest, CdpPage, CommandRequest,
|
ActionRequest, ActionResult, AdapterContext, BrowserRequest, CdpPage, CommandRequest,
|
||||||
CommandResult, ComputerRef, EnsureScreenRequest, EnsureScreenResult, FileEntry,
|
CommandResult, ComputerRef, EnsureScreenRequest, EnsureScreenResult, FileEntry,
|
||||||
ProvisionRequest, RecordingRequest, RecordingResult, RecordingSession, SandboxError,
|
ProvisionRequest, RecordingRequest, RecordingResult, RecordingSession, SandboxError,
|
||||||
SandboxProvider, ScreenSession, observation_from_png, observation_with_elements,
|
SandboxProvider, ScreenSession, image_dimensions, observation_from_png,
|
||||||
parse_ui_elements,
|
observation_with_elements, parse_ui_elements,
|
||||||
};
|
};
|
||||||
use reqwest::Client;
|
use reqwest::Client;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
@ -546,8 +549,13 @@ fn decode_observation(body: &Value) -> Result<ComputerObservation, SandboxError>
|
||||||
.get("elements")
|
.get("elements")
|
||||||
.map(|value| parse_ui_elements(&value.to_string()))
|
.map(|value| parse_ui_elements(&value.to_string()))
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
// The control endpoint does not put the mode on the wire, so the frame
|
||||||
|
// itself decides: a desktop that is not 1280x800 must not be reported to
|
||||||
|
// the model as 1280x800.
|
||||||
|
let (width, height) =
|
||||||
|
image_dimensions(&png).unwrap_or((DEFAULT_SCREEN_WIDTH, DEFAULT_SCREEN_HEIGHT));
|
||||||
let mut observation = observation_with_elements(
|
let mut observation = observation_with_elements(
|
||||||
observation_from_png(png, 1280, 800, cursor, window),
|
observation_from_png(png, width, height, cursor, window),
|
||||||
elements,
|
elements,
|
||||||
);
|
);
|
||||||
observation.native_observation_complete = body
|
observation.native_observation_complete = body
|
||||||
|
|
@ -556,3 +564,30 @@ fn decode_observation(body: &Value) -> Result<ComputerObservation, SandboxError>
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
Ok(observation)
|
Ok(observation)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::decode_observation;
|
||||||
|
use lazyboy_contracts::{DEFAULT_SCREEN_HEIGHT, DEFAULT_SCREEN_WIDTH};
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
/// A 3x2 solid-red PNG: small enough to inline, real enough to decode.
|
||||||
|
const FRAME_3X2: &str = "iVBORw0KGgoAAAANSUhEUgAAAAMAAAACCAIAAAASFvFNAAAAEElEQVR4nGP4z8AAQQxwFgBB\
|
||||||
|
0gX7h/C5SAAAAABJRU5ErkJggg==";
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_mode_is_read_from_the_frame() {
|
||||||
|
let observation = decode_observation(&json!({ "png_base64": FRAME_3X2 }))
|
||||||
|
.expect("a decodable frame makes an observation");
|
||||||
|
assert_eq!(observation.width, 3);
|
||||||
|
assert_eq!(observation.height, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_undecodable_frame_falls_back_to_the_default_mode() {
|
||||||
|
let observation =
|
||||||
|
decode_observation(&json!({ "png_base64": "AAAAAAAAAAA=" })).expect("base64 decodes");
|
||||||
|
assert_eq!(observation.width, DEFAULT_SCREEN_WIDTH);
|
||||||
|
assert_eq!(observation.height, DEFAULT_SCREEN_HEIGHT);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -56,11 +56,59 @@ Element refs are snapshot-scoped (`p1:1`, `p4:1`, … `p28:1`). Reusing an old r
|
||||||
|
|
||||||
## Relevant tools (0.23.2 `list-tools`)
|
## Relevant tools (0.23.2 `list-tools`)
|
||||||
|
|
||||||
Observation / native input: `get_desktop_state`, `list_windows`, `get_accessibility_tree`, `get_window_state`, `click`, `type_text`, `press_key`, `hotkey`, `scroll`, `drag`, `get_cursor_position`, `get_screen_size`.
|
Observation / native input used by `controld`: `get_desktop_state`, `list_windows`,
|
||||||
|
`get_window_state`, `click`, `type_text`, `press_key`, `hotkey`, `scroll`, `drag`,
|
||||||
|
`move_cursor`, `set_value`, `bring_to_front`, `get_cursor_position`, `get_screen_size`.
|
||||||
|
|
||||||
Browser (attach only): `browser_prepare` (`strategy.kind=existing_profile`, `allow_launch=false`), `get_browser_state` (`semantic_v2`), `browser_navigate` (http/https/about only), `browser_click`, `browser_type`.
|
Browser (attach only): `browser_prepare` (`strategy.kind=existing_profile`,
|
||||||
|
`allow_launch=false`), `get_browser_state` (`semantic_v2`), `browser_navigate`
|
||||||
|
(http/https/about only), `browser_click`, `browser_type`.
|
||||||
|
|
||||||
Not used here: recording, isolated `launch_app` browsers, Wayland helpers. These names must not be exposed to the LLM.
|
Lifecycle / diagnostics: `start_session`, `end_session`, `health_report`. Skill
|
||||||
|
teaching also uses `start_recording` / `stop_recording`.
|
||||||
|
|
||||||
|
Not used: `mouse_button_down`, `mouse_button_up`, `mouse_drag` (held-button
|
||||||
|
background X11 tools). LazyBoy's action DSL has no partial-pointer state, so
|
||||||
|
`Pointer{Down}` / `Pointer{Up}` translate to `ControlError::Unsupported` instead
|
||||||
|
of a half-pressed button nobody releases. Also unused: isolated `launch_app`
|
||||||
|
browsers, Wayland helpers, and the deprecated `get_session_state` /
|
||||||
|
`escalate_session` aliases. None of these names may be exposed to the LLM.
|
||||||
|
|
||||||
|
## Driver contract rules (enforced in `crates/control/src/cua`)
|
||||||
|
|
||||||
|
Each of these was confirmed against a real 0.23.2 daemon, and each one fails
|
||||||
|
silently (exit 0) if violated:
|
||||||
|
|
||||||
|
1. **Repeat the `session` label on every call.** `CuaClient::call` injects
|
||||||
|
`lazyboy-<display>` unless the caller already set one. Without it each CLI
|
||||||
|
process gets an ephemeral `cli-<uuid>` session, so trajectory turns,
|
||||||
|
snapshots, and browser binds never line up, and every call also emits a bogus
|
||||||
|
`end_session` turn.
|
||||||
|
2. **Never send `target: {kind: "desktop", display_id: "primary"}`.** The Linux
|
||||||
|
driver rejects it with `invalid_action_target` (exit 0). Omit `target` to use
|
||||||
|
the global input route.
|
||||||
|
3. **Desktop `scroll` needs a point.** With `scope: "desktop"`, `x`/`y` are
|
||||||
|
required (`missing field x`); `dispatch` aims at the pointer and falls back to
|
||||||
|
the screen centre. `amount` is clamped to the schema range `1..=50`.
|
||||||
|
4. **Only a JSON object proves the tool ran.** A refusal or prose banner that
|
||||||
|
arrives with exit 0 is a failure (`decode_stdout`), never an empty success.
|
||||||
|
5. **One escalation retry.** `background_unavailable` carries
|
||||||
|
`escalation.recommended`; the client retries once with that `delivery_mode`
|
||||||
|
and never loops.
|
||||||
|
6. **`get_window_state` can be degraded.** AT-SPI intermittently answers with
|
||||||
|
`degraded: true` and a root-only tree. Such windows are skipped and the
|
||||||
|
observation reports `native_observation_complete: false` rather than failing
|
||||||
|
the whole `observe`.
|
||||||
|
7. **Nothing is validated for you.** The Linux schemas declare
|
||||||
|
`additionalProperties: false` and numeric bounds, but 0.23.2 accepts unknown
|
||||||
|
keys and out-of-range values anyway (a bogus key on `list_windows` and
|
||||||
|
`scroll amount: 0` both return exit 0 with `effect: unverifiable`), so the
|
||||||
|
bounds in `docs/cua-schemas/0.23.2/` are enforced here, by the client.
|
||||||
|
`session` is accepted by every tool, including the ones whose own schema
|
||||||
|
omits it (`list_windows`, `bring_to_front`, `health_report`,
|
||||||
|
`start_recording`), which is what lets rule 1 be applied uniformly.
|
||||||
|
8. **Socket peer uid.** `/tmp/lazyboy/cua*.sock` rejects uid 0; `controld` runs as
|
||||||
|
the desktop user (uid 1000).
|
||||||
|
|
||||||
## Integration notes for the next PR
|
## Integration notes for the next PR
|
||||||
|
|
||||||
|
|
@ -94,3 +142,16 @@ for the build architecture (`cua-driver 0.23.2` on linux/arm64 in this run).
|
||||||
`make cua-smoke` is the acceptance entry: raw Driver smoke, adapter E2E,
|
`make cua-smoke` is the acceptance entry: raw Driver smoke, adapter E2E,
|
||||||
isolation, and pause/restart persistence. Production defaults remain `legacy`.
|
isolation, and pause/restart persistence. Production defaults remain `legacy`.
|
||||||
See [the migration audit](cua-review.md).
|
See [the migration audit](cua-review.md).
|
||||||
|
|
||||||
|
One upstream caveat about that persistence claim: Chromium writes its cookie
|
||||||
|
database on a ~30 s timer and does not flush on `SIGTERM`. A profile survives
|
||||||
|
`docker pause` / `docker restart` once that write has landed; stopping a desktop
|
||||||
|
seconds after a login can still lose the cookie, and no LazyBoy code controls
|
||||||
|
the timer. `scripts/cua-smoke-test.sh` waits for the fixture cookie to reach the
|
||||||
|
profile before it restarts, so the check measures profile persistence instead of
|
||||||
|
the flush timer.
|
||||||
|
|
||||||
|
Image architectures are capped at `linux/amd64` and `linux/arm64` by upstream
|
||||||
|
binaries: the Cua Driver ships only `linux-x86_64` / `linux-arm64` and ONNX
|
||||||
|
Runtime only `linux-x64` / `linux-aarch64`. See
|
||||||
|
[development.md](development.md#映像與-cpu-架構).
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,147 @@
|
||||||
|
name: drag
|
||||||
|
|
||||||
|
description:
|
||||||
|
Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels via XSendEvent (ButtonPress + MotionNotify × steps + ButtonRelease). duration_ms (default 500), steps (default 20). No focus steal.
|
||||||
|
|
||||||
|
input_schema:
|
||||||
|
{
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"button": {
|
||||||
|
"description": "Mouse button. Default \"left\".",
|
||||||
|
"enum": [
|
||||||
|
"left",
|
||||||
|
"right",
|
||||||
|
"middle"
|
||||||
|
],
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"cursor_id": {
|
||||||
|
"description": "Optional multi-cursor instance id. Default: 'default'.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"delivery_mode": {
|
||||||
|
"default": "background",
|
||||||
|
"description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.",
|
||||||
|
"enum": [
|
||||||
|
"background",
|
||||||
|
"foreground"
|
||||||
|
],
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"duration_ms": {
|
||||||
|
"description": "Total drag duration. Default: 500.",
|
||||||
|
"maximum": 10000,
|
||||||
|
"minimum": 0,
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"from_x": {
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
"from_y": {
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
"from_zoom": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"modifier": {
|
||||||
|
"description": "Modifier keys held during the action: cmd, shift, option/alt, ctrl.",
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": "array"
|
||||||
|
},
|
||||||
|
"pid": {
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"scope": {
|
||||||
|
"default": "window",
|
||||||
|
"enum": [
|
||||||
|
"window",
|
||||||
|
"desktop"
|
||||||
|
],
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"session": {
|
||||||
|
"description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"steps": {
|
||||||
|
"description": "Intermediate MotionNotify events. Default: 20.",
|
||||||
|
"maximum": 200,
|
||||||
|
"minimum": 1,
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"target": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.",
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"additionalProperties": true,
|
||||||
|
"properties": {
|
||||||
|
"kind": {
|
||||||
|
"const": "window",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"pid": {
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0,
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"window_id": {
|
||||||
|
"format": "uint64",
|
||||||
|
"minimum": 0,
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"kind",
|
||||||
|
"pid",
|
||||||
|
"window_id"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"additionalProperties": true,
|
||||||
|
"properties": {
|
||||||
|
"display_id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"kind": {
|
||||||
|
"const": "desktop",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"kind",
|
||||||
|
"display_id"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"to_x": {
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
"to_y": {
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
"window_id": {
|
||||||
|
"description": "Target window XID. Required.",
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"from_x",
|
||||||
|
"from_y",
|
||||||
|
"to_x",
|
||||||
|
"to_y"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
name: end_session
|
||||||
|
|
||||||
|
description:
|
||||||
|
End one visible lifecycle session and run its cursor, recording, configuration, and other cleanup hooks exactly once. Omit `session` to end the authenticated transport's implicit session. Idempotent.
|
||||||
|
|
||||||
|
input_schema:
|
||||||
|
{
|
||||||
|
"additionalProperties": true,
|
||||||
|
"properties": {
|
||||||
|
"session": {
|
||||||
|
"description": "Optional public label to end. When omitted, end the caller's attached\nimplicit session.",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [],
|
||||||
|
"type": "object"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,50 @@
|
||||||
|
name: start_session
|
||||||
|
|
||||||
|
description:
|
||||||
|
Optionally create or return a lifecycle session before acting. For multi-call work, prefer a short public `session` label and repeat it on every call that accepts it; an omitted value uses the authenticated transport lease's implicit session instead. This tool is optional because an ordinary action can create or reuse a named run directly. Use it to set the initial cursor theme before acting or to revive a public name after it has ended; ordinary actions never revive ended names. `capture_scope` is deprecated compatibility input; new callers select window or desktop modality per action. Idempotent.
|
||||||
|
|
||||||
|
input_schema:
|
||||||
|
{
|
||||||
|
"additionalProperties": true,
|
||||||
|
"properties": {
|
||||||
|
"capture_scope": {
|
||||||
|
"description": "Deprecated compatibility policy. New callers select window or desktop\nmodality on each action instead of storing it on the session.",
|
||||||
|
"enum": [
|
||||||
|
"auto",
|
||||||
|
"window",
|
||||||
|
"desktop"
|
||||||
|
],
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"cursor_theme": {
|
||||||
|
"description": "Optional initial cursor theme. The host applies it before the cursor is\nfirst made visible, avoiding a flash of the default theme.",
|
||||||
|
"properties": {
|
||||||
|
"reduced_motion": {
|
||||||
|
"default": "auto",
|
||||||
|
"enum": [
|
||||||
|
"auto",
|
||||||
|
"on",
|
||||||
|
"off"
|
||||||
|
],
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"theme_id": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"theme_id"
|
||||||
|
],
|
||||||
|
"type": [
|
||||||
|
"object",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"session": {
|
||||||
|
"description": "Optional stable public label for this run (e.g. \"research-run-1\").\nWhen omitted, the authenticated transport lease's implicit session is\ncreated or returned.",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [],
|
||||||
|
"type": "object"
|
||||||
|
}
|
||||||
|
|
@ -87,6 +87,30 @@ Lint 政策集中在三處,新增 crate 時只要補 `[lints] workspace = true
|
||||||
依賴來源。它不是內建工具,第一次要先 `cargo install --locked cargo-deny`。
|
依賴來源。它不是內建工具,第一次要先 `cargo install --locked cargo-deny`。
|
||||||
已知無法升級的項目會寫進 `deny.toml` 的 `ignore` 並附原因與重檢時機。
|
已知無法升級的項目會寫進 `deny.toml` 的 `ignore` 並附原因與重檢時機。
|
||||||
|
|
||||||
|
### 映像與 CPU 架構
|
||||||
|
|
||||||
|
三個映像(`image/computer`、`image/api`、`image/supervisor`)一律用 `scripts/build-image.sh`
|
||||||
|
建置。支援的架構是 **linux/amd64 與 linux/arm64**,這個上限由上游二進位發行決定,不是實作取捨:
|
||||||
|
|
||||||
|
- Cua Driver `cua-driver-rs-v0.23.2` 的 Linux 發行檔只有 `linux-x86_64` 與 `linux-arm64`。
|
||||||
|
- ONNX Runtime `v1.24.1` 的 Linux 發行檔只有 `linux-x64` 與 `linux-aarch64`。
|
||||||
|
|
||||||
|
`image/computer/Dockerfile` 依 `TARGETARCH` 選檔並驗證 SHA256,`scripts/fetch-onnxruntime.sh`
|
||||||
|
同樣只認這兩種架構,其他架構直接建置失敗;`build-image.sh` 在註冊 QEMU 與開始跨修編
|
||||||
|
之前就先擋掉不支援的 `--platforms`,不會燒掉數 GB 才報錯。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make computer # 本機架構,結果直接進本地 image 庫
|
||||||
|
make computer-multi # amd64 + arm64 -> dist/lazyboy-computer-multi.oci.tar
|
||||||
|
make images-multi # 三個映像都跨修 amd64 + arm64
|
||||||
|
make images-multi PUSH=1 # 直接發布 manifest list 到 registry
|
||||||
|
```
|
||||||
|
|
||||||
|
多架構需要 `docker-container` builder 與 QEMU binfmt,`build-image.sh` 會自行建立 `lazyboy`
|
||||||
|
builder 並註冊 binfmt;主機本身的架構會跳過註冊,否則在特權容器內會誤報失敗。
|
||||||
|
不推送 registry 時輸出 OCI archive(`nerdctl load -i <檔>` 可載入),因為 Docker 的
|
||||||
|
`docker` driver 無法匯出 manifest list。
|
||||||
|
|
||||||
### 專案結構
|
### 專案結構
|
||||||
|
|
||||||
```text
|
```text
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@
|
||||||
FROM rust:1-bookworm AS controld
|
FROM rust:1-bookworm AS controld
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY Cargo.toml Cargo.lock ./
|
COPY Cargo.toml Cargo.lock ./
|
||||||
|
COPY .cargo .cargo
|
||||||
COPY crates crates
|
COPY crates crates
|
||||||
COPY migrations migrations
|
COPY migrations migrations
|
||||||
RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
|
RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
|
||||||
|
|
@ -16,8 +17,14 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
|
||||||
|
|
||||||
FROM debian:bookworm-slim
|
FROM debian:bookworm-slim
|
||||||
|
|
||||||
# Keep Chromium, XFCE, CJK, AT-SPI, git. Skip Debian novnc (pulls nodejs),
|
# Keep Chromium, XFCE, CJK, AT-SPI, git. Skip Debian novnc (pulls nodejs) and
|
||||||
# fonts-noto-core (Latin is DejaVu/Liberation/huninn), and unused xterm.
|
# fonts-noto-core (Latin is DejaVu/Liberation/huninn).
|
||||||
|
#
|
||||||
|
# x11-apps + imagemagick look removable and are not: the legacy driver's
|
||||||
|
# screenshot path is `xwd -root -silent | convert xwd:- … jpeg:-`
|
||||||
|
# (control/src/x11.rs screenshot_command_on), reached by legacy observe_display
|
||||||
|
# and by the supervisor's exec fallback. Nothing else in the image provides
|
||||||
|
# /usr/bin/xwd or /usr/bin/convert, so removing either breaks the default driver.
|
||||||
RUN printf '%s\n' \
|
RUN printf '%s\n' \
|
||||||
'path-include=/usr/share/locale/zh_TW/*' \
|
'path-include=/usr/share/locale/zh_TW/*' \
|
||||||
'path-include=/usr/share/locale/zh/*' \
|
'path-include=/usr/share/locale/zh/*' \
|
||||||
|
|
@ -33,7 +40,6 @@ RUN printf '%s\n' \
|
||||||
fonts-noto-color-emoji \
|
fonts-noto-color-emoji \
|
||||||
git \
|
git \
|
||||||
htop \
|
htop \
|
||||||
imagemagick \
|
|
||||||
locales \
|
locales \
|
||||||
procps \
|
procps \
|
||||||
python3 \
|
python3 \
|
||||||
|
|
@ -49,6 +55,7 @@ RUN printf '%s\n' \
|
||||||
x11vnc \
|
x11vnc \
|
||||||
xclip \
|
xclip \
|
||||||
xdg-utils \
|
xdg-utils \
|
||||||
|
imagemagick \
|
||||||
xfce4-panel \
|
xfce4-panel \
|
||||||
xfce4-settings \
|
xfce4-settings \
|
||||||
xfce4-terminal \
|
xfce4-terminal \
|
||||||
|
|
@ -73,7 +80,10 @@ RUN printf '%s\n' \
|
||||||
&& echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \
|
&& echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \
|
||||||
&& locale-gen \
|
&& locale-gen \
|
||||||
&& mkdir -p /usr/share/novnc \
|
&& mkdir -p /usr/share/novnc \
|
||||||
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* /usr/share/doc/* /usr/share/man/*
|
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* /usr/share/doc/* /usr/share/man/* \
|
||||||
|
/usr/share/i18n /usr/share/info /usr/share/gtk-doc /usr/share/backgrounds \
|
||||||
|
&& find /usr -name __pycache__ -type d -prune -exec rm -rf '{}' + \
|
||||||
|
&& find /usr -name '*.py[co]' -delete
|
||||||
|
|
||||||
# jf open 粉圓 (system UI) + MesloLGS NF (Powerlevel10k glyphs, CJK via fontconfig).
|
# jf open 粉圓 (system UI) + MesloLGS NF (Powerlevel10k glyphs, CJK via fontconfig).
|
||||||
RUN mkdir -p /usr/share/fonts/truetype/huninn /usr/share/fonts/truetype/meslo \
|
RUN mkdir -p /usr/share/fonts/truetype/huninn /usr/share/fonts/truetype/meslo \
|
||||||
|
|
@ -102,6 +112,27 @@ RUN useradd --create-home --uid 1000 --shell /bin/zsh lazyboy \
|
||||||
&& mkdir -p /home/lazyboy /tmp/lazyboy /usr/share/lazyboy/skel /usr/share/lazyboy/xfce-skel /etc/gtk-3.0 /etc/fonts/conf.d \
|
&& mkdir -p /home/lazyboy /tmp/lazyboy /usr/share/lazyboy/skel /usr/share/lazyboy/xfce-skel /etc/gtk-3.0 /etc/fonts/conf.d \
|
||||||
&& chown -R 1000:1000 /home/lazyboy /tmp/lazyboy
|
&& chown -R 1000:1000 /home/lazyboy /tmp/lazyboy
|
||||||
|
|
||||||
|
# Pin Cua Driver outside the persisted /home/lazyboy bind-mount.
|
||||||
|
# Checksums are from the matching official release; select the target architecture.
|
||||||
|
ARG CUA_DRIVER_RS_VERSION=0.23.2
|
||||||
|
ARG TARGETARCH
|
||||||
|
ARG CUA_DRIVER_RS_SHA256_AMD64=01bf8339ec129cc00f4b4b2c6056ef1a7c5b52df39ff83ad17c9b16818aec500
|
||||||
|
ARG CUA_DRIVER_RS_SHA256_ARM64=be22768a207796a4bc1de50c52f32f9ef680b5e86e58c059e02eec2caba2e7bb
|
||||||
|
RUN case "$TARGETARCH" in \
|
||||||
|
amd64) cua_arch=x86_64; cua_sha="$CUA_DRIVER_RS_SHA256_AMD64" ;; \
|
||||||
|
arm64) cua_arch=arm64; cua_sha="$CUA_DRIVER_RS_SHA256_ARM64" ;; \
|
||||||
|
*) echo "Unsupported Cua architecture: $TARGETARCH" >&2; exit 1 ;; \
|
||||||
|
esac \
|
||||||
|
&& curl -fsSL -o /tmp/cua-driver.tar.gz \
|
||||||
|
"https://github.com/trycua/cua/releases/download/cua-driver-rs-v${CUA_DRIVER_RS_VERSION}/cua-driver-rs-${CUA_DRIVER_RS_VERSION}-linux-${cua_arch}-binary.tar.gz" \
|
||||||
|
&& echo "${cua_sha} /tmp/cua-driver.tar.gz" | sha256sum -c \
|
||||||
|
&& mkdir -p /usr/local/lib/cua-driver \
|
||||||
|
&& tar -xzf /tmp/cua-driver.tar.gz -C /usr/local/lib/cua-driver \
|
||||||
|
&& chmod 755 /usr/local/lib/cua-driver/cua-driver \
|
||||||
|
&& ln -sf /usr/local/lib/cua-driver/cua-driver /usr/local/bin/cua-driver \
|
||||||
|
&& rm -f /tmp/cua-driver.tar.gz \
|
||||||
|
&& cua-driver --version
|
||||||
|
|
||||||
COPY --from=controld --chmod=755 /lazyboy-controld /usr/local/bin/lazyboy-controld
|
COPY --from=controld --chmod=755 /lazyboy-controld /usr/local/bin/lazyboy-controld
|
||||||
COPY --chmod=755 image/computer/rotate-logs.py /usr/local/bin/lazyboy-rotate-logs
|
COPY --chmod=755 image/computer/rotate-logs.py /usr/local/bin/lazyboy-rotate-logs
|
||||||
COPY --chmod=755 image/computer/lazyboy-screen /usr/local/bin/lazyboy-screen
|
COPY --chmod=755 image/computer/lazyboy-screen /usr/local/bin/lazyboy-screen
|
||||||
|
|
@ -140,27 +171,6 @@ COPY --chmod=644 image/computer/xfce/helper-browser.desktop /usr/share/xfce4/hel
|
||||||
COPY --chmod=755 image/computer/chromium /usr/local/bin/chromium
|
COPY --chmod=755 image/computer/chromium /usr/local/bin/chromium
|
||||||
RUN sed -i 's|^Exec=/usr/bin/chromium|Exec=/usr/local/bin/lazyboy-browser|' /usr/share/applications/chromium.desktop || true
|
RUN sed -i 's|^Exec=/usr/bin/chromium|Exec=/usr/local/bin/lazyboy-browser|' /usr/share/applications/chromium.desktop || true
|
||||||
|
|
||||||
# Pin Cua Driver outside the persisted /home/lazyboy bind-mount.
|
|
||||||
# Checksums are from the matching official release; select the target architecture.
|
|
||||||
ARG CUA_DRIVER_RS_VERSION=0.23.2
|
|
||||||
ARG TARGETARCH
|
|
||||||
ARG CUA_DRIVER_RS_SHA256_AMD64=01bf8339ec129cc00f4b4b2c6056ef1a7c5b52df39ff83ad17c9b16818aec500
|
|
||||||
ARG CUA_DRIVER_RS_SHA256_ARM64=be22768a207796a4bc1de50c52f32f9ef680b5e86e58c059e02eec2caba2e7bb
|
|
||||||
RUN case "$TARGETARCH" in \
|
|
||||||
amd64) cua_arch=x86_64; cua_sha="$CUA_DRIVER_RS_SHA256_AMD64" ;; \
|
|
||||||
arm64) cua_arch=arm64; cua_sha="$CUA_DRIVER_RS_SHA256_ARM64" ;; \
|
|
||||||
*) echo "Unsupported Cua architecture: $TARGETARCH" >&2; exit 1 ;; \
|
|
||||||
esac \
|
|
||||||
&& curl -fsSL -o /tmp/cua-driver.tar.gz \
|
|
||||||
"https://github.com/trycua/cua/releases/download/cua-driver-rs-v${CUA_DRIVER_RS_VERSION}/cua-driver-rs-${CUA_DRIVER_RS_VERSION}-linux-${cua_arch}-binary.tar.gz" \
|
|
||||||
&& echo "${cua_sha} /tmp/cua-driver.tar.gz" | sha256sum -c \
|
|
||||||
&& mkdir -p /usr/local/lib/cua-driver \
|
|
||||||
&& tar -xzf /tmp/cua-driver.tar.gz -C /usr/local/lib/cua-driver \
|
|
||||||
&& chmod 755 /usr/local/lib/cua-driver/cua-driver \
|
|
||||||
&& ln -sf /usr/local/lib/cua-driver/cua-driver /usr/local/bin/cua-driver \
|
|
||||||
&& rm -f /tmp/cua-driver.tar.gz \
|
|
||||||
&& cua-driver --version
|
|
||||||
|
|
||||||
COPY --chmod=755 image/computer/start.sh /usr/local/bin/lazyboy-computer
|
COPY --chmod=755 image/computer/start.sh /usr/local/bin/lazyboy-computer
|
||||||
COPY --chmod=755 image/computer/entrypoint.sh /usr/local/bin/lazyboy-entrypoint
|
COPY --chmod=755 image/computer/entrypoint.sh /usr/local/bin/lazyboy-entrypoint
|
||||||
COPY --chmod=644 image/computer/cua-smoke.html /usr/share/lazyboy/cua-smoke.html
|
COPY --chmod=644 image/computer/cua-smoke.html /usr/share/lazyboy/cua-smoke.html
|
||||||
|
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
[begin] (桌面)
|
|
||||||
[exec] (終端機) {/usr/local/bin/lazyboy-terminal}
|
|
||||||
[exec] (瀏覽器) {/usr/local/bin/lazyboy-browser https://duckduckgo.com}
|
|
||||||
[end]
|
|
||||||
|
|
@ -1,92 +0,0 @@
|
||||||
! LazyBoy Fluxbox style — flat dark, Traditional Chinese UI.
|
|
||||||
! Debian ships Fluxbox 1.3.5 (last stable). This is a custom skin, not a fork.
|
|
||||||
toolbar: flat solid
|
|
||||||
toolbar.color: #0b1220
|
|
||||||
toolbar.borderColor: #1e293b
|
|
||||||
toolbar.borderWidth: 0
|
|
||||||
toolbar.height: 36
|
|
||||||
toolbar.justify: center
|
|
||||||
toolbar.clock: flat solid
|
|
||||||
toolbar.clock.color: #0b1220
|
|
||||||
toolbar.clock.textColor: #cbd5e1
|
|
||||||
toolbar.clock.font: jf open 粉圓 2.1-11
|
|
||||||
toolbar.workspace: flat solid
|
|
||||||
toolbar.workspace.color: #0b1220
|
|
||||||
toolbar.workspace.textColor: #7dd3fc
|
|
||||||
toolbar.workspace.font: jf open 粉圓 2.1-11
|
|
||||||
toolbar.iconbar.empty: flat solid
|
|
||||||
toolbar.iconbar.empty.color: #0b1220
|
|
||||||
toolbar.iconbar.focused: flat solid
|
|
||||||
toolbar.iconbar.focused.color: #1d4ed8
|
|
||||||
toolbar.iconbar.focused.textColor: #f8fafc
|
|
||||||
toolbar.iconbar.focused.font: jf open 粉圓 2.1-11
|
|
||||||
toolbar.iconbar.unfocused: flat solid
|
|
||||||
toolbar.iconbar.unfocused.color: #162032
|
|
||||||
toolbar.iconbar.unfocused.textColor: #94a3b8
|
|
||||||
toolbar.iconbar.unfocused.font: jf open 粉圓 2.1-11
|
|
||||||
toolbar.button: flat solid
|
|
||||||
toolbar.button.color: #0b1220
|
|
||||||
toolbar.button.picColor: #7dd3fc
|
|
||||||
toolbar.button.pressed.color: #1e3a8a
|
|
||||||
|
|
||||||
menu: flat solid
|
|
||||||
menu.borderColor: #1e293b
|
|
||||||
menu.borderWidth: 1
|
|
||||||
menu.bullet: triangle
|
|
||||||
menu.bullet.position: right
|
|
||||||
menu.title: flat solid
|
|
||||||
menu.title.color: #1e3a8a
|
|
||||||
menu.title.textColor: #f8fafc
|
|
||||||
menu.title.font: jf open 粉圓 2.1-12
|
|
||||||
menu.title.justify: center
|
|
||||||
menu.frame: flat solid
|
|
||||||
menu.frame.color: #0f172a
|
|
||||||
menu.frame.textColor: #e2e8f0
|
|
||||||
menu.frame.disableColor: #64748b
|
|
||||||
menu.frame.font: jf open 粉圓 2.1-12
|
|
||||||
menu.hilight: flat solid
|
|
||||||
menu.hilight.color: #2563eb
|
|
||||||
menu.hilight.textColor: #ffffff
|
|
||||||
|
|
||||||
window.title.height: 22
|
|
||||||
window.justify: left
|
|
||||||
window.borderWidth: 1
|
|
||||||
window.borderColor: #334155
|
|
||||||
window.handleWidth: 3
|
|
||||||
window.bevelWidth: 0
|
|
||||||
window.roundCorners: none
|
|
||||||
window.font: jf open 粉圓 2.1-11
|
|
||||||
window.title.focus: flat solid
|
|
||||||
window.title.focus.color: #1e293b
|
|
||||||
window.title.unfocus: flat solid
|
|
||||||
window.title.unfocus.color: #0f172a
|
|
||||||
window.label.focus: flat solid
|
|
||||||
window.label.focus.color: #1e293b
|
|
||||||
window.label.focus.textColor: #f8fafc
|
|
||||||
window.label.unfocus: flat solid
|
|
||||||
window.label.unfocus.color: #0f172a
|
|
||||||
window.label.unfocus.textColor: #64748b
|
|
||||||
window.handle.focus: flat solid
|
|
||||||
window.handle.focus.color: #1e293b
|
|
||||||
window.handle.unfocus: flat solid
|
|
||||||
window.handle.unfocus.color: #0f172a
|
|
||||||
window.grip.focus: flat solid
|
|
||||||
window.grip.focus.color: #2563eb
|
|
||||||
window.grip.unfocus: flat solid
|
|
||||||
window.grip.unfocus.color: #1e293b
|
|
||||||
window.button.focus: flat solid
|
|
||||||
window.button.focus.color: #1e293b
|
|
||||||
window.button.focus.picColor: #e2e8f0
|
|
||||||
window.button.unfocus: flat solid
|
|
||||||
window.button.unfocus.color: #0f172a
|
|
||||||
window.button.unfocus.picColor: #64748b
|
|
||||||
window.button.pressed.color: #2563eb
|
|
||||||
window.close.pressed.color: #dc2626
|
|
||||||
|
|
||||||
*font: jf open 粉圓 2.1-11
|
|
||||||
borderWidth: 1
|
|
||||||
bevelWidth: 0
|
|
||||||
handleWidth: 3
|
|
||||||
borderColor: #334155
|
|
||||||
background: solid
|
|
||||||
background.color: #0f172a
|
|
||||||
|
|
@ -9,7 +9,9 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
|
||||||
--mount=type=cache,target=/src/target,id=lazyboy-supervisor-release,sharing=locked \
|
--mount=type=cache,target=/src/target,id=lazyboy-supervisor-release,sharing=locked \
|
||||||
cargo build --locked --release -p lazyboy-supervisor && cp /src/target/release/lazyboy-supervisor /lazyboy-supervisor
|
cargo build --locked --release -p lazyboy-supervisor && cp /src/target/release/lazyboy-supervisor /lazyboy-supervisor
|
||||||
|
|
||||||
FROM gcr.io/distroless/cc-debian12:latest
|
# Pin the multi-arch (amd64 + arm64) index by digest: `latest` is mutable and a
|
||||||
|
# floating base silently changes the supervisor's libc on every rebuild.
|
||||||
|
FROM gcr.io/distroless/cc-debian12@sha256:e5d81ddde149641e2a9ba55be4545bc125c67de07508b03ba4c22e6eb0ded5aa
|
||||||
COPY --from=build /lazyboy-supervisor /usr/local/bin/lazyboy-supervisor
|
COPY --from=build /lazyboy-supervisor /usr/local/bin/lazyboy-supervisor
|
||||||
EXPOSE 7091
|
EXPOSE 7091
|
||||||
CMD ["/usr/local/bin/lazyboy-supervisor"]
|
CMD ["/usr/local/bin/lazyboy-supervisor"]
|
||||||
|
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
root="$(cd "$(dirname "$0")/.." && pwd)"
|
|
||||||
cd "$root"
|
|
||||||
docker build -f image/computer/Dockerfile -t lazyboy/computer:local .
|
|
||||||
echo "built lazyboy/computer:local"
|
|
||||||
|
|
@ -0,0 +1,136 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Build one LazyBoy image. Single entry point for everything in image/.
|
||||||
|
#
|
||||||
|
# scripts/build-image.sh # desktop image, host arch, into docker
|
||||||
|
# scripts/build-image.sh --file image/api/Dockerfile # any image, host arch
|
||||||
|
# scripts/build-image.sh --platforms linux/arm64 # one foreign arch, needs QEMU
|
||||||
|
# scripts/build-image.sh --multi # amd64 + arm64 -> OCI archive
|
||||||
|
# scripts/build-image.sh --multi --push # amd64 + arm64 -> registry
|
||||||
|
#
|
||||||
|
# Supported arches: linux/amd64 and linux/arm64 -- see the ceiling note below.
|
||||||
|
# --multi/--platforms cannot use the default "docker" builder: that driver
|
||||||
|
# cannot export a manifest list. A docker-container builder plus QEMU binfmt is
|
||||||
|
# bootstrapped here.
|
||||||
|
set -euo pipefail
|
||||||
|
root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
cd "$root"
|
||||||
|
|
||||||
|
file="image/computer/Dockerfile"
|
||||||
|
platforms=""
|
||||||
|
multi=0
|
||||||
|
push=0
|
||||||
|
target=""
|
||||||
|
oci_name=""
|
||||||
|
builder="${BUILDX_BUILDER:-lazyboy}"
|
||||||
|
tag=""
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--file) file="$2"; shift 2 ;;
|
||||||
|
--tag) tag="$2"; shift 2 ;;
|
||||||
|
--target) target="$2"; shift 2 ;;
|
||||||
|
--platforms) platforms="$2"; shift 2 ;;
|
||||||
|
--oci) oci_name="$2"; shift 2 ;;
|
||||||
|
--builder) builder="$2"; shift 2 ;;
|
||||||
|
--multi) multi=1; shift ;;
|
||||||
|
--push) push=1; shift ;;
|
||||||
|
-h|--help) sed -n '2,13p' "$0"; exit 0 ;;
|
||||||
|
*) echo "unknown option: $1" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# Defaults follow docker-compose.yml: the desktop image is published under a
|
||||||
|
# lazyboy/ namespace, the services under the compose project name.
|
||||||
|
if [ -z "$tag" ]; then
|
||||||
|
case "$file" in
|
||||||
|
image/computer/Dockerfile) tag="lazyboy/computer:local" ;;
|
||||||
|
image/supervisor/Dockerfile) tag="lazyboy-supervisor:latest" ;;
|
||||||
|
image/api/Dockerfile) tag="lazyboy-api:latest" ;;
|
||||||
|
*) echo "--tag is required for $file" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
if [ "$multi" = 1 ] && [ -z "$platforms" ]; then
|
||||||
|
platforms="linux/amd64,linux/arm64"
|
||||||
|
fi
|
||||||
|
if [ -z "$oci_name" ]; then
|
||||||
|
case "$file" in
|
||||||
|
image/computer/Dockerfile) oci_name="dist/lazyboy-computer-multi.oci.tar" ;;
|
||||||
|
*) oci_name="dist/$(basename "$(dirname "$file")")-multi.oci.tar" ;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$push" = 1 ] && [ -z "$platforms" ]; then
|
||||||
|
echo "--push only makes sense with --multi or --platforms" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Architecture ceiling, set by two upstream binary distributions: the Cua Driver
|
||||||
|
# (cua-driver-rs-v0.23.2) ships Linux builds for x86_64 and arm64 only, and
|
||||||
|
# ONNX Runtime (v1.24.1) for x64 and aarch64 only. image/computer/Dockerfile and
|
||||||
|
# scripts/fetch-onnxruntime.sh fail fast on anything else; checking here rejects
|
||||||
|
# a bad --platforms before binfmt + GBs of cross-compilation are spent.
|
||||||
|
supported_arches="amd64 arm64"
|
||||||
|
for platform in ${platforms//,/ }; do
|
||||||
|
os="${platform%%/*}"
|
||||||
|
arch="${platform#*/}"
|
||||||
|
arch="${arch%%/*}"
|
||||||
|
if [ "$os" != "linux" ]; then
|
||||||
|
echo "--platforms: only Linux images are built (got $platform)" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
case " $supported_arches " in
|
||||||
|
*" $arch "*) ;;
|
||||||
|
*)
|
||||||
|
echo "--platforms: unsupported architecture '$arch' in '$platform'." >&2
|
||||||
|
echo " LazyBoy images exist for linux/amd64 and linux/arm64: the Cua" >&2
|
||||||
|
echo " Driver ships only linux-x86_64/linux-arm64 and ONNX Runtime only" >&2
|
||||||
|
echo " linux-x64/linux-aarch64, so no other Linux arch can run them." >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -z "$platforms" ]; then
|
||||||
|
# Native build: the docker driver, so the result lands in the local store and
|
||||||
|
# `docker compose up` picks it up without a registry.
|
||||||
|
args=(docker build -f "$file" -t "$tag")
|
||||||
|
[ -n "$target" ] && args+=(--target "$target")
|
||||||
|
"${args[@]}" .
|
||||||
|
echo "built $tag"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker buildx inspect "$builder" >/dev/null 2>&1 \
|
||||||
|
|| docker buildx create --name "$builder" --driver docker-container --bootstrap
|
||||||
|
# QEMU is only needed for architectures the host cannot run natively; asking
|
||||||
|
# binfmt to register the host's own handler fails inside a privileged container
|
||||||
|
# and would look like a real failure, so the native arch is skipped.
|
||||||
|
native="$(dpkg --print-architecture 2>/dev/null || true)"
|
||||||
|
if [ -z "$native" ]; then
|
||||||
|
case "$(uname -m)" in
|
||||||
|
x86_64) native=amd64 ;;
|
||||||
|
aarch64) native=arm64 ;;
|
||||||
|
*) native=unknown ;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
for platform in ${platforms//,/ }; do
|
||||||
|
arch="${platform#*/}"
|
||||||
|
if [ "$arch" != "$native" ]; then
|
||||||
|
docker run --privileged --rm tonistiigi/binfmt --install "$arch" >/dev/null 2>&1 \
|
||||||
|
|| echo "WARNING: no QEMU handler for $arch; that build fails with 'exec format error'" >&2
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
args=(docker buildx build --builder "$builder" -f "$file" -t "$tag" --platform "$platforms")
|
||||||
|
[ -n "$target" ] && args+=(--target "$target")
|
||||||
|
if [ "$push" = 1 ]; then
|
||||||
|
args+=(--push)
|
||||||
|
else
|
||||||
|
mkdir -p "$(dirname "$oci_name")"
|
||||||
|
args+=(--output "type=oci,dest=$oci_name")
|
||||||
|
fi
|
||||||
|
"${args[@]}" .
|
||||||
|
if [ "$push" = 1 ]; then
|
||||||
|
echo "published $tag for ${platforms}"
|
||||||
|
else
|
||||||
|
echo "wrote $oci_name for ${platforms} (load with: nerdctl load -i $oci_name)"
|
||||||
|
fi
|
||||||
|
|
@ -100,6 +100,18 @@ if [[ "$code" -eq 0 ]]; then
|
||||||
code=$?
|
code=$?
|
||||||
fi
|
fi
|
||||||
if [[ "$code" -eq 0 ]]; then
|
if [[ "$code" -eq 0 ]]; then
|
||||||
|
# Chromium writes its cookie database on a ~30 s timer and does not flush on
|
||||||
|
# SIGTERM, so restarting right away races that timer: the check below would
|
||||||
|
# then measure flush timing instead of profile persistence. Wait until the
|
||||||
|
# fixture cookie is on disk before restarting.
|
||||||
|
echo "waiting for the browser to persist its cookie"
|
||||||
|
for _ in $(seq 1 90); do
|
||||||
|
if docker exec -u 1000:1000 "$name" sh -c \
|
||||||
|
'grep -qa cuaSmoke "$HOME"/.browser-profiles/displays/*/Default/Cookies 2>/dev/null'; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
docker restart "$name" >/dev/null
|
docker restart "$name" >/dev/null
|
||||||
code=$?
|
code=$?
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,13 @@ import fs from 'node:fs';
|
||||||
import vm from 'node:vm';
|
import vm from 'node:vm';
|
||||||
import ts from '../apps/web/node_modules/typescript/lib/typescript.js';
|
import ts from '../apps/web/node_modules/typescript/lib/typescript.js';
|
||||||
function loadCatalog(file){
|
function loadCatalog(file){
|
||||||
const code=ts.transpileModule(fs.readFileSync(file,'utf8'),{compilerOptions:{module:ts.ModuleKind.CommonJS}}).outputText;
|
const source=fs.readFileSync(file,'utf8');
|
||||||
|
// transpileModule prints output even for a catalog that does not parse, so a
|
||||||
|
// missing comma silently loads a shorter catalog and every key check below
|
||||||
|
// still passes. Ask the parser, which actually keeps its errors.
|
||||||
|
const parsed=ts.createSourceFile(file,source,ts.ScriptTarget.ESNext,true,ts.ScriptKind.TS);
|
||||||
|
assert.deepEqual((parsed.parseDiagnostics||[]).map(d=>ts.flattenDiagnosticMessageText(d.messageText,' ')),[],`${file} must parse`);
|
||||||
|
const code=ts.transpileModule(source,{compilerOptions:{module:ts.ModuleKind.CommonJS}}).outputText;
|
||||||
const sandbox={exports:{},require:()=>({})};
|
const sandbox={exports:{},require:()=>({})};
|
||||||
vm.runInNewContext(code,sandbox);
|
vm.runInNewContext(code,sandbox);
|
||||||
return sandbox.exports.zhTW||sandbox.exports.en;
|
return sandbox.exports.zhTW||sandbox.exports.en;
|
||||||
|
|
@ -274,16 +280,14 @@ test('mobile shortcut row sends modifiers and hides with the keyboard',()=>{
|
||||||
assert.equal(v.keyboard.focused,false);
|
assert.equal(v.keyboard.focused,false);
|
||||||
assert.equal(v.elements['keyboard-shortcuts'].hidden,true);
|
assert.equal(v.elements['keyboard-shortcuts'].hidden,true);
|
||||||
});
|
});
|
||||||
test('status and HUD sit outside the remote pixels; screenshots stay opt-in',()=>{
|
test('status sits outside the remote pixels; screenshots stay opt-in',()=>{
|
||||||
const app=fs.readFileSync('apps/web/src/App.tsx','utf8');
|
const app=fs.readFileSync('apps/web/src/App.tsx','utf8');
|
||||||
assert.doesNotMatch(app,/RunStatus/);
|
assert.doesNotMatch(app,/RunStatus/);
|
||||||
assert.match(app,/<RunProbe runId=\{computer\.busyRunId\|\|computer\.waitingRunId\}><Avatar/);
|
assert.match(app,/<RunProbe runId=\{computer\.busyRunId\|\|computer\.waitingRunId\}><Avatar/);
|
||||||
assert.match(app,/\{!computerOpen&&hud\}<div className="preview">/);
|
|
||||||
assert.match(app,/--visible-height/);
|
assert.match(app,/--visible-height/);
|
||||||
assert.match(app,/<div className=\{`composer-dock \$\{statusMembers\.length\?"has-status":""\}`\}>\s*\{error&&<div className="error-banner"/);
|
assert.match(app,/<div className=\{`composer-dock \$\{statusMembers\.length\?"has-status":""\}`\}>\s*\{error&&<div className="error-banner"/);
|
||||||
const css=fs.readFileSync('apps/web/src/computer.css','utf8');
|
const css=fs.readFileSync('apps/web/src/computer.css','utf8');
|
||||||
assert.doesNotMatch(css,/\.run-status/);
|
assert.doesNotMatch(css,/\.run-status/);
|
||||||
assert.match(css,/\.computer-hud\{position:static/);
|
|
||||||
const chat=fs.readFileSync('apps/web/src/chat.css','utf8');
|
const chat=fs.readFileSync('apps/web/src/chat.css','utf8');
|
||||||
assert.match(chat,/\.composer-dock\{position:relative/);
|
assert.match(chat,/\.composer-dock\{position:relative/);
|
||||||
assert.doesNotMatch(chat,/\.composer-dock\{position:absolute/);
|
assert.doesNotMatch(chat,/\.composer-dock\{position:absolute/);
|
||||||
|
|
@ -447,3 +451,162 @@ test('chat follows the event stream instead of a fixed two second poll',()=>{
|
||||||
assert.doesNotMatch(app,/const timer=setInterval\(\(\)=>\{refresh\(\)/,'the 2s transcript poll should be gone');
|
assert.doesNotMatch(app,/const timer=setInterval\(\(\)=>\{refresh\(\)/,'the 2s transcript poll should be gone');
|
||||||
assert.match(app,/const heartbeat=window\.setInterval\(\(\)=>\{const beat=roomsRef\.current[\s\S]*\},HEARTBEAT_MS\)/,'the heartbeat keeps its own minute cadence');
|
assert.match(app,/const heartbeat=window\.setInterval\(\(\)=>\{const beat=roomsRef\.current[\s\S]*\},HEARTBEAT_MS\)/,'the heartbeat keeps its own minute cadence');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// The screen veil and the frame that survives it: both used to be driven by a
|
||||||
|
// timer plus "null the url", which is what made booting and handing over feel
|
||||||
|
// like a stall instead of a gesture.
|
||||||
|
const handoffJs=ts.transpileModule(fs.readFileSync('apps/web/src/handoff.ts','utf8'),{compilerOptions:{module:ts.ModuleKind.CommonJS}}).outputText;
|
||||||
|
const handoffBox={exports:{},require:()=>({})};vm.runInNewContext(handoffJs,handoffBox);
|
||||||
|
const {viewerPath,keepScreenUrl,handoffRemaining,nextVeil,viewOnlyFor,HANDOFF_MS,HANDOFF_MIN_MS,VEIL_FADE_MS}=handoffBox.exports;
|
||||||
|
|
||||||
|
test('the viewer path is stable, so a frame can mount before the desktop answers',()=>{
|
||||||
|
assert.equal(viewerPath('bot 7'),'/view/bot%207/vnc.html');
|
||||||
|
assert.equal(viewerPath('a/b'),'/view/a%2Fb/vnc.html');
|
||||||
|
assert.equal(viewerPath('bot'),viewerPath('bot'),'no nonce, or the frame would remount every poll');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('the path the frame mounts on is the byte the api hands back',()=>{
|
||||||
|
// The viewer mounts on the client's own path while the desktop is coming up,
|
||||||
|
// then adopts the url from the status round trip. If the two ever differ the
|
||||||
|
// iframe remounts mid-boot, which is the black flash this whole path exists
|
||||||
|
// to avoid, so the invariant is pinned to the server's own format string.
|
||||||
|
const routes=fs.readFileSync('crates/api/src/routes.rs','utf8');
|
||||||
|
const served=(routes.match(/"url": format!\("([^"]+)"\)/)||[])[1];
|
||||||
|
assert.equal(served,'/view/{id}/vnc.html');
|
||||||
|
const id='6f1d3a0e-2b7c-4d5e-8f90-1a2b3c4d5e6f';
|
||||||
|
assert.equal(viewerPath(id),served.replace('{id}',id));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('pixels are dropped only for a computer that is really gone',()=>{
|
||||||
|
const url='/view/bot/vnc.html';
|
||||||
|
for(const state of ['booting','suspended','running'])assert.equal(keepScreenUrl(url,null,state),url,state+' must keep its VNC session');
|
||||||
|
for(const state of ['stopped','error'])assert.equal(keepScreenUrl(url,null,state),null,state+' has no desktop to keep');
|
||||||
|
assert.equal(keepScreenUrl(null,null,'running'),null,'nothing was ever mounted');
|
||||||
|
assert.equal(keepScreenUrl(url,'/other','booting'),'/other','a fresh url still wins');
|
||||||
|
assert.equal(keepScreenUrl(null,url,'booting'),url,'the first url arrives while booting');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a handoff holds one visible beat, never the whole round trip',()=>{
|
||||||
|
assert.ok(HANDOFF_MIN_MS<HANDOFF_MS,'the beat needs a floor and a ceiling');
|
||||||
|
assert.ok(VEIL_FADE_MS<=HANDOFF_MIN_MS,'the fade has to finish inside the beat');
|
||||||
|
assert.ok(HANDOFF_MS<=1_000,'a reply that never lands cannot strand the mascot');
|
||||||
|
assert.equal(handoffRemaining(1_000,1_000),HANDOFF_MIN_MS,'a reply this instant still owes the beat');
|
||||||
|
assert.equal(handoffRemaining(1_000,1_000+HANDOFF_MIN_MS),0,'served');
|
||||||
|
assert.equal(handoffRemaining(1_000,1_000+(HANDOFF_MIN_MS+HANDOFF_MS)/2),0,'past the floor is never a wait');
|
||||||
|
assert.equal(handoffRemaining(1_000,1_000+HANDOFF_MS),0);
|
||||||
|
const midway=handoffRemaining(1_000,1_000+HANDOFF_MIN_MS-10);
|
||||||
|
assert.equal(midway,10,'the remaining beat shrinks with the clock');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('the veil fades in with its label and back out through the same mascot',()=>{
|
||||||
|
const empty={label:null,leaving:false};
|
||||||
|
assert.equal(nextVeil(null,empty),empty,'with nothing to say there is nothing to show');
|
||||||
|
const shown=nextVeil('換手中',empty);
|
||||||
|
assert.equal(shown.label,'換手中');assert.equal(shown.leaving,false);
|
||||||
|
assert.equal(nextVeil('換手中',shown),shown,'a repeated status must not restart the animation');
|
||||||
|
const leaving=nextVeil(null,shown);
|
||||||
|
assert.equal(leaving.label,'換手中','the label is held while it fades');assert.equal(leaving.leaving,true);
|
||||||
|
assert.equal(nextVeil(null,leaving),leaving,'it leaves exactly once');
|
||||||
|
const replaced=nextVeil('喚醒中',shown);
|
||||||
|
assert.equal(replaced.label,'喚醒中');assert.equal(replaced.leaving,false,'a new status replaces the label');
|
||||||
|
const revived=nextVeil('喚醒中',leaving);
|
||||||
|
assert.equal(revived.label,'喚醒中');assert.equal(revived.leaving,false,'and pulls it back out of the fade');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('only a human moves the mouse, and the veil never holds it back',()=>{
|
||||||
|
assert.equal(viewOnlyFor('user'),false);
|
||||||
|
for(const holder of ['none','bot'])assert.equal(viewOnlyFor(holder),true,holder);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('taking the screen flips the mouse on the click, not on the reply',()=>{
|
||||||
|
const app=fs.readFileSync('apps/web/src/App.tsx','utf8');
|
||||||
|
const body=app.slice(app.indexOf('async function setControl('));
|
||||||
|
const flip=body.indexOf('controlHolder:holder');
|
||||||
|
const gate=body.indexOf('pushViewOnly(viewOnlyFor(holder))');
|
||||||
|
const post=body.indexOf('await api(`/api/computer/${botId}/${holder==="user"?"takeover":"release"}`');
|
||||||
|
const readBack=body.indexOf('finally{');
|
||||||
|
assert.ok(flip>0&&gate>0&&post>0&&readBack>0,'optimistic flip, gate, request, and read back all present');
|
||||||
|
assert.ok(flip<post&&gate<post,'badge and input move before the server answers');
|
||||||
|
assert.ok(readBack>post,'the truth is read back after the request');
|
||||||
|
assert.match(body.slice(readBack),/await refresh\(\)/);
|
||||||
|
assert.match(body,/if\(!botId\|\|controlBusyRef\.current\)return/,'a double click cannot fight itself');
|
||||||
|
assert.match(app,/const listener=\(event:MessageEvent\)=>\{[\s\S]*?lazyboy-request-control[\s\S]*?void setControl\("user"\)/);
|
||||||
|
assert.match(app,/if\(expectedHolderRef\.current&&status\.controlHolder===expectedHolderRef\.current\)/,'agreement, not a timer, ends the handoff');
|
||||||
|
assert.match(app,/onClick=\{onTakeOver\}/);
|
||||||
|
assert.match(app,/onClick=\{onRelease\}/);
|
||||||
|
assert.doesNotMatch(app,/action\(\(\)=>api\(`\/api\/computer\/[^`]*takeover/,'takeover no longer rides the global busy path');
|
||||||
|
assert.equal((app.match(/\/api\/computer\/\$\{[^}]*\}\/(takeover|release)/g)||[]).length,0,'every handoff goes through setControl');
|
||||||
|
assert.equal((app.match(/holder==="user"\?"takeover":"release"/g)||[]).length,1,'one request path, one owner of it');
|
||||||
|
assert.match(app,/void setControl\("user",active\.id\)/,'the call overlay hands over the same way');
|
||||||
|
assert.match(app,/await setControl\("user",id\)/,'the login screen boots, then takes the mouse without a second spinner');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('the waiting veil covers the desktop the way it always did',()=>{
|
||||||
|
const app=fs.readFileSync('apps/web/src/App.tsx','utf8');
|
||||||
|
assert.match(app,/<div className="preview">\{computerOpen\?<EmptyComputer state=\{computer\.state\}\/>:frame\}\{!computerOpen&&hud\}<\/div>/);
|
||||||
|
assert.match(app,/<div className="overlay-desktop">\{frame\}\{hud\}<\/div>/);
|
||||||
|
assert.match(app,/const hud=paneBot&&veil\.label\?<ComputerHud bot=\{paneBot\} label=\{veil\.label\} leaving=\{veil\.leaving\}\/>:null/);
|
||||||
|
const css=fs.readFileSync('apps/web/src/computer.css','utf8');
|
||||||
|
// Both parents are what make inset:0 cover the desktop rather than the page.
|
||||||
|
assert.match(css,/\.computer-part \.preview\{position:relative/);
|
||||||
|
assert.match(css,/\.overlay-screen \.overlay-desktop\{position:relative/);
|
||||||
|
// These values were compared against main in a real browser: rendered pixel
|
||||||
|
// for pixel identical, so drift here means the veil visibly changed.
|
||||||
|
const hud=css.match(/^\.computer-hud\{([^}]*)\}/m)[1];
|
||||||
|
for(const rule of ['position:absolute','inset:0','z-index:3','display:grid','align-content:center','justify-items:center','gap:14px','padding:10px 12px 8px','border-radius:inherit','background:radial-gradient(ellipse at center,#142722e8,#101012ed)','backdrop-filter:blur(8px)','pointer-events:none'])assert.ok(hud.includes(rule),`the veil needs ${rule}`);
|
||||||
|
assert.match(css,/\.computer-hud\.is-leaving\{opacity:0\}/);
|
||||||
|
assert.match(css,/\.computer-hud\{[^}]*transition:opacity \.22s ease\}/,'it fades instead of blinking off');
|
||||||
|
// main drew the label as static mint text; the shimmer belongs to the chat
|
||||||
|
// working label, and the veil keeps the mascot as the only moving part.
|
||||||
|
const label=css.match(/^\.computer-hud-label\{([^}]*)\}/m)[1];
|
||||||
|
assert.ok(!label.includes('animation'),'the veil label does not animate');
|
||||||
|
assert.ok(label.includes('color:#c9ddd5')&&label.includes('font-size:12px')&&label.includes('text-align:center')&&label.includes('max-width:90%'));
|
||||||
|
assert.ok(!/\.computer-hud[^{]*\{[^}]*working-shimmer/.test(css),'nothing shimmers inside the veil');
|
||||||
|
assert.match(css,/\.computer-signal\{[^}]*width:64px[^}]*animation:monitor-breathe/);
|
||||||
|
assert.match(css,/\.computer-signal::before\{[^}]*monitor-orbit/);
|
||||||
|
const responsive=fs.readFileSync('apps/web/src/responsive.css','utf8');
|
||||||
|
assert.match(responsive,/prefers-reduced-motion:reduce\)\{\.computer-signal,\.computer-signal::before,\.computer-signal-face i[^}]*animation:none/,'reduced motion still has to still the mascot');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a desktop that is still starting is dialled again fast, a dropped session calmly',()=>{
|
||||||
|
const source=fs.readFileSync('apps/web/vnc.html','utf8').match(/<script type="module">([\s\S]*?)<\/script>/)[1].replace(/import RFB[^;]+;/,'');
|
||||||
|
// Any element the viewer touches answers, so the disconnect path runs the
|
||||||
|
// same way it does in a browser without a DOM to stand in.
|
||||||
|
const element=()=>new Proxy({style:{},classList:{toggle(){}},dataset:{},hidden:false,textContent:''},{get:(node,key)=>key in node?node[key]:()=>undefined,set:(node,key,value)=>{node[key]=value;return true}});
|
||||||
|
const instances=[],scheduled=[];
|
||||||
|
class RFB{constructor(){this._handlers={};instances.push(this._handlers);}
|
||||||
|
addEventListener(name,handler){(this._handlers[name]||(this._handlers[name]=[])).push(handler);} focus(){} blur(){} sendKey(){}}
|
||||||
|
const window={location:{pathname:'/vnc.html',protocol:'http:',host:'localhost',origin:'http://localhost',hash:''},parent:{postMessage(){}},addEventListener(){}};
|
||||||
|
vm.runInNewContext(source,{window,document:{location:{href:'http://localhost/vnc.html?view_only=false'},getElementById:()=>element(),querySelector:()=>null},navigator:{clipboard:{}},RFB,setTimeout:(fn,ms)=>{scheduled.push(ms);return scheduled.length},clearTimeout(){}});
|
||||||
|
const fire=(name,event)=>{for(const handlers of instances)(handlers[name]||[]).forEach(handler=>handler(event||{}));};
|
||||||
|
assert.equal(instances.length,1,'the viewer dials once on load');
|
||||||
|
fire('disconnect',{clean:false});
|
||||||
|
assert.equal(scheduled.at(-1),350,'a desktop that has never answered is retried eagerly');
|
||||||
|
fire('connect');
|
||||||
|
fire('disconnect',{clean:true});
|
||||||
|
assert.equal(scheduled.at(-1),1500,'a session that was really there is retried calmly');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a session dialed after a handoff starts with the mouse already handed over',()=>{
|
||||||
|
const vnc=fs.readFileSync('apps/web/vnc.html','utf8');
|
||||||
|
// Run the viewer rather than read it. The host hands the mouse over while the
|
||||||
|
// viewer sits between two sessions: exactly the moment a fresh RFB used to
|
||||||
|
// fall back to its view-only default and leave a taken-over desktop deaf with
|
||||||
|
// the veil already lifted. Only the gate message arriving is real here.
|
||||||
|
const body=vnc.match(/<script type="module">([\s\S]*?)<\/script>/)[1].replace(/import RFB[^;]+;/,'');
|
||||||
|
const element=()=>new Proxy({style:{},classList:{toggle(){}},dataset:{},hidden:false,textContent:''},{get:(node,key)=>key in node?node[key]:()=>undefined,set:(node,key,value)=>{node[key]=value;return true}});
|
||||||
|
const instances=[],scheduled=[],messages=[];
|
||||||
|
class RFB{constructor(){this._handlers={};instances.push(this);}addEventListener(name,handler){(this._handlers[name]||(this._handlers[name]=[])).push(handler);}focus(){this.focused=true;}blur(){}}
|
||||||
|
const parent={postMessage(){}};
|
||||||
|
const window={location:{pathname:'/vnc.html',protocol:'http:',host:'localhost',origin:'http://localhost',hash:''},parent,addEventListener(name,handler){if(name==='message')messages.push(handler);}};
|
||||||
|
vm.runInNewContext(body,{window,document:{location:{href:'http://localhost/vnc.html'},getElementById:()=>element(),querySelector:()=>null},navigator:{clipboard:{}},RFB,setTimeout:(fn)=>{scheduled.push(fn);return scheduled.length},clearTimeout(){}});
|
||||||
|
const fire=(name,event)=>instances.forEach(rfb=>(rfb._handlers[name]||[]).forEach(handler=>handler(event||{})));
|
||||||
|
fire('disconnect',{clean:false});
|
||||||
|
assert.ok(messages.length,'the viewer listens for the host gate');
|
||||||
|
messages.forEach(handler=>handler({origin:'http://localhost',source:parent,data:{type:'lazyboy-view-only',viewOnly:false}}));
|
||||||
|
scheduled.at(-1)();
|
||||||
|
fire('connect');
|
||||||
|
assert.equal(instances.length,2,'the retry dialed a new session');
|
||||||
|
assert.equal(instances.at(-1).viewOnly,false,'a handoff made mid-reconnect still owns the new session');
|
||||||
|
assert.equal(instances.at(-1).focused,true,'taking over lands the first keystroke');
|
||||||
|
});
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue