From 80fe9a8ef19bc637f96cb11aad9aae508abbadda Mon Sep 17 00:00:00 2001 From: daniel wang Date: Thu, 10 Sep 2026 14:42:17 +0000 Subject: [PATCH] fix frontend css issue --- .env.example | 11 + Cargo.lock | 3 + Makefile | 67 + apps/web/src/App.tsx | 47 +- apps/web/src/locales/en.ts | 19 +- apps/web/src/locales/zh-TW.ts | 10 +- apps/web/src/refinements.css | 6 + apps/web/src/run-monitor.tsx | 11 + apps/web/src/types.ts | 3 +- crates/api/Cargo.toml | 4 + crates/api/src/artifacts.rs | 182 +++ crates/api/src/computer.rs | 372 ++++- crates/api/src/db.rs | 5 +- crates/api/src/job_store.rs | 10 + crates/api/src/main.rs | 4 + crates/api/src/mcp.rs | 178 ++- crates/api/src/memory.rs | 44 + crates/api/src/monitor.rs | 28 +- crates/api/src/operations.rs | 473 +++++++ crates/api/src/retention.rs | 10 + .../api/src/retention/computer_operations.sql | 4 + crates/api/src/retention/operation_outbox.sql | 4 + crates/api/src/routes.rs | 31 +- crates/api/src/runs.rs | 139 +- crates/api/src/skills.rs | 2 +- crates/api/src/tool_install.rs | 786 +++++++++++ crates/api/src/tools.rs | 1113 ++++++++++++++- crates/contracts/src/action.rs | 4 + crates/contracts/src/computer.rs | 12 + crates/contracts/src/lib.rs | 2 + crates/contracts/src/tool.rs | 141 ++ crates/control/Cargo.toml | 1 + crates/control/src/a11y.rs | 1 + crates/control/src/actions.rs | 1 + crates/control/src/browser_page.rs | 10 + crates/control/src/capability.rs | 87 ++ crates/control/src/controller.rs | 6 + crates/control/src/cua/browser.rs | 257 +++- crates/control/src/cua/mod.rs | 2 + crates/control/src/cua/native.rs | 1 + crates/control/src/display_backend.rs | 110 ++ crates/control/src/file_cas.rs | 48 + crates/control/src/form.rs | 94 ++ crates/control/src/gmail.rs | 236 ++++ crates/control/src/jobs.rs | 729 ++++++++++ crates/control/src/lib.rs | 35 +- crates/control/src/mcp_policy.rs | 92 ++ crates/control/src/observe.rs | 154 ++- crates/control/src/operation.rs | 173 +++ crates/control/src/outlook.rs | 94 ++ crates/control/src/pause.rs | 66 + crates/control/src/readiness.rs | 88 ++ crates/control/src/sandbox.rs | 29 +- crates/control/src/secrets.rs | 129 ++ crates/control/src/tool_manager.rs | 389 ++++++ crates/control/src/verifier.rs | 62 + crates/control/src/wait.rs | 42 + crates/control/src/x11.rs | 4 + crates/harness/src/policy.rs | 37 +- crates/sandbox/Cargo.toml | 3 + crates/sandbox/src/docker.rs | 95 +- crates/sandbox/src/fake.rs | 332 ++++- crates/supervisor/src/docker.rs | 211 ++- crates/supervisor/src/main.rs | 39 +- docs/agent-computer-progress.md | 361 +++++ ...Y_AGENT_COMPUTER_IMPLEMENTATION_PLAN_V3.md | 1227 +++++++++++++++++ .../LAZYBOY_CODING_AGENT_START_HERE_V3.md | 34 + .../plan/LAZYBOY_OPTIMIZATION_CHECKLIST_V3.md | 86 ++ image/computer/Dockerfile | 1 + image/computer/lazyboy-screen | 65 +- image/computer/start.sh | 7 + migrations/022_agent_computer.sql | 84 ++ migrations/023_agent_computer_fixes.sql | 17 + scripts/sample-mcp/echo_server.py | 54 + scripts/sample-mcp/manifest.yaml | 23 + 75 files changed, 9063 insertions(+), 278 deletions(-) create mode 100644 crates/api/src/artifacts.rs create mode 100644 crates/api/src/job_store.rs create mode 100644 crates/api/src/operations.rs create mode 100644 crates/api/src/retention/computer_operations.sql create mode 100644 crates/api/src/retention/operation_outbox.sql create mode 100644 crates/api/src/tool_install.rs create mode 100644 crates/contracts/src/tool.rs create mode 100644 crates/control/src/capability.rs create mode 100644 crates/control/src/display_backend.rs create mode 100644 crates/control/src/file_cas.rs create mode 100644 crates/control/src/form.rs create mode 100644 crates/control/src/gmail.rs create mode 100644 crates/control/src/jobs.rs create mode 100644 crates/control/src/mcp_policy.rs create mode 100644 crates/control/src/operation.rs create mode 100644 crates/control/src/outlook.rs create mode 100644 crates/control/src/pause.rs create mode 100644 crates/control/src/readiness.rs create mode 100644 crates/control/src/secrets.rs create mode 100644 crates/control/src/tool_manager.rs create mode 100644 crates/control/src/verifier.rs create mode 100644 crates/control/src/wait.rs create mode 100644 docs/agent-computer-progress.md create mode 100644 docs/plan/LAZYBOY_AGENT_COMPUTER_IMPLEMENTATION_PLAN_V3.md create mode 100644 docs/plan/LAZYBOY_CODING_AGENT_START_HERE_V3.md create mode 100644 docs/plan/LAZYBOY_OPTIMIZATION_CHECKLIST_V3.md create mode 100644 migrations/022_agent_computer.sql create mode 100644 migrations/023_agent_computer_fixes.sql create mode 100644 scripts/sample-mcp/echo_server.py create mode 100644 scripts/sample-mcp/manifest.yaml diff --git a/.env.example b/.env.example index d9c019a..4e960a2 100644 --- a/.env.example +++ b/.env.example @@ -67,3 +67,14 @@ LAZYBOY_DB_WARN_MB=1024 # Docker 網路名稱：API 與 Agent 電腦的 noVNC 透過它相通（容器內用，不對外）。 # 同時跑多組 LazyBoy 時改這個名字避免相撞。 LAZYBOY_SCREEN_NETWORK=lazyboy_screen + +# Executor path version (not a ComputerMode). legacy = GUI-first; hybrid = native tools. +LAZYBOY_EXECUTION_PROFILE=hybrid +# xvfb_x11vnc is the default. tigervnc_xvnc is a candidate backend (PR-09); keep rollback. +LAZYBOY_DISPLAY_BACKEND=xvfb_x11vnc +LAZYBOY_BROWSER_BACKEND=cua +LAZYBOY_TOOL_AUDIT_REQUIRED=true +LAZYBOY_NATIVE_JOB_CONCURRENCY=2 +LAZYBOY_SERVICE_READ_CONCURRENCY=4 +LAZYBOY_TOOL_INSTALL_ENABLED=true +LAZYBOY_TOOL_INSTALL_REQUIRE_APPROVAL=true diff --git a/Cargo.lock b/Cargo.lock index 6d1057d..a35b6c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1905,6 +1905,7 @@ dependencies = [ "lazyboy-control", "lazyboy-harness", "lazyboy-sandbox", + "ort", "rand 0.8.8", "reqwest 0.12.28", "rig-core", @@ -1942,6 +1943,7 @@ dependencies = [ "hex", "image", "lazyboy-contracts", + "regex", "serde", "serde_json", "sha2", @@ -1991,6 +1993,7 @@ dependencies = [ "lazyboy-control", "reqwest 0.12.28", "serde_json", + "tokio", ] [[package]] diff --git a/Makefile b/Makefile index f743246..a184267 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,7 @@ help: ## Show this help @echo " make lint The Rust gate: clippy with -D warnings" @echo " make audit cargo deny: RustSec advisories, licenses, sources" @echo " make test cargo test --workspace (DB tests need: make postgres)" + @echo " make test-agent-computer Screenshot / locator / native-file contract tests" @echo " make web Build the frontend in $(WEB_DIR) (needs node/npm)" @echo " make clean cargo clean" @echo "" @@ -192,6 +193,72 @@ audit: ## Supply-chain check (RustSec advisories, licenses, dependency sources) test: ## Run the test suite cargo test --workspace +test-agent-computer: ## Agent-computer contract tests (screenshot, locators, native files, jobs, connectors) + cargo test --workspace --lib -- \ + should_deliver_observation_image \ + classify_browser_locator \ + native_file_payload \ + file_bytes_prefer_base64 \ + file_list_refuses \ + fake_sandbox_keeps_invalid_utf8 \ + pixel_actions_need_vision \ + typed_browser_errors \ + chat_tools_do_not_need_the_desktop \ + plan_batch_modify \ + classify_graph_item \ + validate_manifest \ + zip_slip \ + pausing_agent \ + same_operation \ + journal_failure \ + cancel_stops \ + gmail_with_grant \ + policy_denied \ + slot_zero \ + wait_counts \ + canary_is_stripped \ + stale_hash \ + team_computers_share \ + short_command_returns \ + form_fill_and_computer_mcp \ + form_locator_resolves \ + begin_then_complete \ + running_count_and_quota \ + operation_id_is_not_part \ + team_packages_live \ + catalog_stdio_is_not_an_api_child \ + native_exec_does_not_need_desktop \ + revoke_drops_computer_mcp \ + update_pins_old_version \ + t59_keeps_ac \ + native_tools_do_not_boot \ + mcp_rs_does_not_spawn \ + agent_computer_migration_does_not_unique \ + native_exec_and_files_do_not_call_ensure_screen \ + update_switches_the_single_ready_binding \ + start_sh_skips_desktop \ + background_exec_runs_on_the_computer \ + computer_background_launch \ + background_launch_keeps_posix_quoted \ + exec_background_does_not_spawn \ + reap_background_job_failed \ + command_result_json_keeps \ + supervisor_exec_returns_full + cargo test -p lazyboy-api -- \ + native_file_payload \ + pixel_actions_need_vision \ + typed_browser_errors \ + native_exec_is_in_the_schema \ + chat_tools_do_not_need_the_desktop \ + form_fill_and_computer_mcp \ + form_locator_resolves \ + operation_id_is_not_part \ + team_packages_live \ + mcp_rs_does_not_spawn \ + native_tools_do_not_boot \ + agent_computer_migration_does_not_unique \ + exec_background_does_not_spawn + web: ## Build the frontend (needs node/npm) cd $(WEB_DIR) && npm install && npm run build diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index 273ad92..4f82c39 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -22,7 +22,7 @@ import { clockTime, dayLabel, sameDay } from "./chat-time"; import { HANDOFF_MS, VEIL_FADE_MS, handoffRemaining, keepScreenUrl, nextVeil, viewOnlyFor, viewerPath, type Veil } from "./handoff"; import { Avatar, AvatarLookProvider, AvatarStack, BLOBATAR_BACKGROUNDS, BLOBATAR_EXPRESSIONS, BLOBATAR_SHAPES, DEFAULT_LOOK, persistBlobatarShape, readAvatarLooks, resolveBlobatarShape, writeAvatarLook, type AvatarBackground, type AvatarExpression, type AvatarLook } from "./avatar"; import { dateLocale, getLocale, listJoin, setLocale, t, useLocale, type MessageKey } from "./i18n"; -import type { AvatarShape, Bot, ComputerMode, ComputerStatus, FileSkill, McpCatalogEntry, McpServer, McpTransport, MemoryItem, MemoryStatus, Message, MessageFile, Playbook, PlaybookInput, PlaybookStep, Room, RoomMember, Session, TaughtSkill, VoiceSettings } from "./types"; +import type { AvatarShape, Bot, ComputerMode, ComputerStatus, ComputerToolBinding, FileSkill, McpCatalogEntry, McpServer, McpTransport, MemoryItem, MemoryStatus, Message, MessageFile, Playbook, PlaybookInput, PlaybookStep, Room, RoomMember, Session, TaughtSkill, VoiceSettings } from "./types"; import { ChatMarkdown, MentionText, copyText } from "./markdown"; import { RunProbe, errorActions, errorTitle } from "./run-monitor"; import { ScheduleEditor, ScheduleList, cronFromPreset, defaultCronPreset, presetFromCron, scheduleWhen, type CronPreset, type ScheduleItem } from "./schedule"; @@ -680,7 +680,7 @@ export function App(){ } - {rightPart==="plugins"&&} + {rightPart==="plugins"&&} {rightPart==="settings"&&active&&{writeAvatarLook(active.id,look);setLooks(readAvatarLooks())}} saved={loadBots} onDelete={()=>setDeleteOpen(true)}/>} @@ -847,11 +847,12 @@ function MemoryPane({bot,changed}:{bot:Bot;changed:()=>Promise}){ } function parsePairs(text:string){const out:Record={};for(const line of text.split(/\n+/)){const trimmed=line.trim();if(!trimmed)continue;const cut=trimmed.indexOf("=");if(cut<=0)continue;out[trimmed.slice(0,cut).trim()]=trimmed.slice(cut+1)}return out} -function McpPane({servers,reload}:{servers:McpServer[];reload:()=>Promise}){ +function McpPane({bot,servers,reload}:{bot?:Bot;servers:McpServer[];reload:()=>Promise}){ const[picker,setPicker]=useState(false);const[busy,setBusy]=useState(false);const[error,setError]=useState("");const[openId,setOpenId]=useState(null); async function run(work:()=>Promise){setBusy(true);setError("");try{await work();await reload()}catch(e){setError(e instanceof Error?localizeError(e.message):t("operationFailed"))}finally{setBusy(false)}} return

{t("mcpHelp")}

+ {picker&&server.name)} close={()=>setPicker(false)} connected={async()=>{setPicker(false);await reload()}}/>}
{servers.length===0?

{t("noMcp")}

:servers.map(server=>
@@ -873,6 +874,46 @@ function McpPane({servers,reload}:{servers:McpServer[];reload:()=>Promise}
} +function ComputerToolsPane({bot}:{bot?:Bot}){ + const[tools,setTools]=useState([]); + const[busy,setBusy]=useState(false); + const[error,setError]=useState(""); + const[echoText,setEchoText]=useState("hello"); + const[echoOut,setEchoOut]=useState(""); + const load=useCallback(async()=>{ + if(!bot){setTools([]);return} + const result=await api<{tools:ComputerToolBinding[]}>(`/api/bots/${bot.id}/tools`).catch(()=>({tools:[] as ComputerToolBinding[]})); + setTools(result.tools||[]); + },[bot]); + useEffect(()=>{void load()},[load]); + async function run(work:()=>Promise){setBusy(true);setError("");try{await work();await load()}catch(e){setError(e instanceof Error?localizeError(e.message):t("operationFailed"))}finally{setBusy(false)}} + return
+ {t("computerToolsTitle")} +

{t("computerToolsHelp")}

+ {!bot?

{t("chooseBotForTools")}

:<> +
+ + +
+
{tools.length===0?

{t("noComputerTools")}

:tools.map(tool=>
+
+ {tool.packageId} + {tool.ready?t("computerToolsReady"):tool.bindingStatus==="revoked"?t("computerToolsRevoked"):tool.bindingStatus} · {t("toolOnComputer")} +
+ {tool.ready&&
+ setEchoText(e.target.value)} placeholder={t("echoPlaceholder")} disabled={busy}/> + + + + +
} +
)}
+ {echoOut&&
{t("echoCallResult")}: {echoOut}
} + } + {error&&
{error}
} +
+} + function McpPickerDialog({added,close,connected}:{added:string[];close:()=>void;connected:()=>Promise}){ const[query,setQuery]=useState("");const[custom,setCustom]=useState(false); const[items,setItems]=useState([]);const[loading,setLoading]=useState(true); diff --git a/apps/web/src/locales/en.ts b/apps/web/src/locales/en.ts index 73b8910..078e8ac 100644 --- a/apps/web/src/locales/en.ts +++ b/apps/web/src/locales/en.ts @@ -248,7 +248,22 @@ export const en: { [K in keyof typeof zhTW]: string } = { clearAllMemoryConfirm: "Clear all memory?", clearMemoryConfirm: "Clear", clearAll: "Clear all", - mcpHelp: "Press “Choose MCP” to connect from the catalog. If a key is required, fill it in first. Local stdio servers start inside the API container.", + mcpHelp: "Catalog HTTP/SSE MCP is a remote workspace connection. Local stdio is not started in the API process — reviewed packages run on the assigned Computer.", + computerToolsTitle: "Computer packages", + computerToolsHelp: "Reviewed packages are written onto this Computer. Team mode stores them under shared/tools. The sample echo server is the one installable package today.", + installSampleEcho: "Install echo sample", + updateSampleEcho: "Update to 0.0.2", + rollbackPackage: "Roll back version", + removePackage: "Remove package", + noComputerTools: "No packages on this Computer yet.", + toolOnComputer: "Runs on the assigned Computer", + revokeBinding: "Revoke binding", + callSampleEcho: "Call echo", + echoPlaceholder: "Text to echo", + chooseBotForTools: "Select a bot to install packages on its Computer.", + computerToolsReady: "Ready", + computerToolsRevoked: "Revoked", + echoCallResult: "Result", mcpNamePlaceholder: "For example: github", mcpConnectFailed: "Couldn’t connect. Check the key and try again.", command: "Command", @@ -359,7 +374,7 @@ export const en: { [K in keyof typeof zhTW]: string } = { helpMemoryTitle: "Memory", helpMemory: "Clearing a chat doesn’t wipe long-term memory. Ask the agent to remember, or add it in Memory on the right.", helpMcpTitle: "MCP plugins", - helpMcp: "Plugins in the lower left connect MCP servers. Their tools show up on screen and the agent can use them in chat.", + helpMcp: "Plugins in the lower left connect catalog MCP servers (workspace-level). The same pane can install a reviewed package onto this Computer; that process runs in the container.", helpSkillsTitle: "Skills", helpSkills: "+ → Teach a task, demonstrate once, and it becomes a skill. Export JSON after, or import someone else’s file onto another bot.", helpAttachTitle: "Attachments", diff --git a/apps/web/src/locales/zh-TW.ts b/apps/web/src/locales/zh-TW.ts index 76cc487..f479b8c 100644 --- a/apps/web/src/locales/zh-TW.ts +++ b/apps/web/src/locales/zh-TW.ts @@ -88,7 +88,13 @@ export const zhTW = { memoryHelp: "清除對話不會刪這些。只存明確偏好或事實，不要存密碼。", enableLongTermMemory: "啟用長期記憶", addMemory: "新增記憶", memoryPlaceholder: "只儲存明確偏好或事實；密碼與 token 會被拒絕。", memorySearch: "搜尋", filterMemory: "過濾記憶", noMemory: "還沒有長期記憶。對話裡講過的偏好，可以叫 Agent 記住，或你在這裡新增。", noMatchingMemory: "沒有符合的記憶。", save: "儲存", edit: "編輯", clearAllMemoryConfirm: "確定清除全部記憶？", clearMemoryConfirm: "確定清除", clearAll: "全部清除", - mcpHelp: "點「選擇 MCP」從市集接入。標了需要金鑰的，先填 token 才能連。本機 stdio 會在 API 容器裡啟動。", mcpNamePlaceholder: "例如：github", mcpConnectFailed: "接入失敗，請檢查金鑰或稍後重試。", + mcpHelp: "市集 HTTP/SSE MCP 是遠端工作區連線。本機 stdio 不會在 API 行程啟動；審核過的套件在指派的 Computer 裡執行。", mcpNamePlaceholder: "例如：github", mcpConnectFailed: "接入失敗，請檢查金鑰或稍後重試。", + computerToolsTitle: "Computer 套件", computerToolsHelp: "審核過的套件寫進這台 Computer。Team 模式放在 shared/tools。目前可裝範例 echo。", + installSampleEcho: "安裝 echo 範例", updateSampleEcho: "更新到 0.0.2", rollbackPackage: "回滾版本", removePackage: "移除套件", + noComputerTools: "這台 Computer 還沒有安裝套件。", + toolOnComputer: "在指派的 Computer 執行", revokeBinding: "撤銷綁定", callSampleEcho: "呼叫 echo", + echoPlaceholder: "要回傳的文字", chooseBotForTools: "選一個機器人後，才能在它的 Computer 上安裝套件。", + computerToolsReady: "可用", computerToolsRevoked: "已撤銷", echoCallResult: "回傳", command: "Command", commandPlaceholder: "npx 或 uvx 或完整路徑", arguments: "參數", environmentVariables: "環境變數 KEY=value", headers: "Headers KEY=value", connecting: "連線中…", connectMcp: "接入 MCP", chooseMcp: "選擇 MCP", mcpPickerTitle: "選擇 MCP", mcpPickerLead: "從精選或官方市集挑一個，點下去接入。所有 Agent 都能用它的工具。", searchMcp: "搜尋 MCP", mcpRemote: "遠端", mcpLocal: "本機", mcpAdded: "已接入", mcpNeedsKey: "需要金鑰", @@ -123,7 +129,7 @@ export const zhTW = { meetingMode: "會議模式", exitMeetingMode: "結束會議模式", helpMeetingTitle: "會議模式", helpMeeting: "電腦版標題列可開啟會議模式：螢幕放大放中間，對話移到旁邊，像分享畫面時邊看邊聊。手機沒有這個模式。", helpMemoryTitle: "記憶", helpMemory: "清除對話不會刪長期記憶。可以叫 Agent 記住，或在右側「記憶」手動新增。", - helpMcpTitle: "MCP 外掛", helpMcp: "左下「外掛程式」接入 MCP server。連上的工具會顯示在畫面上，對話時 Agent 可以使用。", + helpMcpTitle: "MCP 外掛", helpMcp: "左下「外掛程式」接入市集 MCP（工作區連線）。同一頁也可把審核過的套件裝到這台 Computer 上，行程在容器裡跑。", helpSkillsTitle: "技能", helpSkills: "＋ → 教它一項任務，示範一次就會整理成技能。示範結束後可以匯出 JSON，或把別人的技能檔匯入，換一個機器人也適用。", helpAttachTitle: "附件", helpAttach: "＋ → 附加檔案。圖片這則訊息就會給模型看，不會存進對話紀錄。若機器人電腦要打開原檔，會暫放 inbox/，兩小時後自動刪，避免把磁碟塞滿。", helpShortcutsTitle: "快捷鍵", helpShortcuts: "Enter 送出，Shift+Enter 換行。正在回覆時送出鈕會變成停止。通話中空白鍵插話，Esc 掛斷。", diff --git a/apps/web/src/refinements.css b/apps/web/src/refinements.css index e798a5c..cd2dd51 100644 --- a/apps/web/src/refinements.css +++ b/apps/web/src/refinements.css @@ -313,6 +313,12 @@ .mcp-choose{display:inline-flex;align-items:center;gap:8px;width:fit-content} +.mcp-computer{display:grid;gap:10px;padding-bottom:12px;border-bottom:1px solid var(--border)} +.mcp-computer-title{font-size:13px} +.mcp-echo-row{display:flex;flex-wrap:wrap;gap:6px;padding:0 12px 12px;align-items:center} +.mcp-echo-row input{flex:1;min-width:8rem;height:32px;padding:0 10px;border:1px solid var(--border);border-radius:8px;background:transparent;color:inherit;font:inherit} +.mcp-echo-out{margin:0;padding:0 12px 12px;font-size:12px;color:var(--muted);white-space:pre-wrap;word-break:break-word} + .mcp-choose svg{width:16px;height:16px} diff --git a/apps/web/src/run-monitor.tsx b/apps/web/src/run-monitor.tsx index 2a749df..21da100 100644 --- a/apps/web/src/run-monitor.tsx +++ b/apps/web/src/run-monitor.tsx @@ -72,6 +72,12 @@ export function formatElapsed(ms: number): string { } /** 340ms / 6.4s / 1:05 — durations inside the trail stay glanceable. */ +/** Operation ids are UUIDs; the first block is enough to match a ledger row by eye. */ +function shortId(id: string): string { + const head = id.split("-")[0] ?? id; + return head.length > 12 ? `${head.slice(0, 12)}…` : head; +} + export function shortDuration(ms?: number | null): string { if (typeof ms !== "number" || !Number.isFinite(ms) || ms < 0) return ""; if (ms < 1000) return `${Math.round(ms)}ms`; @@ -85,6 +91,8 @@ function kindLabel(kind: string): string { if (kind === "tool") return t("monitorKindTool"); if (kind === "retry") return t("monitorKindRetry"); if (kind === "notice") return t("monitorKindNotice"); + if (kind === "job") return t("monitorKindTool"); + if (kind === "verify") return t("monitorKindRun"); return t("monitorKindRun"); } @@ -118,6 +126,9 @@ export function trailText(entry: RunActivityEntry): string { const parts = [entry.step || entry.name || t("monitorKindTool")]; const status = statusLabel(entry.status); if (status) parts.push(status); + if (entry.errorCode) parts.push(entry.errorCode); + if (entry.jobId) parts.push(entry.jobId); + if (entry.operationId) parts.push(`op ${shortId(entry.operationId)}`); const time = shortDuration(entry.elapsedMs); if (time) parts.push(time); const detail = entry.snippet ? ` — ${entry.snippet}` : ""; diff --git a/apps/web/src/types.ts b/apps/web/src/types.ts index c44472e..cab1ba4 100644 --- a/apps/web/src/types.ts +++ b/apps/web/src/types.ts @@ -12,7 +12,7 @@ export interface Room { id:string; name:string; members:RoomMember[]; hostBotId? export interface ComputerStatus { botId:string; mode:ComputerMode; state:ComputerState; sharedInput?:boolean; controlHolder:"none"|"bot"|"user"; takeoverRequested:boolean; busyBotName:string|null; busySessionId:string|null; busyRunId:string|null; busyStep?:string|null; usingComputer?:boolean; waitingRunId?:string|null; waitingSessionId?:string|null; queuedRuns?:number; display:string|null; profileMode:string; screenAvailable:boolean } /** One line of the live trail a run writes while it works. */ export type RunActivityKind = "run"|"model"|"tool"|"retry"|"notice"|"memory"; -export interface RunActivityEntry { memories?:{id:string;revision:number}[]; enabled?:boolean; botId?:string; id:number; kind:RunActivityKind; createdAt:string; turn?:number|null; event?:string|null; task?:string|null; reason?:string|null; turns?:number|null; limit?:number|null; error?:string|null; name?:string|null; step?:string|null; status?:string|null; elapsedMs?:number|null; toolCalls?:number|null; text?:string|null; snippet?:string|null; attempt?:number|null; gaveUp?:boolean|null } +export interface RunActivityEntry { memories?:{id:string;revision:number}[]; enabled?:boolean; botId?:string; id:number; kind:RunActivityKind; createdAt:string; turn?:number|null; event?:string|null; task?:string|null; reason?:string|null; turns?:number|null; limit?:number|null; error?:string|null; errorCode?:string|null; jobId?:string|null; operationId?:string|null; name?:string|null; step?:string|null; status?:string|null; elapsedMs?:number|null; toolCalls?:number|null; text?:string|null; snippet?:string|null; attempt?:number|null; gaveUp?:boolean|null } export interface RunActivityError { code:string; headline:string; action?:string; raw:string } export interface RunActivity { runId:string; status:string; turn:number|null; turnLimit:number|null; step:string|null; stepAt?:string|null; elapsedMs:number|null; error:RunActivityError|null; activity:RunActivityEntry[] } @@ -28,6 +28,7 @@ export interface McpTool { name:string; exposedName:string; description:string } export interface McpServer { id:string; name:string; transport:McpTransport; command:string|null; args:string[]; env:Record; url:string|null; headers:Record; enabled:boolean; status:"connected"|"disconnected"|"disabled"; error:string|null; tools:McpTool[]; createdAt:string; updatedAt:string } export interface McpSecretField { name:string; required:boolean; secret:boolean; hint:string } export interface McpCatalogEntry { id:string; title:string; description:string; transport:McpTransport; command:string|null; args:string[]; url:string|null; envKeys:McpSecretField[]; headerKeys:McpSecretField[]; source:"featured"|"registry"; remote:boolean } +export interface ComputerToolBinding { bindingId:string; packageId:string; version:string; sha256:string; installStatus:string; bindingStatus:string; ready:boolean; executionLocation?:string } export type ModelProviderId = "xai" | "opencode-go" | "openai-compatible"; export type VoiceProviderId = "xai" | "openai" | "scripted"; export interface VoiceSettings { diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index 97a1bba..4ca874e 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -31,6 +31,10 @@ tokio-tungstenite.workspace = true futures-util = "0.3" dotenvy = "0.15" fastembed = { version = "6.0.2", default-features = false, features = ["hf-hub-rustls-tls", "ort-load-dynamic"] } +# Same ort fastembed pins. Used only to pre-load the ONNX Runtime dylib through a +# fallible path: ort's lazy loader `expect`s while holding its global lock, and the +# poisoned lock aborts the process in ort's exit hook. +ort = { version = "=2.0.0-rc.13", default-features = false, features = ["load-dynamic"] } rmcp = { version = "3.2", default-features = false, features = ["client", "transport-child-process", "transport-streamable-http-client-reqwest"] } http = "1" aes-gcm = "0.10" diff --git a/crates/api/src/artifacts.rs b/crates/api/src/artifacts.rs new file mode 100644 index 0000000..f01bacf --- /dev/null +++ b/crates/api/src/artifacts.rs @@ -0,0 +1,182 @@ +//! Artifact catalog: bytes live on the Computer; the API only stores +//! references and streams a download. +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, HeaderValue, StatusCode}; +use axum::routing::get; +use axum::{Json, Router}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::db::Actor; +use crate::state::AppState; +use crate::tools::ToolCtx; + +type ApiError = (StatusCode, Json); + +pub fn router() -> Router { + Router::new() + .route("/api/bots/{id}/artifacts", get(list_artifacts)) + .route( + "/api/bots/{id}/artifacts/{artifact_id}", + get(download_artifact), + ) +} + +pub async fn record_file( + ctx: &ToolCtx, + relative_path: &str, + bytes: &[u8], + operation_id: Option<&str>, +) { + let Some(computer_id): Option = + sqlx::query_scalar("SELECT computer_id FROM bots WHERE id=$1") + .bind(&ctx.bot_id) + .fetch_optional(&ctx.pool) + .await + .ok() + .flatten() + .flatten() + else { + return; + }; + let digest = { + use sha2::{Digest, Sha256}; + hex::encode(Sha256::digest(bytes)) + }; + let _ = sqlx::query( + "INSERT INTO computer_artifacts (id, computer_id, bot_id, relative_path, sha256, size, operation_id) + VALUES ($1,$2,$3,$4,$5,$6,$7)", + ) + .bind(Uuid::new_v4().to_string()) + .bind(computer_id) + .bind(&ctx.bot_id) + .bind(relative_path) + .bind(digest) + .bind(bytes.len() as i64) + .bind(operation_id) + .execute(&ctx.pool) + .await; +} + +async fn list_artifacts( + State(state): State, + actor: Actor, + Path(bot_id): Path, +) -> Result, ApiError> { + type ArtifactRow = ( + String, + String, + String, + i64, + Option, + chrono::DateTime, + ); + let rows: Vec = sqlx::query_as( + "SELECT a.id, a.relative_path, a.sha256, a.size, a.operation_id, a.created_at + FROM computer_artifacts a + JOIN bots b ON b.id=a.bot_id + WHERE a.bot_id=$1 AND b.space_id=$2 AND b.user_id=$3 + ORDER BY a.created_at DESC LIMIT 100", + ) + .bind(&bot_id) + .bind(&actor.space_id) + .bind(&actor.user_id) + .fetch_all(state.pool()) + .await + .map_err(internal)?; + Ok(Json(json!({ + "artifacts": rows.into_iter().map(|(id, path, sha, size, op, at)| json!({ + "id": id, + "path": path, + "sha256": sha, + "size": size, + "operationId": op, + "createdAt": at, + })).collect::>() + }))) +} + +async fn download_artifact( + State(state): State, + actor: Actor, + Path((bot_id, artifact_id)): Path<(String, String)>, +) -> Result<(HeaderMap, Vec), ApiError> { + let row: Option<(String, String, String)> = sqlx::query_as( + "SELECT a.relative_path, a.sha256, a.computer_id + FROM computer_artifacts a + JOIN bots b ON b.id=a.bot_id + WHERE a.id=$1 AND a.bot_id=$2 AND b.space_id=$3 AND b.user_id=$4", + ) + .bind(&artifact_id) + .bind(&bot_id) + .bind(&actor.space_id) + .bind(&actor.user_id) + .fetch_optional(state.pool()) + .await + .map_err(internal)?; + let Some((relative, sha, computer_id)) = row else { + return Err(( + StatusCode::NOT_FOUND, + Json(json!({"message":"artifact not found"})), + )); + }; + let computer = state + .db + .get_computer(&computer_id) + .await + .map_err(internal)? + .ok_or(( + StatusCode::NOT_FOUND, + Json(json!({"message":"computer not found"})), + ))?; + let Some(computer_ref) = crate::computer::computer_ref(&computer) else { + return Err(( + StatusCode::CONFLICT, + Json(json!({"message":"computer is not running"})), + )); + }; + let bytes = state + .sandbox + .read_file( + &computer_ref, + &relative, + &crate::computer::adapter_context(&actor, &bot_id, "artifact"), + ) + .await + .map_err(|error| { + ( + StatusCode::BAD_GATEWAY, + Json(json!({"message": error.to_string()})), + ) + })?; + let actual = hex::encode(Sha256::digest(&bytes)); + if actual != sha { + return Err(( + StatusCode::CONFLICT, + Json(json!({"message":"artifact bytes changed on the Computer"})), + )); + } + let mut headers = HeaderMap::new(); + headers.insert( + axum::http::header::CONTENT_TYPE, + HeaderValue::from_static("application/octet-stream"), + ); + headers.insert( + axum::http::header::CONTENT_DISPOSITION, + HeaderValue::from_str(&format!( + "attachment; filename=\"{}\"", + relative.rsplit('/').next().unwrap_or("artifact") + )) + .unwrap_or_else(|_| HeaderValue::from_static("attachment")), + ); + Ok((headers, bytes)) +} + +fn internal(error: E) -> ApiError { + tracing::error!("artifacts: {error}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"message":"internal error"})), + ) +} diff --git a/crates/api/src/computer.rs b/crates/api/src/computer.rs index 1ebd972..00f9864 100644 --- a/crates/api/src/computer.rs +++ b/crates/api/src/computer.rs @@ -8,9 +8,10 @@ use lazyboy_contracts::{ }; use lazyboy_control::{ AdapterContext, CommandRequest, EnsureScreenRequest, ProvisionRequest, admit_gui, - admit_new_screen, browser_profile_path, execution_blocks_user_takeover, profile_lock_key, - screen_layout, team_bot_workspace_directory, + admit_new_screen, attach_display_for_tool, browser_profile_path, + execution_blocks_user_takeover, profile_lock_key, screen_layout, team_bot_workspace_directory, }; +use serde_json::json; use uuid::Uuid; use crate::db::{ @@ -165,6 +166,7 @@ pub async fn refresh_cursor_color( cwd: None, timeout_ms: Some(5_000), stdin: None, + ..CommandRequest::default() }, &adapter_context_for(actor, bot_id, "cursor-color", Some(&screen), None), ) @@ -402,6 +404,7 @@ async fn probe_computer_container( cwd: None, timeout_ms: Some(5_000), stdin: None, + ..CommandRequest::default() }, &adapter_context(actor, bot_id, "probe"), ) @@ -544,6 +547,16 @@ pub async fn take_profile_lock( } pub async fn boot(state: &AppState, actor: &Actor, bot_id: &str) -> Result { + boot_for(state, actor, bot_id, true).await +} + +/// `need_gui=false` provisions the Runner only: no `ensure_screen`, no viewer. +pub async fn boot_for( + state: &AppState, + actor: &Actor, + bot_id: &str, + need_gui: bool, +) -> Result { let bot = state .db .get_bot(actor, bot_id) @@ -562,11 +575,17 @@ pub async fn boot(state: &AppState, actor: &Actor, bot_id: &str) -> Result Result return Ok(status), Err(error) => { tracing::warn!("computer {computer_id} resume failed: {error}"); @@ -625,6 +645,7 @@ pub async fn boot(state: &AppState, actor: &Actor, bot_id: &str) -> Result Result Result Result Result { let ctx = adapter_context(actor, bot_id, "resume"); let home = home_path(&state.data_dir, &computer.home_key); @@ -717,6 +750,7 @@ async fn resume_paused( home_key: computer.home_key.clone(), home_path: home.to_string_lossy().into_owned(), provider_ref: computer.provider_ref.clone(), + runner_only: !need_gui, }, &ctx, ), @@ -745,10 +779,14 @@ async fn resume_paused( .map_err(|error| error.to_string())? .ok_or_else(|| "computer not found".to_string())?; if current.state == "running" { - let screen = ensure_bot_screen(state, actor, bot_id, ¤t, None) - .await - .ok() - .and_then(|bound| bound.row); + let screen = if attach_display_for_tool(need_gui) { + ensure_bot_screen(state, actor, bot_id, ¤t, None) + .await + .ok() + .and_then(|bound| bound.row) + } else { + None + }; return Ok(status_from(bot_id, ¤t, screen.as_ref(), None)); } return Err("computer resume was superseded".into()); @@ -759,11 +797,17 @@ async fn resume_paused( .await .map_err(|error| error.to_string())? .ok_or_else(|| "computer not found".to_string())?; - let screen = ensure_bot_screen(state, actor, bot_id, &computer, None) - .await - .ok() - .and_then(|bound| bound.row); - restore_computer_screens(state, actor, &computer, bot_id).await; + let screen = if attach_display_for_tool(need_gui) { + ensure_bot_screen(state, actor, bot_id, &computer, None) + .await + .ok() + .and_then(|bound| bound.row) + } else { + None + }; + if attach_display_for_tool(need_gui) { + restore_computer_screens(state, actor, &computer, bot_id).await; + } Ok(status_from(bot_id, &computer, screen.as_ref(), None)) } @@ -888,13 +932,14 @@ pub async fn restart( "UPDATE computers SET state = 'stopped', provider_ref = NULL, control_holder = 'none', control_lease_id = NULL, control_lease_expires_at = NULL, control_bot_id = NULL, control_run_id = NULL, execution_bot_id = NULL, execution_run_id = NULL, - execution_lease_expires_at = NULL, updated_at = now() + execution_lease_expires_at = NULL, generation = generation + 1, updated_at = now() WHERE id = $1", ) .bind(&computer_id) .execute(state.pool()) .await .map_err(|error| error.to_string())?; + interrupt_computer_jobs(state, &computer_id).await; boot(state, actor, bot_id).await } @@ -1091,13 +1136,65 @@ pub async fn idle_loop(state: AppState) { } } +/// `computer_jobs.status` is only written when a tool asks the Computer, so a +/// background job nobody polled would keep the Computer "busy" forever. Before +/// the idle reaper trusts those rows, ask the Computer whether they still run. +async fn refresh_running_jobs(state: &AppState, computer: &ComputerRow) { + let Some(computer_ref) = computer_ref(computer) else { + return; + }; + let ids: Vec = sqlx::query_scalar( + "SELECT id FROM computer_jobs WHERE computer_id=$1 AND status='running' + ORDER BY updated_at LIMIT 16", + ) + .bind(&computer.id) + .fetch_all(state.pool()) + .await + .unwrap_or_default(); + for id in ids { + let outcome = state + .sandbox + .execute( + &computer_ref, + CommandRequest { + argv: Vec::new(), + timeout_ms: Some(5_000), + job_id: Some(id.clone()), + job_op: Some("status".into()), + ..CommandRequest::default() + }, + &idle_adapter(computer, "job-status"), + ) + .await; + let status = match outcome { + Ok(result) => match result.status { + Some(status) if status != "running" => status, + _ => continue, + }, + Err(error) if error.to_string().contains("unknown job") => "interrupted".into(), + Err(_) => continue, + }; + let _ = sqlx::query( + "UPDATE computer_jobs SET status=$2, updated_at=now() WHERE id=$1 AND status='running'", + ) + .bind(&id) + .bind(&status) + .execute(state.pool()) + .await; + } +} + async fn computer_has_active_work(state: &AppState, computer_id: &str) -> bool { - let active: Result, _> = sqlx::query_as( + // Row presence only: `SELECT 1` is INT4 and decoding it as i64 used to fail + // on every call, which silently turned this guard into "never busy". + let active = sqlx::query( "SELECT 1 FROM runs WHERE status IN ('queued','leased','running','waiting_input','waiting_takeover') AND bot_id IN (SELECT id FROM bots WHERE computer_id = $1) UNION ALL SELECT 1 FROM taught_skills WHERE status IN ('recording','drafting') AND bot_id IN (SELECT id FROM bots WHERE computer_id = $1) + UNION ALL + SELECT 1 FROM computer_jobs WHERE computer_id = $1 AND status IN ('running','accepted') LIMIT 1", ) .bind(computer_id) @@ -1121,7 +1218,7 @@ async fn pause_idle_computers(state: &AppState) { "SELECT id, space_id, user_id, scope, scope_key, home_key, home_revision, kind, provider_ref, state, control_holder, control_lease_id, control_lease_expires_at, control_bot_id, control_run_id, execution_run_id, execution_bot_id, execution_lease_expires_at, execution_fence, - browser_profile_mode + browser_profile_mode, generation FROM computers WHERE state = 'running' AND updated_at < $1", ) .bind(cutoff) @@ -1129,6 +1226,7 @@ async fn pause_idle_computers(state: &AppState) { .await; let Ok(rows) = rows else { return }; for computer in rows { + refresh_running_jobs(state, &computer).await; if computer_has_active_work(state, &computer.id).await { continue; } @@ -1157,7 +1255,7 @@ async fn stop_parked_computers(state: &AppState) { "SELECT id, space_id, user_id, scope, scope_key, home_key, home_revision, kind, provider_ref, state, control_holder, control_lease_id, control_lease_expires_at, control_bot_id, control_run_id, execution_run_id, execution_bot_id, execution_lease_expires_at, execution_fence, - browser_profile_mode + browser_profile_mode, generation FROM computers WHERE state = 'suspended' AND updated_at < $1", ) .bind(cutoff) @@ -1184,6 +1282,101 @@ async fn stop_parked_computers(state: &AppState) { } } +async fn interrupt_computer_jobs(state: &AppState, computer_id: &str) { + crate::job_store::interrupt_computer(computer_id); + let _ = sqlx::query( + "UPDATE computer_jobs SET status='interrupted', updated_at=now() + WHERE computer_id=$1 AND status IN ('running','accepted')", + ) + .bind(computer_id) + .execute(state.pool()) + .await; +} + +pub async fn component_health( + state: &AppState, + actor: &Actor, + bot_id: &str, +) -> Result { + let bot = state + .db + .get_bot(actor, bot_id) + .await + .map_err(|error| error.to_string())? + .ok_or_else(|| "bot not found".to_string())?; + let computer_id = bot + .computer_id + .ok_or_else(|| "bot has no computer".to_string())?; + let computer = state + .db + .get_computer(&computer_id) + .await + .map_err(|error| error.to_string())? + .ok_or_else(|| "computer not found".to_string())?; + let display_backend: String = + sqlx::query_scalar("SELECT display_backend FROM computers WHERE id=$1") + .bind(&computer.id) + .fetch_optional(state.pool()) + .await + .ok() + .flatten() + .unwrap_or_else(|| { + std::env::var("LAZYBOY_DISPLAY_BACKEND").unwrap_or_else(|_| "xvfb_x11vnc".into()) + }); + let jobs_running: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM computer_jobs WHERE computer_id=$1 AND status='running'", + ) + .bind(&computer.id) + .fetch_one(state.pool()) + .await + .unwrap_or(0); + let jobs_quota = lazyboy_control::native_job_concurrency() as i64; + let sandbox = if computer.state != "running" || computer.provider_ref.is_none() { + json!({"name":"sandbox","ok":true,"detail":"computer not running"}) + } else { + match probe_computer_container(state, actor, bot_id, &computer).await { + ContainerProbe::Alive => json!({"name":"sandbox","ok":true,"detail":"alive"}), + ContainerProbe::Missing => { + json!({"name":"sandbox","ok":false,"detail":"container missing"}) + } + ContainerProbe::Unknown => json!({"name":"sandbox","ok":false,"detail":"probe failed"}), + } + }; + let journal_ok = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM computer_operations") + .fetch_one(state.pool()) + .await + .is_ok(); + let components = vec![ + json!({"name":"database","ok":true,"detail":"reachable"}), + sandbox, + json!({"name":"display","ok":true,"detail":display_backend}), + json!({ + "name":"jobs", + "ok": jobs_running <= jobs_quota, + "detail": format!("{jobs_running}/{jobs_quota} running") + }), + json!({ + "name":"operations", + "ok": journal_ok, + "detail": if journal_ok { "available" } else { "unreachable" } + }), + ]; + let ok = components + .iter() + .all(|item| item["ok"].as_bool() == Some(true)); + Ok(json!({ + "computerId": computer.id, + "botId": bot_id, + "state": computer.state, + "generation": computer.generation, + "displayBackend": display_backend, + "jobsRunning": jobs_running, + "jobsQuota": jobs_quota, + "ok": ok, + "components": components, + })) +} + pub fn computer_ref(computer: &ComputerRow) -> Option { let provider_ref = computer.provider_ref.clone()?; Some(lazyboy_control::ComputerRef { @@ -1315,6 +1508,7 @@ mod shared_input_tests { execution_lease_expires_at: None, execution_fence: 1, browser_profile_mode: "per-bot".into(), + generation: 1, }; let screen = ScreenRow { id: "screen".into(), @@ -1363,3 +1557,125 @@ mod shared_input_tests { assert!(!user_can_interact(&computer, Some(&screen), "bot")); } } + +#[cfg(test)] +mod idle_job_tests { + use super::*; + + fn app(pool: sqlx::PgPool) -> AppState { + AppState { + db: crate::db::Db { pool }, + sandbox: std::sync::Arc::new(lazyboy_sandbox::FakeSandbox::new()), + data_dir: String::new(), + auth: crate::auth::AuthConfig::from_env(), + memory: crate::memory::MemoryService::from_env(), + mcp: crate::mcp::McpHub::new(), + calls: crate::state::CallRegistry::default(), + wakes: crate::state::WakeBus::default(), + } + } + + async fn job_status(pool: &sqlx::PgPool, id: &str) -> String { + sqlx::query_scalar("SELECT status FROM computer_jobs WHERE id=$1") + .bind(id) + .fetch_one(pool) + .await + .unwrap() + } + + /// A background job that finished (or vanished with its supervisor) while + /// nobody polled it must not keep the Computer busy forever. + #[sqlx::test(migrations = "../../migrations")] + async fn idle_reaper_reconciles_unpolled_jobs_with_the_computer(pool: sqlx::PgPool) { + sqlx::query("INSERT INTO users(id,name) VALUES ('u','test')") + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO spaces(id,user_id,name) VALUES ('s','u','test')") + .execute(&pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO computers(id,space_id,user_id,scope,scope_key,home_key,provider_ref,state) + VALUES ('c','s','u','team','team:s','home','container','running')", + ) + .execute(&pool) + .await + .unwrap(); + let state = app(pool.clone()); + let computer: ComputerRow = sqlx::query_as( + "SELECT id, space_id, user_id, scope, scope_key, home_key, home_revision, kind, provider_ref, state, + control_holder, control_lease_id, control_lease_expires_at, control_bot_id, control_run_id, + execution_run_id, execution_bot_id, execution_lease_expires_at, execution_fence, + browser_profile_mode, generation + FROM computers WHERE id='c'", + ) + .fetch_one(&pool) + .await + .unwrap(); + let computer_ref = computer_ref(&computer).unwrap(); + let adapter = idle_adapter(&computer, "test"); + + // Three jobs the DB believes are running: one really is, one finished + // unobserved, one the Computer has never heard of (supervisor restart). + for (id, argv) in [("job-live", "/bin/sleep"), ("job-done", "/bin/true")] { + state + .sandbox + .execute( + &computer_ref, + CommandRequest { + argv: vec![argv.into(), "30".into()], + background: true, + job_id: Some(id.into()), + ..CommandRequest::default() + }, + &adapter, + ) + .await + .unwrap(); + } + for id in ["job-live", "job-done", "job-lost"] { + sqlx::query( + "INSERT INTO computer_jobs(id,computer_id,bot_id,status) VALUES ($1,'c','b','running')", + ) + .bind(id) + .execute(&pool) + .await + .unwrap(); + } + assert!(computer_has_active_work(&state, "c").await); + + refresh_running_jobs(&state, &computer).await; + assert_eq!(job_status(&pool, "job-live").await, "running"); + assert_eq!(job_status(&pool, "job-done").await, "succeeded"); + assert_eq!(job_status(&pool, "job-lost").await, "interrupted"); + assert!(computer_has_active_work(&state, "c").await); + + sqlx::query("DELETE FROM computer_jobs WHERE id='job-live'") + .execute(&pool) + .await + .unwrap(); + assert!(!computer_has_active_work(&state, "c").await); + + // A live run on a bot bound to this Computer is work too (this guard + // used to be dead: `SELECT 1` never decoded). + sqlx::query( + "INSERT INTO bots(id,space_id,user_id,name,computer_id) VALUES ('b','s','u','bot','c')", + ) + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO threads(id,space_id,user_id,bot_id) VALUES ('t','s','u','b')") + .execute(&pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO runs(id,space_id,user_id,bot_id,thread_id,status,trigger,prompt) + VALUES ('r','s','u','b','t','running','message','go')", + ) + .execute(&pool) + .await + .unwrap(); + assert!(computer_has_active_work(&state, "c").await); + } +} diff --git a/crates/api/src/db.rs b/crates/api/src/db.rs index 219e0eb..ac22627 100644 --- a/crates/api/src/db.rs +++ b/crates/api/src/db.rs @@ -75,6 +75,7 @@ pub struct ComputerRow { pub execution_lease_expires_at: Option>, pub execution_fence: i32, pub browser_profile_mode: String, + pub generation: i32, } #[derive(Debug, Clone, FromRow)] @@ -578,7 +579,7 @@ impl Db { "SELECT id, space_id, user_id, scope, scope_key, home_key, home_revision, kind, provider_ref, state, control_holder, control_lease_id, control_lease_expires_at, control_bot_id, control_run_id, execution_run_id, execution_bot_id, execution_lease_expires_at, execution_fence, - browser_profile_mode + browser_profile_mode, generation FROM computers WHERE id = $1", ) .bind(computer_id) @@ -734,7 +735,7 @@ async fn ensure_computer( "SELECT id, space_id, user_id, scope, scope_key, home_key, home_revision, kind, provider_ref, state, control_holder, control_lease_id, control_lease_expires_at, control_bot_id, control_run_id, execution_run_id, execution_bot_id, execution_lease_expires_at, execution_fence, - browser_profile_mode + browser_profile_mode, generation FROM computers WHERE scope_key = $1", ) .bind(scope_key) diff --git a/crates/api/src/job_store.rs b/crates/api/src/job_store.rs new file mode 100644 index 0000000..33438c9 --- /dev/null +++ b/crates/api/src/job_store.rs @@ -0,0 +1,10 @@ +//! Process-local Runner jobs. Shared so Computer recreate can interrupt +//! leftovers without a tools ↔ computer module cycle. +use lazyboy_control::JobSupervisor; + +pub static JOBS: std::sync::LazyLock = + std::sync::LazyLock::new(JobSupervisor::default); + +pub fn interrupt_computer(computer_id: &str) { + JOBS.interrupt_computer(computer_id); +} diff --git a/crates/api/src/main.rs b/crates/api/src/main.rs index 0346d5b..20e85e8 100644 --- a/crates/api/src/main.rs +++ b/crates/api/src/main.rs @@ -1,14 +1,17 @@ mod accounts; +mod artifacts; mod attachments; mod auth; mod computer; mod context_fit; mod db; mod file_skills; +mod job_store; mod mcp; mod mcp_catalog; mod memory; mod monitor; +mod operations; mod retention; mod rooms; mod routes; @@ -19,6 +22,7 @@ mod screen_proxy; mod sessions; mod skills; mod state; +mod tool_install; mod tools; mod vault; mod voice; diff --git a/crates/api/src/mcp.rs b/crates/api/src/mcp.rs index 4498dc0..7379db9 100644 --- a/crates/api/src/mcp.rs +++ b/crates/api/src/mcp.rs @@ -12,11 +12,9 @@ use rig_core::completion::ToolDefinition; use rmcp::model::{CallToolRequestParams, ClientInfo, Tool}; use rmcp::service::RunningService; use rmcp::transport::StreamableHttpClientTransport; -use rmcp::transport::child_process::TokioChildProcess; use rmcp::transport::streamable_http_client::StreamableHttpClientTransportConfig; use rmcp::{RoleClient, ServiceExt}; use serde_json::{Map, Value, json}; -use tokio::process::Command; use tokio::sync::Mutex; use uuid::Uuid; @@ -34,7 +32,9 @@ pub struct McpHub { } struct Live { - client: LiveClient, + space_id: String, + user_id: String, + client: std::sync::Arc, tools: Vec, defs: Vec, } @@ -74,38 +74,53 @@ impl McpHub { .collect() } - pub async fn definitions(&self) -> Vec { + pub async fn definitions_for(&self, actor: &Actor) -> Vec { let live = self.inner.lock().await; - live.values().flat_map(|entry| entry.defs.clone()).collect() + live.values() + .filter(|entry| entry.space_id == actor.space_id && entry.user_id == actor.user_id) + .flat_map(|entry| entry.defs.clone()) + .collect() } - pub async fn call(&self, exposed: &str, args: &Value) -> Result { - let live = self.inner.lock().await; - for entry in live.values() { - for tool in &entry.tools { - if tool.exposed_name != exposed { - continue; + pub async fn call_for( + &self, + actor: Option<&Actor>, + exposed: &str, + args: &Value, + ) -> Result { + let found = { + let live = self.inner.lock().await; + live.values().find_map(|entry| { + if let Some(actor) = actor + && (entry.space_id != actor.space_id || entry.user_id != actor.user_id) + { + return None; } - let arguments = match args { - Value::Object(map) => map.clone(), - Value::Null => Map::new(), - other => { - let mut map = Map::new(); - map.insert("value".into(), other.clone()); - map - } - }; - let params = - CallToolRequestParams::new(tool.name.clone()).with_arguments(arguments); - let result = entry - .client - .call_tool(params) - .await - .map_err(|error| error.to_string())?; - return serde_json::to_string_pretty(&result).map_err(|error| error.to_string()); + entry + .tools + .iter() + .find(|tool| tool.exposed_name == exposed) + .map(|tool| (entry.client.clone(), tool.name.clone())) + }) + }; + let Some((client, name)) = found else { + return Err(format!("unknown MCP tool {exposed}")); + }; + let arguments = match args { + Value::Object(map) => map.clone(), + Value::Null => Map::new(), + other => { + let mut map = Map::new(); + map.insert("value".into(), other.clone()); + map } - } - Err(format!("unknown MCP tool {exposed}")) + }; + let params = CallToolRequestParams::new(name).with_arguments(arguments); + let result = client + .call_tool(params) + .await + .map_err(|error| error.to_string())?; + serde_json::to_string_pretty(&result).map_err(|error| error.to_string()) } pub async fn disconnect(&self, id: &str) { @@ -178,7 +193,9 @@ impl McpHub { self.inner.lock().await.insert( row.id.clone(), Live { - client, + space_id: row.space_id.clone(), + user_id: row.user_id.clone(), + client: std::sync::Arc::new(client), tools: tools.clone(), defs, }, @@ -262,43 +279,8 @@ fn exposed_name_for(slug: &str, tool: &str) -> String { async fn connect_client(row: &McpRow) -> Result { match row.transport.as_str() { "stdio" => { - let command = row - .command - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - .ok_or_else(|| "stdio 需要 command".to_string())?; - let mut cmd = Command::new(command); - cmd.env_clear(); - for key in [ - "PATH", - "HOME", - "LANG", - "LC_ALL", - "TMPDIR", - "PYTHONPATH", - "NODE_EXTRA_CA_CERTS", - ] { - if let Some(value) = std::env::var_os(key) { - cmd.env(key, value); - } - } - cmd.kill_on_drop(true); - cmd.args(&row.args); - cmd.stdin(std::process::Stdio::piped()); - cmd.stdout(std::process::Stdio::piped()); - cmd.stderr(std::process::Stdio::piped()); - for (key, value) in &row.env { - if let Some(text) = value.as_str() { - cmd.env(key, text); - } - } - let transport = TokioChildProcess::new(cmd) - .map_err(|error| humanize_mcp_error(Some(command), &error.to_string()))?; - ClientInfo::default() - .serve(transport) - .await - .map_err(|error| humanize_mcp_error(Some(command), &error.to_string())) + debug_assert!(!lazyboy_control::api_host_may_spawn_stdio()); + Err(lazyboy_control::catalog_stdio_api_error().into()) } "http" | "sse" => { let url = row @@ -359,6 +341,8 @@ fn humanize_mcp_error(command: Option<&str>, error: &str) -> String { #[derive(Clone)] pub struct McpRow { pub id: String, + pub space_id: String, + pub user_id: String, pub name: String, pub transport: String, pub command: Option, @@ -410,6 +394,8 @@ type RowTuple = ( fn row_from(tuple: RowTuple) -> McpRow { McpRow { id: tuple.0, + space_id: String::new(), + user_id: String::new(), name: tuple.1, transport: tuple.2, command: tuple.3, @@ -449,7 +435,15 @@ async fn load_rows(pool: &sqlx::PgPool, actor: &Actor) -> Result, sq .bind(&actor.user_id) .fetch_all(pool) .await?; - Ok(rows.into_iter().map(row_from).collect()) + Ok(rows + .into_iter() + .map(|tuple| { + let mut row = row_from(tuple); + row.space_id = actor.space_id.clone(); + row.user_id = actor.user_id.clone(); + row + }) + .collect()) } async fn load_row( @@ -466,7 +460,12 @@ async fn load_row( .bind(&actor.user_id) .fetch_optional(pool) .await?; - Ok(row.map(row_from)) + Ok(row.map(|tuple| { + let mut row = row_from(tuple); + row.space_id = actor.space_id.clone(); + row.user_id = actor.user_id.clone(); + row + })) } pub fn router() -> Router { @@ -533,7 +532,13 @@ fn validate_input(input: &UpsertMcpServerInput) -> Result<(), ApiError> { match input.transport.as_str() { "stdio" => { if input.command.as_deref().unwrap_or("").trim().is_empty() { - return Err(bad("stdio 需要 command，例如 npx 或 uvx")); + return Err(bad("stdio 需要 command，例如已審查的 Computer 套件路徑")); + } + if lazyboy_control::refuse_unpinned_npx( + input.command.as_deref().unwrap_or(""), + &input.args, + ) { + return Err(bad(lazyboy_control::catalog_stdio_api_error())); } } "http" | "sse" => { @@ -592,7 +597,15 @@ async fn create_server( } .to_string(); let mut tools = Vec::new(); - if row.enabled { + let mut error = None; + if row.enabled && row.transport == "stdio" { + error = Some(lazyboy_control::catalog_stdio_api_error().to_string()); + let _ = sqlx::query("UPDATE mcp_servers SET last_error=$2 WHERE id=$1") + .bind(&id) + .bind(error.as_deref()) + .execute(state.pool()) + .await; + } else if row.enabled { match tokio::time::timeout(CONNECT_TIMEOUT, state.mcp.connect_row(&row)).await { Ok(Ok(connected)) => { status = "connected".into(); @@ -610,7 +623,7 @@ async fn create_server( } Ok(( StatusCode::CREATED, - Json(row.into_server(status, None, tools)), + Json(row.into_server(status, error, tools)), )) } @@ -776,3 +789,22 @@ async fn reconnect_server( } } } + +#[cfg(test)] +mod tests { + #[test] + fn mcp_rs_does_not_spawn_stdio_on_the_api_host() { + let src = include_str!("mcp.rs"); + let child = format!("{}{}{}", "use rmcp::", "transport::", "child_process"); + let command = format!("{}{}", "use tokio::process::", "Command"); + assert!( + !src.contains(&child), + "catalog stdio must not be a child of the API process" + ); + assert!(!src.contains(&command)); + assert_eq!( + lazyboy_control::mcp_execution_location("stdio"), + lazyboy_control::McpExecutionLocation::AssignedComputer + ); + } +} diff --git a/crates/api/src/memory.rs b/crates/api/src/memory.rs index 542481a..7890c01 100644 --- a/crates/api/src/memory.rs +++ b/crates/api/src/memory.rs @@ -43,6 +43,35 @@ impl ModelState { } } +/// Load the ONNX Runtime shared library before fastembed touches `ort`. +/// +/// `ort`'s lazy loader panics (`expect`) inside its global environment mutex +/// when the dylib is missing. `catch_unwind` around fastembed would swallow that +/// panic but leave the mutex poisoned, and ort's `.fini_array` hook then panics +/// again at process exit, turning a clean shutdown into SIGABRT. `init_from` +/// walks the same `ORT_DYLIB_PATH` / default-name resolution but returns `Err`. +fn preload_onnx_runtime() -> Result<(), String> { + static LOADED: std::sync::OnceLock> = std::sync::OnceLock::new(); + LOADED + .get_or_init(|| { + let path = match std::env::var("ORT_DYLIB_PATH") { + Ok(value) if !value.is_empty() => value, + #[cfg(target_os = "macos")] + _ => "libonnxruntime.dylib".to_string(), + #[cfg(not(target_os = "macos"))] + _ => "libonnxruntime.so".to_string(), + }; + load_onnx_runtime(std::path::Path::new(&path)) + }) + .clone() +} + +fn load_onnx_runtime(path: &std::path::Path) -> Result<(), String> { + ort::init_from(path) + .map(|_builder| ()) + .map_err(|error| format!("ONNX Runtime unavailable: {error}")) +} + #[derive(Debug, Clone, Serialize, FromRow)] #[serde(rename_all = "camelCase")] pub struct MemoryItem { @@ -208,6 +237,7 @@ impl MemoryService { } let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { if !matches!(*state, ModelState::Ready(_)) { + preload_onnx_runtime()?; let options = TextInitOptions::new(EmbeddingModel::ParaphraseMLMiniLML12V2) .with_cache_dir(cache_dir) .with_show_download_progress(false); @@ -865,6 +895,15 @@ mod tests { use std::sync::{Arc, Mutex}; use uuid::Uuid; + #[test] + fn a_missing_onnx_runtime_is_an_error_not_a_panic() { + // Going through ort's lazy loader here would `expect` while holding its + // global lock and abort the whole test binary at exit. + let missing = std::env::temp_dir().join(format!("no-ort-{}.so", Uuid::new_v4())); + let error = super::load_onnx_runtime(&missing).unwrap_err(); + assert!(error.starts_with("ONNX Runtime unavailable"), "{error}"); + } + #[tokio::test] #[ignore = "requires the downloaded embedding model and ONNX runtime"] async fn memory_model_recovers_after_cache_failure() { @@ -1207,6 +1246,11 @@ mod tests { ); let mut recall_service = service.clone(); recall_service.enabled = true; + // Model the "embedding backend is down" case explicitly instead of relying + // on a missing ONNX runtime to fail (that path is environment-dependent). + recall_service.model = Arc::new(Mutex::new(ModelState::Unavailable { + retry_at: std::time::Instant::now() + std::time::Duration::from_secs(3600), + })); let actor = Actor { user_id: "u".into(), space_id: "s".into(), diff --git a/crates/api/src/monitor.rs b/crates/api/src/monitor.rs index c37710a..a012f7e 100644 --- a/crates/api/src/monitor.rs +++ b/crates/api/src/monitor.rs @@ -87,15 +87,31 @@ async fn memory_usage_items( /// Append one line to the run's trail. Diagnostics never fail a run: a write /// that cannot land is logged and dropped. pub async fn record(state: &AppState, run_id: &str, kind: &str, payload: Value) { - let result = sqlx::query("INSERT INTO run_activity (run_id,kind,payload) VALUES ($1,$2,$3)") + record_pool(state.pool(), run_id, kind, payload).await; +} + +pub async fn record_pool(pool: &sqlx::PgPool, run_id: &str, kind: &str, payload: Value) { + if let Err(error) = try_record_pool(pool, run_id, kind, payload).await { + tracing::warn!(run_id, kind, "failed to record run activity: {error}"); + } +} + +/// Same as [`record_pool`] but lets the caller decide what a failed write +/// means (the outbox must not mark an event delivered on failure). +pub async fn try_record_pool( + pool: &sqlx::PgPool, + run_id: &str, + kind: &str, + mut payload: Value, +) -> Result<(), sqlx::Error> { + lazyboy_control::redact_json(&mut payload); + sqlx::query("INSERT INTO run_activity (run_id,kind,payload) VALUES ($1,$2,$3)") .bind(run_id) .bind(kind) .bind(clamp_strings(&payload, MAX_STRING_CHARS)) - .execute(state.pool()) - .await; - if let Err(error) = result { - tracing::warn!(run_id, kind, "failed to record run activity: {error}"); - } + .execute(pool) + .await + .map(|_| ()) } /// One line of text for the trail: single line, bounded, never an image. diff --git a/crates/api/src/operations.rs b/crates/api/src/operations.rs new file mode 100644 index 0000000..c386ce4 --- /dev/null +++ b/crates/api/src/operations.rs @@ -0,0 +1,473 @@ +//! Durable operation ledger and outbox. Mutations are not started if the +//! journal insert fails. The same operation_id returns the stored result; +//! a different payload hash is rejected. +//! +//! Postgres is the only source of truth here. Ledger rows are keyed per bot +//! (`{bot_id}:{operation_id}`): an operation id is chosen by the model, so two +//! bots picking the same string must never see each other's results. +use serde_json::{Value, json}; +use sqlx::PgPool; +use uuid::Uuid; + +use lazyboy_control::{OperationLedger, redact_json, redact_secret_patterns}; + +use crate::tools::{ToolCtx, ToolOutcome}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Begin { + Proceed, + Replay { + text: String, + error_code: Option, + pause: bool, + }, + /// Same id and payload, but the earlier attempt never recorded a result + /// (crash, timeout, halt). Its effect is unknown, so it is not re-run. + InProgress, + PayloadMismatch, + JournalUnavailable, +} + +pub fn payload_hash(name: &str, args: &Value) -> String { + let mut args = args.clone(); + if let Some(object) = args.as_object_mut() { + object.remove("operation_id"); + object.remove("operationId"); + } + OperationLedger::payload_hash(format!("{name}:{args}").as_bytes()) +} + +pub fn mutating_tool(name: &str, args: &Value) -> bool { + match name { + "computer_act" | "shell" | "write_file" | "launch_app" | "open_path" + | "create_schedule" | "cancel_schedule" | "remember" | "forget_memory" + | "use_saved_login" | "request_takeover" | "form_fill" | "computer_mcp" => true, + // Asking about a job changes nothing; starting or cancelling one does. + "exec" => !matches!(args.get("action").and_then(Value::as_str), Some("status")), + "browser" => matches!( + args.get("action").and_then(Value::as_str), + Some("click") | Some("type") | Some("navigate") | Some("press") + ), + _ => false, + } +} + +/// Ledger row id. Namespaced per bot so a model-chosen id cannot collide with +/// (or replay) another bot's operation. +pub fn ledger_key(bot_id: &str, operation_id: &str) -> String { + format!("{bot_id}:{operation_id}") +} + +pub async fn begin(ctx: &ToolCtx, name: &str, args: &Value, operation_id: &str) -> Begin { + if !mutating_tool(name, args) { + return Begin::Proceed; + } + let hash = payload_hash(name, args); + // A DB error here is a journal failure, not "this bot has no Computer": + // tools that need no Computer (memory, schedules) are journaled with NULL. + let computer_id = match bot_computer_id(ctx).await { + Ok(id) => id, + Err(_) => return Begin::JournalUnavailable, + }; + persist_begin( + &ctx.pool, + &ledger_key(&ctx.bot_id, operation_id), + computer_id.as_deref(), + &ctx.bot_id, + &ctx.run_id, + &hash, + ) + .await + .unwrap_or(Begin::JournalUnavailable) +} + +async fn persist_begin( + pool: &PgPool, + key: &str, + computer_id: Option<&str>, + bot_id: &str, + run_id: &str, + hash: &str, +) -> Result { + let existing: Option<(String, Option, String)> = sqlx::query_as( + "SELECT payload_hash, result, status FROM computer_operations WHERE id=$1 AND bot_id=$2", + ) + .bind(key) + .bind(bot_id) + .fetch_optional(pool) + .await?; + if let Some((stored_hash, result, status)) = existing { + if stored_hash != hash { + return Ok(Begin::PayloadMismatch); + } + if status != "accepted" + && let Some(text) = result.filter(|value| !value.is_empty()) + { + return Ok(replay_from_stored(&text)); + } + return Ok(Begin::InProgress); + } + sqlx::query( + "INSERT INTO computer_operations (id, computer_id, bot_id, run_id, payload_hash, status) + VALUES ($1,$2,$3,$4,$5,'accepted')", + ) + .bind(key) + .bind(computer_id) + .bind(bot_id) + .bind(run_id) + .bind(hash) + .execute(pool) + .await?; + Ok(Begin::Proceed) +} + +fn replay_from_stored(stored: &str) -> Begin { + if let Ok(value) = serde_json::from_str::(stored) { + Begin::Replay { + text: value + .get("text") + .and_then(Value::as_str) + .unwrap_or(stored) + .to_string(), + error_code: value + .get("errorCode") + .and_then(Value::as_str) + .map(str::to_string), + pause: value.get("pause").and_then(Value::as_bool).unwrap_or(false), + } + } else { + Begin::Replay { + text: stored.to_string(), + error_code: None, + pause: false, + } + } +} + +/// Transport-level failures (`exec failed: …`, `write_file failed: …`) are +/// reported as prose by the native tools; storing them as `succeeded` would +/// replay the failure forever instead of letting the same operation retry. +fn outcome_status(outcome: &ToolOutcome) -> &'static str { + if outcome.error_code.is_some() { + return "failed"; + } + let head: String = outcome.text.chars().take(80).collect(); + if head.contains(" failed: ") { + return "failed"; + } + "succeeded" +} + +pub async fn finish( + ctx: &ToolCtx, + operation_id: &str, + name: &str, + args: &Value, + outcome: &ToolOutcome, +) { + if !mutating_tool(name, args) { + return; + } + let key = ledger_key(&ctx.bot_id, operation_id); + let status = outcome_status(outcome); + let text = redact_secret_patterns(&outcome.text); + // Transport failures leave no durable result: the same operation id may be + // retried, and `persist_begin` will let it proceed again. + let stored = if status == "failed" && outcome.error_code.is_none() { + None + } else { + Some( + json!({ + "text": text, + "errorCode": outcome.error_code, + "pause": outcome.pause, + }) + .to_string(), + ) + }; + let row_status = if stored.is_some() { + status + } else { + "retryable" + }; + if stored.is_some() { + let _ = sqlx::query( + "UPDATE computer_operations SET status=$3, result=$4 + WHERE id=$1 AND bot_id=$2 AND status='accepted'", + ) + .bind(&key) + .bind(&ctx.bot_id) + .bind(row_status) + .bind(&stored) + .execute(&ctx.pool) + .await; + } else { + // Drop the accepted row so a retry with the same id is not treated as + // in progress. + let _ = sqlx::query( + "DELETE FROM computer_operations WHERE id=$1 AND bot_id=$2 AND status='accepted'", + ) + .bind(&key) + .bind(&ctx.bot_id) + .execute(&ctx.pool) + .await; + } + let event_key = format!("tool:{key}"); + let mut payload = json!({ + "runId": ctx.run_id, + "botId": ctx.bot_id, + "name": name, + "operationId": operation_id, + "errorCode": outcome.error_code, + "snippet": crate::monitor::snippet(&text, 200), + }); + redact_json(&mut payload); + let _ = sqlx::query( + "INSERT INTO operation_outbox (id, event_key, payload) VALUES ($1,$2,$3) + ON CONFLICT (event_key) DO NOTHING", + ) + .bind(Uuid::new_v4().to_string()) + .bind(&event_key) + .bind(&payload) + .execute(&ctx.pool) + .await; +} + +pub async fn flush_outbox(pool: &PgPool) { + let rows: Vec<(String, String, Value)> = sqlx::query_as( + "SELECT id, event_key, payload FROM operation_outbox + WHERE delivered=false AND created_at < now() - interval '2 seconds' + ORDER BY created_at LIMIT 50", + ) + .fetch_all(pool) + .await + .unwrap_or_default(); + for (id, _key, payload) in rows { + if let Some(run_id) = payload.get("runId").and_then(Value::as_str) { + let operation_id = payload + .get("operationId") + .and_then(Value::as_str) + .unwrap_or(""); + // Row presence only; do not decode `SELECT 1` (INT4) into i64 — that + // errored on every row and re-recorded each operation. + let exists = sqlx::query( + "SELECT 1 FROM run_activity + WHERE run_id=$1 AND kind='tool' AND payload->>'operationId'=$2 + LIMIT 1", + ) + .bind(run_id) + .bind(operation_id) + .fetch_optional(pool) + .await; + match exists { + Ok(Some(_)) => {} + Ok(None) => { + // At-least-once: only a successful write may mark the row + // delivered; otherwise it stays for the next pass. + if crate::monitor::try_record_pool(pool, run_id, "tool", payload.clone()) + .await + .is_err() + { + continue; + } + } + // Cannot tell whether it was delivered: leave the row for the + // next pass instead of risking a duplicate or a lost event. + Err(_) => continue, + } + } + let _ = sqlx::query("UPDATE operation_outbox SET delivered=true WHERE id=$1") + .bind(id) + .execute(pool) + .await; + } +} + +async fn bot_computer_id(ctx: &ToolCtx) -> Result, sqlx::Error> { + let row: Option> = + sqlx::query_scalar("SELECT computer_id FROM bots WHERE id=$1") + .bind(&ctx.bot_id) + .fetch_optional(&ctx.pool) + .await?; + Ok(row.flatten()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn same_payload_hashes_equal() { + let args = json!({"path":"a.txt"}); + assert_eq!( + payload_hash("write_file", &args), + payload_hash("write_file", &args) + ); + assert_ne!( + payload_hash("write_file", &args), + payload_hash("write_file", &json!({"path":"b.txt"})) + ); + } + + #[test] + fn operation_id_is_not_part_of_the_payload_hash() { + let a = json!({"path":"a.txt","operationId":"op-1"}); + let b = json!({"path":"a.txt","operation_id":"op-2"}); + assert_eq!( + payload_hash("write_file", &a), + payload_hash("write_file", &b) + ); + } + + /// T17: the outbox only fills the gap; an operation already in the + /// activity trail must not be recorded a second time. + #[sqlx::test(migrations = "../../migrations")] + async fn outbox_flush_does_not_duplicate_recorded_operations(pool: sqlx::PgPool) { + for sql in [ + "INSERT INTO users(id,name) VALUES ('u','test')", + "INSERT INTO spaces(id,user_id,name) VALUES ('s','u','test')", + "INSERT INTO bots(id,space_id,user_id,name) VALUES ('b','s','u','bot')", + "INSERT INTO threads(id,space_id,user_id,bot_id) VALUES ('t','s','u','b')", + "INSERT INTO runs(id,space_id,user_id,bot_id,thread_id,status,trigger,prompt) + VALUES ('r','s','u','b','t','running','message','go')", + "INSERT INTO run_activity(run_id,kind,payload) + VALUES ('r','tool','{\"operationId\":\"op-seen\",\"tool\":\"write_file\"}')", + ] { + sqlx::query(sql).execute(&pool).await.unwrap(); + } + for (key, op) in [("k1", "op-seen"), ("k2", "op-missed"), ("k3", "op-fresh")] { + // k3 is brand new: the direct activity write may still be on its + // way, so the flush must leave it alone this pass. + let age = if key == "k3" { + "0 seconds" + } else { + "10 seconds" + }; + sqlx::query(&format!( + "INSERT INTO operation_outbox(id,event_key,payload,created_at) + VALUES ($1,$1,$2,now() - interval '{age}')" + )) + .bind(key) + .bind(json!({"runId":"r","operationId":op,"tool":"write_file"})) + .execute(&pool) + .await + .unwrap(); + } + + flush_outbox(&pool).await; + + let count = |op: &'static str| { + let pool = pool.clone(); + async move { + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM run_activity WHERE run_id='r' AND payload->>'operationId'=$1", + ) + .bind(op) + .fetch_one(&pool) + .await + .unwrap() + } + }; + assert_eq!(count("op-seen").await, 1, "already recorded: no duplicate"); + assert_eq!(count("op-missed").await, 1, "missing one is back-filled"); + assert_eq!(count("op-fresh").await, 0, "too new to judge: not touched"); + let pending: Vec = sqlx::query_scalar( + "SELECT event_key FROM operation_outbox WHERE delivered=false ORDER BY event_key", + ) + .fetch_all(&pool) + .await + .unwrap(); + assert_eq!(pending, vec!["k3".to_string()]); + } + + /// One bot's operation id must not replay another bot's result, and an + /// operation that never finished is not silently run a second time. + #[sqlx::test(migrations = "../../migrations")] + async fn ledger_is_per_bot_and_does_not_rerun_in_flight_work(pool: sqlx::PgPool) { + for sql in [ + "INSERT INTO users(id,name) VALUES ('u','test')", + "INSERT INTO spaces(id,user_id,name) VALUES ('s','u','test')", + ] { + sqlx::query(sql).execute(&pool).await.unwrap(); + } + // Tools that need no Computer are journaled with a NULL computer_id. + let hash = payload_hash("remember", &json!({"content":"x"})); + let first = persist_begin(&pool, &ledger_key("a", "op-1"), None, "a", "r", &hash) + .await + .unwrap(); + assert_eq!(first, Begin::Proceed); + // Same id, same payload, no result yet: unknown effect, do not re-run. + let again = persist_begin(&pool, &ledger_key("a", "op-1"), None, "a", "r", &hash) + .await + .unwrap(); + assert_eq!(again, Begin::InProgress); + // Bot b choosing the same operation id is a different operation. + let other = persist_begin(&pool, &ledger_key("b", "op-1"), None, "b", "r", &hash) + .await + .unwrap(); + assert_eq!(other, Begin::Proceed); + + sqlx::query( + "UPDATE computer_operations SET status='succeeded', + result='{\"text\":\"done\",\"pause\":true}' WHERE id=$1", + ) + .bind(ledger_key("a", "op-1")) + .execute(&pool) + .await + .unwrap(); + let replay = persist_begin(&pool, &ledger_key("a", "op-1"), None, "a", "r", &hash) + .await + .unwrap(); + assert_eq!( + replay, + Begin::Replay { + text: "done".into(), + error_code: None, + pause: true, + } + ); + let other_hash = payload_hash("remember", &json!({"content":"y"})); + let mismatch = persist_begin(&pool, &ledger_key("a", "op-1"), None, "a", "r", &other_hash) + .await + .unwrap(); + assert_eq!(mismatch, Begin::PayloadMismatch); + } + + #[test] + fn form_fill_and_computer_mcp_are_mutations() { + assert!(mutating_tool("form_fill", &json!({}))); + assert!(mutating_tool("computer_mcp", &json!({}))); + assert!(!mutating_tool("list_files", &json!({}))); + assert!(mutating_tool("browser", &json!({"action":"type"}))); + assert!(!mutating_tool("browser", &json!({"action":"snapshot"}))); + assert!(mutating_tool("exec", &json!({"argv":["ls"]}))); + assert!(mutating_tool( + "exec", + &json!({"action":"cancel","jobId":"j"}) + )); + assert!(!mutating_tool( + "exec", + &json!({"action":"status","jobId":"j"}) + )); + } + + #[test] + fn transport_failures_are_not_stored_as_success() { + let failed = ToolOutcome { + text: "write_file failed: connection reset".into(), + image: None, + pause: false, + blocks: Vec::new(), + error_code: None, + }; + assert_eq!(outcome_status(&failed), "failed"); + let ok = ToolOutcome { + text: "{\"ok\":true}".into(), + image: None, + pause: false, + blocks: Vec::new(), + error_code: None, + }; + assert_eq!(outcome_status(&ok), "succeeded"); + } +} diff --git a/crates/api/src/retention.rs b/crates/api/src/retention.rs index 1d161b1..6f86321 100644 --- a/crates/api/src/retention.rs +++ b/crates/api/src/retention.rs @@ -50,6 +50,16 @@ pub async fn retention_loop(state: AppState) { days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90), ), ("leases", include_str!("retention/leases.sql"), 7), + ( + "operation_outbox", + include_str!("retention/operation_outbox.sql"), + days("LAZYBOY_RUN_ACTIVITY_RETENTION_DAYS", 7), + ), + ( + "computer_operations", + include_str!("retention/computer_operations.sql"), + days("LAZYBOY_RUN_RETENTION_DAYS", 90), + ), ( "profile_locks", include_str!("retention/profile_locks.sql"), diff --git a/crates/api/src/retention/computer_operations.sql b/crates/api/src/retention/computer_operations.sql new file mode 100644 index 0000000..af1e6f4 --- /dev/null +++ b/crates/api/src/retention/computer_operations.sql @@ -0,0 +1,4 @@ +DELETE FROM computer_operations WHERE id IN ( + SELECT id FROM computer_operations WHERE status <> 'accepted' AND created_at < now() - make_interval(days => $1) + ORDER BY created_at LIMIT $2 FOR UPDATE SKIP LOCKED +) diff --git a/crates/api/src/retention/operation_outbox.sql b/crates/api/src/retention/operation_outbox.sql new file mode 100644 index 0000000..60a169c --- /dev/null +++ b/crates/api/src/retention/operation_outbox.sql @@ -0,0 +1,4 @@ +DELETE FROM operation_outbox WHERE id IN ( + SELECT id FROM operation_outbox WHERE delivered AND created_at < now() - make_interval(days => $1) + ORDER BY created_at LIMIT $2 FOR UPDATE SKIP LOCKED +) diff --git a/crates/api/src/routes.rs b/crates/api/src/routes.rs index 41c80a8..8508855 100644 --- a/crates/api/src/routes.rs +++ b/crates/api/src/routes.rs @@ -24,6 +24,8 @@ pub fn router(state: AppState) -> Router { .route("/api/file-skills", get(file_skills)) .merge(crate::vault::router()) .merge(crate::schedules::router()) + .merge(crate::artifacts::router()) + .merge(crate::tool_install::router()) .route("/api/bots", get(list_bots).post(create_bot)) .route( "/api/bots/{id}", @@ -44,6 +46,7 @@ pub fn router(state: AppState) -> Router { .route("/api/computer/{id}/takeover", post(takeover)) .route("/api/computer/{id}/release", post(release)) .route("/api/computer/{id}/heartbeat", post(heartbeat)) + .route("/api/computer/{id}/health", get(computer_health)) .route("/api/computer/{id}/input", post(input)) .route("/view/{id}/", any(crate::screen_proxy::view_root)) .route("/view/{id}/{*rest}", any(crate::screen_proxy::view_path)) @@ -370,6 +373,21 @@ async fn delete_bot( .execute(&mut *tx) .await .map_err(internal_error)?; + // Agent-computer rows keyed by bot_id have no FK to bots; without this they + // would outlive the bot as orphans (and a Team Computer would keep + // counting a deleted bot's jobs against its quota). + for table in [ + "tool_bindings", + "computer_jobs", + "computer_operations", + "computer_artifacts", + ] { + sqlx::query(&format!("DELETE FROM {table} WHERE bot_id = $1")) + .bind(&id) + .execute(&mut *tx) + .await + .map_err(internal_error)?; + } sqlx::query("DELETE FROM bots WHERE id = $1 AND space_id = $2 AND user_id = $3") .bind(&id) .bind(&actor.space_id) @@ -411,7 +429,7 @@ async fn delete_environment( "SELECT id, space_id, user_id, scope, scope_key, home_key, home_revision, kind, provider_ref, state, control_holder, control_lease_id, control_lease_expires_at, control_bot_id, control_run_id, execution_run_id, execution_bot_id, execution_lease_expires_at, execution_fence, - browser_profile_mode FROM computers WHERE space_id = $1 AND user_id = $2", + browser_profile_mode, generation FROM computers WHERE space_id = $1 AND user_id = $2", ) .bind(&id) .bind(&actor.user_id) @@ -688,6 +706,17 @@ async fn heartbeat( .map_err(|_| StatusCode::BAD_REQUEST) } +async fn computer_health( + State(state): State, + actor: Actor, + Path(id): Path, +) -> Result, StatusCode> { + computer::component_health(&state, &actor, &id) + .await + .map(Json) + .map_err(|_| StatusCode::NOT_FOUND) +} + #[derive(Deserialize)] struct InputBody { kind: String, diff --git a/crates/api/src/runs.rs b/crates/api/src/runs.rs index 8bf9828..0b59ba4 100644 --- a/crates/api/src/runs.rs +++ b/crates/api/src/runs.rs @@ -38,21 +38,21 @@ Be brief and friendly. If they ask who you are, say you can chat and also do wor const SYSTEM: &str = "You are this bot's assistant. You have a Linux desktop you can use, but most conversation does not need it. -Reply in text — no tools — for greetings, small talk, questions you can answer from knowledge, planning, or explaining. Do not call computer_observe, computer_act, browser, launch_app, open_path, wait, list_files, or shell just to check the screen or because a desktop exists. A hello does not need a screenshot or a file listing. +Reply in text — no tools — for greetings, small talk, questions you can answer from knowledge, planning, or explaining. Do not call computer_observe, computer_act, browser, launch_app, open_path, wait, list_files, exec, or shell just to check the screen or because a desktop exists. A hello does not need a screenshot or a file listing. Use tools only when the user wants something done on the computer: open a site, click through a UI, run a command, read/write workspace files, or follow a taught skill. Route directly by task: -1) Website, video, search, email, or anything in Chromium: use browser first. Navigate directly, then snapshot/click/type/press by element id. Do not use shell/curl to inspect a web page. For canvas or controls the browser tool cannot operate, call computer_observe and use Cua computer_act coordinates from that fresh screenshot. -2) Workspace files or commands: use list_files, read_file, write_file, or shell. -3) Connected services: use an MCP tool when it directly matches the task. +1) Website, video, search, or anything in Chromium: use browser first. Navigate directly, then snapshot/click/type/press by element id. Do not use shell/curl to inspect a web page. For canvas or controls the browser tool cannot operate, call computer_observe and use Cua computer_act coordinates from that fresh screenshot. Gmail/Outlook with an authorized connector use that API — never fall back to the browser after POLICY_DENIED. +2) Workspace files or commands: use list_files, read_file, write_file, or exec. Use shell only when the user asked to watch the visible GUI terminal. +3) Connected services: use an MCP tool when it directly matches the task. Packages installed on this Computer (computer_mcp) run inside the assigned container, not the API. Long web forms: prefer form_fill with locators from the latest snapshot; it stops on the first failed field. 4) Opening a local file or non-browser app: use open_path or launch_app. 5) Native GUI with no DOM (dialogs, file manager, XFCE): use computer_act by element id. Those ids are AT-SPI controls, not window boxes. -The shell is a visible Cua-controlled terminal on the shared VNC screen. The same session keeps its directory, exports and background jobs. Results are screenshots, not hidden stdout: inspect the prompt to decide whether a command finished. A timeout does not stop the job. Omit command to inspect it again, use keys \"C-c\" to interrupt, and never type a second command while busy. File tools also work through this visible terminal; read_file supports start_line and lines, and you can scroll to inspect longer output. Clicking, typing, and browsing need a vision model; a text-only model can still read computer_observe as an element tree. +exec returns real stdout/stderr/exit from the bound Computer. Prefer argv. The visible `shell` tool types into a Cua-controlled terminal on the shared VNC screen and returns screenshots — use it for TUI collaboration, not ordinary commands. A timeout on either tool does not prove the process stopped. File tools are native (text, hash, no screenshot); read_file supports start_line and lines, and binary files are not decoded as UTF-8. Pixel clicks, the visible GUI terminal, and opening desktop apps need a vision model. A text-only model can still read computer_observe as an element tree, drive Chromium through browser snapshot/click/type by element id, and use exec/list_files/read_file/write_file. When you ARE using the desktop: the human can interact with the same live screen while you work; this does not pause your task. Prefer browser/native element actions over moving the shared pointer. If the screen changes unexpectedly, observe again and continue from the current state; do not undo human changes or replay an uncertain click. Request human assistance only when the task needs it. Only the latest screenshot you received is current; they may have interacted since. Call computer_observe before coordinate clicks, after navigation, when the outcome is uncertain, and before describing what is on screen. Never guess the screen state from files, history or memory. Never kill or restart the browser, display, or desktop processes; if the browser tool reports it is unavailable, use computer_observe / computer_act on the existing window instead. When you use the browser tool: -- snapshot first; click {\"action\":\"click\",\"element\":N}; type {\"action\":\"type\",\"element\":N,\"text\":\"...\"}; open a URL with navigate. +- snapshot first; click {\"action\":\"click\",\"element\":N}; type {\"action\":\"type\",\"element\":N,\"text\":\"...\"}; open a URL with navigate. CSS selectors are not supported (SELECTOR_UNSUPPORTED); use the numbered snapshot ref. - Yellow numbered marks on the screenshot match the element list. Click the number, not guessed pixels. - Elements tagged [below viewport ...] / [above viewport ...] are outside the visible area but still clickable by id; the click scrolls to them. Do not scroll manually just to reach them. - A control is disabled only when its entry says [disabled]. Never claim a button is disabled, counting down or loading unless the element list or the screenshot shows that. @@ -335,6 +335,12 @@ type RetryCandidateRow = (String, String, String, String, String, String); pub async fn worker_loop(state: AppState) { let inflight = Arc::new(tokio::sync::Semaphore::new(16)); let lease_owner = format!("api-{}", Uuid::new_v4()); + let _ = sqlx::query( + "UPDATE computer_jobs SET status='interrupted', updated_at=now() + WHERE status IN ('running','accepted')", + ) + .execute(state.pool()) + .await; // Without a knock the loop sits on its 200 ms timer before it can see a run // that was queued a moment ago; going straight to the claim query is what // makes the thinking indicator follow the message instead of the timer. @@ -344,6 +350,7 @@ pub async fn worker_loop(state: AppState) { _ = wakes.wait_any() => {} _ = tokio::time::sleep(Duration::from_millis(200)) => {} } + crate::operations::flush_outbox(state.pool()).await; let interrupted: Vec<(String, String, String)> = sqlx::query_as( "WITH doomed AS ( UPDATE runs SET status='failed',error=$1,completed_at=now(), @@ -510,7 +517,7 @@ async fn execute_run( .map_err(|error| error.to_string())? .ok_or_else(|| "computer not found".to_string())?; - let (model, vision) = bot_model(state, actor, &bot, thread_id).await?; + let (model, vision, model_id) = bot_model(state, actor, &bot, thread_id).await?; let skills = crate::skills::saved_skills(state.pool(), bot_id).await; let ctx = Arc::new(ToolCtx { @@ -526,9 +533,15 @@ async fn execute_run( mode: parse_mode(&computer.scope), bot_id: bot_id.to_string(), vision, + model_id, gui_block: std::sync::Mutex::new(None), previous_frame: std::sync::Mutex::new(None), previous_signature: std::sync::Mutex::new(None), + delivered_frame: std::sync::Mutex::new(None), + delivered_signature: std::sync::Mutex::new(None), + delivered_model_id: std::sync::Mutex::new(None), + force_image: std::sync::Mutex::new(false), + pending_image_delivery: std::sync::Mutex::new(None), elements: std::sync::Mutex::new(Vec::new()), miss_streak: std::sync::Mutex::new(0), last_click_key: std::sync::Mutex::new(None), @@ -542,10 +555,26 @@ async fn execute_run( run_id: run_id.to_string(), memory_enabled: bot.memory_enabled && state.memory.globally_enabled(), mcp: state.mcp.clone(), + last_operation_id: std::sync::Mutex::new(None), }); let mut defs = tool_definitions(ctx.memory_enabled); - let mcp_defs = state.mcp.definitions().await; + let computer_mcp_bound = crate::tool_install::package_is_bound( + state.pool(), + actor, + bot_id, + crate::tool_install::ECHO_ID, + ) + .await + .unwrap_or(false); + let mcp_defs = state.mcp.definitions_for(actor).await; + let granted: Vec = mcp_defs.iter().map(|tool| tool.name.clone()).collect(); + let allowed = lazyboy_control::filter_run_tool_names( + defs.iter().map(|tool| tool.name.as_str()), + computer_mcp_bound, + &granted, + ); + defs.retain(|tool| allowed.iter().any(|name| name == &tool.name)); if !mcp_defs.is_empty() { defs.extend(mcp_defs); } @@ -761,12 +790,14 @@ async fn execute_run( prepare_run_computer(state, actor, bot_id, run_id, &ctx, true).await?; } if resume_after_takeover && ctx.gui_block.lock().unwrap().is_none() { + ctx.request_force_image(); let outcome = dispatch(&ctx, "computer_observe", &json!({})).await; first.push(UserContent::text(outcome.text)); if let Some(image) = outcome.image { screenshot_bytes += image.len() as u64; screenshots += 1; first.extend(screenshot_parts(image)); + ctx.commit_image_delivery(); } } let mut pending = Message::User { content: first }; @@ -785,6 +816,8 @@ async fn execute_run( content.push(UserContent::text("Resumed after a completed tool batch. Do not repeat completed actions. Observe current browser/desktop before any new mutation; prior element references may be stale.")); } pending = next; + // Checkpoints strip images, so the model no longer holds the last frame. + ctx.request_force_image(); } let mut final_text = String::new(); @@ -1044,6 +1077,9 @@ async fn execute_run( defs_chars, None, ); + if !history.iter().any(has_screenshot) && !has_screenshot(&pending) { + ctx.request_force_image(); + } if fit.compacted { tracing::info!( run_id, @@ -1227,6 +1263,7 @@ async fn execute_run( screenshot_bytes += image.len() as u64; screenshots += 1; content.extend(screenshot_parts(image)); + ctx.commit_image_delivery(); } } pending = Message::User { content }; @@ -1310,6 +1347,8 @@ async fn execute_run( | "wait" | "use_saved_login" | "request_takeover" + | "shell" + | "form_fill" ); did_work = true; let step = describe_step(&name, &call.function.arguments); @@ -1350,6 +1389,7 @@ async fn execute_run( image: None, pause: false, blocks: Vec::new(), + error_code: None, } } } @@ -1365,6 +1405,11 @@ async fn execute_run( "tool call" ); let status = tool_status(tool_timed_out, outcome.pause, &outcome.text); + // Background jobs answer with JSON carrying jobId; surface it so the + // monitor can show which job a status/cancel line refers to. + let job_id = serde_json::from_str::(&outcome.text) + .ok() + .and_then(|value| value.get("jobId")?.as_str().map(str::to_string)); crate::monitor::record( state, run_id, @@ -1374,6 +1419,9 @@ async fn execute_run( "name": name.clone(), "step": step.clone(), "status": status, + "errorCode": outcome.error_code, + "jobId": job_id, + "operationId": ctx.last_operation_id.lock().unwrap().clone(), "elapsedMs": tool_started.elapsed().as_millis() as u64, "snippet": crate::monitor::snippet(&outcome.text, 200), }), @@ -1498,6 +1546,7 @@ async fn execute_run( screenshot_bytes += png.len() as u64; screenshots += 1; results.extend(screenshot_parts(png)); + ctx.commit_image_delivery(); } pending = Message::User { content: results }; save_harness_checkpoint( @@ -1651,7 +1700,7 @@ pub(crate) async fn bot_model( actor: &Actor, bot: &crate::db::BotRow, session: &str, -) -> Result<(DynModel, bool), String> { +) -> Result<(DynModel, bool, String), String> { let space = state .db .get_space(actor) @@ -1685,7 +1734,7 @@ pub(crate) async fn bot_model( }) .map_err(|error| error.to_string())?; let model = connect_model(&backend, session).map_err(|error| error.to_string())?; - Ok((model, backend.capabilities.vision)) + Ok((model, backend.capabilities.vision, backend.model_id)) } pub(crate) async fn complete_once( @@ -3100,6 +3149,7 @@ fn tool_needs_sandbox(name: &str) -> bool { matches!( name, "shell" + | "exec" | "list_files" | "read_file" | "write_file" @@ -3112,6 +3162,8 @@ fn tool_needs_sandbox(name: &str) -> bool { | "wait" | "use_saved_login" | "request_takeover" + | "form_fill" + | "computer_mcp" ) } @@ -3119,9 +3171,6 @@ fn tool_needs_gui(name: &str) -> bool { matches!( name, "shell" - | "list_files" - | "read_file" - | "write_file" | "computer_observe" | "computer_act" | "browser" @@ -3131,6 +3180,7 @@ fn tool_needs_gui(name: &str) -> bool { | "wait" | "use_saved_login" | "request_takeover" + | "form_fill" ) } @@ -3163,7 +3213,7 @@ async fn prepare_run_computer( }; set_run_step(state, run_id, step).await; } - computer::boot(state, actor, bot_id).await?; + computer::boot_for(state, actor, bot_id, need_gui).await?; let computer = state .db .get_computer(bot.computer_id.as_deref().unwrap_or("")) @@ -3173,6 +3223,7 @@ async fn prepare_run_computer( let computer_ref = computer::computer_ref(&computer) .ok_or_else(|| "computer is not running".to_string())?; *ctx.computer.lock().unwrap() = Some(computer_ref); + ctx.context.lock().unwrap().computer_generation = Some(computer.generation); } if !need_gui || ctx.adapter().display.is_some() { return Ok(()); @@ -3213,6 +3264,7 @@ async fn prepare_run_computer( *ctx.gui_block.lock().unwrap() = gui_block; *ctx.context.lock().unwrap() = adapter_context_for(actor, bot_id, "run", screen.as_ref(), Some(run_id)); + ctx.context.lock().unwrap().computer_generation = Some(computer.generation); Ok(()) } @@ -3283,6 +3335,20 @@ fn describe_step(name: &str, args: &Value) -> String { .trim() .to_string() } + "exec" => { + if let Some(argv) = args.get("argv").and_then(Value::as_array) { + let line = argv + .iter() + .filter_map(Value::as_str) + .take(6) + .collect::>() + .join(" "); + short(Some(&line), 60) + } else { + short(get("command"), 60) + } + } + "list_files" | "read_file" | "write_file" | "list_dir" => short(get("path"), 40), "shell" => { // The terminal does four different things; the feed says which. let session = get("session").filter(|name| !name.trim().is_empty() && *name != "main"); @@ -3312,7 +3378,6 @@ fn describe_step(name: &str, args: &Value) -> String { .trim() .to_string(), "launch_app" | "open_path" => short(get("app").or(get("path")), 40), - "read_file" | "write_file" | "list_dir" => short(get("path"), 40), "use_skill" => format!("讀取技能 {}", short(get("name"), 30)), "use_saved_login" => "填入已存帳號".into(), "list_accounts" => "列出已存帳號".into(), @@ -3320,6 +3385,14 @@ fn describe_step(name: &str, args: &Value) -> String { "list_schedules" => "列出排程".into(), "cancel_schedule" => "取消排程".into(), "request_takeover" => short(get("site").or(get("reason")), 40), + "form_fill" => format!( + "{} fields", + args.get("fields") + .and_then(Value::as_array) + .map(|fields| fields.len()) + .unwrap_or(0) + ), + "computer_mcp" => short(get("packageId").or(get("text")), 40), _ => String::new(), }; if detail.is_empty() { @@ -3443,7 +3516,8 @@ fn action_changes_state(name: &str, args: &Value) -> bool { match name { "computer_act" | "shell" | "write_file" | "launch_app" | "open_path" | "create_schedule" | "cancel_schedule" | "remember" | "forget_memory" - | "use_saved_login" | "request_takeover" => true, + | "use_saved_login" | "request_takeover" | "form_fill" | "computer_mcp" => true, + "exec" => !matches!(args.get("action").and_then(Value::as_str), Some("status")), "browser" => matches!( args.get("action").and_then(Value::as_str), Some("click") | Some("type") | Some("navigate") | Some("press") @@ -3612,10 +3686,41 @@ mod tests { assert!(!tool_needs_sandbox("recall_memory")); assert!(!tool_needs_sandbox("use_skill")); assert!(tool_needs_sandbox("shell")); + assert!(tool_needs_sandbox("exec")); + assert!(tool_needs_sandbox("list_files")); assert!(tool_needs_gui("computer_observe")); assert!(tool_needs_gui("browser")); assert!(tool_needs_gui("shell")); - assert!(tool_needs_gui("list_files")); + assert!(!tool_needs_gui("list_files")); + assert!(!tool_needs_gui("read_file")); + assert!(!tool_needs_gui("write_file")); + assert!(!tool_needs_gui("exec")); + assert!(tool_needs_sandbox("form_fill")); + assert!(tool_needs_gui("form_fill")); + assert!(tool_needs_sandbox("computer_mcp")); + assert!(!tool_needs_gui("computer_mcp")); + } + + #[test] + fn native_tools_do_not_boot_the_desktop() { + for name in [ + "exec", + "list_files", + "read_file", + "write_file", + "computer_mcp", + ] { + assert!(tool_needs_sandbox(name), "{name}"); + assert!(!tool_needs_gui(name), "{name}"); + } + assert!(!lazyboy_control::viewer_blocks_native()); + assert!(!lazyboy_control::desktop_required_for_native()); + assert!(lazyboy_control::native_work_allowed( + lazyboy_control::ComponentReadiness { + runner: true, + ..lazyboy_control::ComponentReadiness::default() + } + )); } #[test] diff --git a/crates/api/src/skills.rs b/crates/api/src/skills.rs index 3707f52..9e0cc86 100644 --- a/crates/api/src/skills.rs +++ b/crates/api/src/skills.rs @@ -1270,7 +1270,7 @@ async fn distill( frames: &[Value], dir: &std::path::Path, ) -> Result { - let (model, vision) = crate::runs::bot_model(state, actor, bot, skill_id).await?; + let (model, vision, _) = crate::runs::bot_model(state, actor, bot, skill_id).await?; let t0 = events .iter() .chain(frames.iter()) diff --git a/crates/api/src/tool_install.rs b/crates/api/src/tool_install.rs new file mode 100644 index 0000000..3f84c63 --- /dev/null +++ b/crates/api/src/tool_install.rs @@ -0,0 +1,786 @@ +//! Install a reviewed local package onto the bound Computer and bind it to a bot. +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use lazyboy_contracts::ComputerMode; +use lazyboy_control::{ + AdapterContext, CommandRequest, ComputerRef, ReadyBinding, ReadyBindingChange, RollbackChange, + SandboxProvider, ToolManifest, artifact_digest, package_gc_allowed, pin_running_jobs, + plan_version_switch, previous_version, rollback_one_ready_binding, upsert_one_ready_binding, + validate_manifest, +}; +use serde::Deserialize; +use serde_json::{Value, json}; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::computer::{self, adapter_context}; +use crate::db::{Actor, parse_mode}; +use crate::state::AppState; + +const ECHO_SOURCE: &str = include_str!("../../../scripts/sample-mcp/echo_server.py"); +pub const ECHO_ID: &str = "lazyboy.example.echo"; +pub const ECHO_VERSION: &str = "0.0.1"; +pub const ECHO_VERSION_NEXT: &str = "0.0.2"; + +type ApiError = (StatusCode, Json); + +pub fn router() -> Router { + Router::new() + .route("/api/bots/{id}/tools", get(list_tools).post(install_sample)) + .route("/api/bots/{id}/tools/call", post(call_installed)) + .route( + "/api/bots/{id}/tools/{binding_id}/revoke", + post(revoke_binding), + ) + .route( + "/api/bots/{id}/tools/{binding_id}/rollback", + post(rollback_binding), + ) + .route( + "/api/bots/{id}/tools/{binding_id}/remove", + post(remove_binding), + ) +} + +#[derive(Deserialize)] +struct InstallBody { + #[serde(default, rename = "packageId", alias = "package_id")] + pub package_id: Option, + #[serde(default)] + pub version: Option, +} + +/// Immutable packages live on the Computer, not in the bot workspace folder. +pub fn echo_relative_path(mode: ComputerMode, version: &str) -> String { + match mode { + ComputerMode::Team => format!("shared/tools/{ECHO_ID}/{version}/echo_server.py"), + ComputerMode::Dedicated => format!("tools/{ECHO_ID}/{version}/echo_server.py"), + } +} + +fn echo_version(requested: Option<&str>) -> Result<&str, ApiError> { + match requested.unwrap_or(ECHO_VERSION) { + ECHO_VERSION => Ok(ECHO_VERSION), + ECHO_VERSION_NEXT => Ok(ECHO_VERSION_NEXT), + other => Err(( + StatusCode::BAD_REQUEST, + Json(json!({"message": format!("unsupported echo version {other}")})), + )), + } +} + +pub fn tool_install_enabled() -> bool { + std::env::var("LAZYBOY_TOOL_INSTALL_ENABLED") + .map(|value| value != "false" && value != "0") + .unwrap_or(true) +} + +async fn list_tools( + State(state): State, + actor: Actor, + Path(bot_id): Path, +) -> Result, ApiError> { + let rows: Vec<(String, String, String, String, String, String)> = sqlx::query_as( + "SELECT b.id, p.package_id, p.version, p.sha256, p.status, b.status + FROM tool_bindings b + JOIN tool_packages p ON p.id=b.package_row_id + JOIN bots bot ON bot.id=b.bot_id + WHERE b.bot_id=$1 AND bot.space_id=$2 AND bot.user_id=$3 + ORDER BY b.created_at", + ) + .bind(&bot_id) + .bind(&actor.space_id) + .bind(&actor.user_id) + .fetch_all(state.pool()) + .await + .map_err(internal)?; + Ok(Json(json!({ + "tools": rows.into_iter().map(|(id, pkg, ver, sha, install, bind)| json!({ + "bindingId": id, + "packageId": pkg, + "version": ver, + "sha256": sha, + "installStatus": install, + "bindingStatus": bind, + "ready": install == "installed" && bind == "ready", + "executionLocation": "assigned_computer", + })).collect::>() + }))) +} + +async fn install_sample( + State(state): State, + actor: Actor, + Path(bot_id): Path, + Json(body): Json, +) -> Result, ApiError> { + if !tool_install_enabled() { + return Err(( + StatusCode::FORBIDDEN, + Json(json!({"message":"tool install is disabled"})), + )); + } + let package_id = body.package_id.unwrap_or_else(|| ECHO_ID.into()); + if package_id != ECHO_ID { + return Err(( + StatusCode::BAD_REQUEST, + Json( + json!({"message":"only lazyboy.example.echo is installable without a reviewed artifact"}), + ), + )); + } + let version = echo_version(body.version.as_deref())?.to_string(); + let digest = artifact_digest(ECHO_SOURCE.as_bytes()); + let manifest = ToolManifest { + id: ECHO_ID.into(), + version: version.clone(), + sha256: digest.clone(), + entrypoint: vec!["python3".into(), "./echo_server.py".into()], + share_immutable_package: true, + }; + validate_manifest(&manifest).map_err(|error| { + ( + StatusCode::BAD_REQUEST, + Json(json!({"message": format!("{error:?}")})), + ) + })?; + let bot = state + .db + .get_bot(&actor, &bot_id) + .await + .map_err(internal)? + .ok_or(( + StatusCode::NOT_FOUND, + Json(json!({"message":"bot not found"})), + ))?; + computer::boot_for(&state, &actor, &bot_id, false) + .await + .map_err(|error| (StatusCode::BAD_GATEWAY, Json(json!({"message": error}))))?; + let computer = state + .db + .get_computer(bot.computer_id.as_deref().unwrap_or("")) + .await + .map_err(internal)? + .ok_or(( + StatusCode::NOT_FOUND, + Json(json!({"message":"computer not found"})), + ))?; + let computer_ref = computer::computer_ref(&computer).ok_or(( + StatusCode::CONFLICT, + Json(json!({"message":"computer is not running"})), + ))?; + let mode = parse_mode(&computer.scope); + let relative = echo_relative_path(mode, &version); + state + .sandbox + .write_file( + &computer_ref, + &relative, + ECHO_SOURCE.as_bytes(), + &adapter_context(&actor, &bot_id, "tool-install"), + ) + .await + .map_err(|error| { + ( + StatusCode::BAD_GATEWAY, + Json(json!({"message": error.to_string()})), + ) + })?; + let package_row = Uuid::new_v4().to_string(); + sqlx::query( + "INSERT INTO tool_packages (id, computer_id, package_id, version, sha256, status) + VALUES ($1,$2,$3,$4,$5,'installed') + ON CONFLICT (computer_id, package_id, version) DO UPDATE SET status='installed', sha256=EXCLUDED.sha256", + ) + .bind(&package_row) + .bind(&computer.id) + .bind(ECHO_ID) + .bind(&version) + .bind(&digest) + .execute(state.pool()) + .await + .map_err(internal)?; + let row_id: String = sqlx::query_scalar( + "SELECT id FROM tool_packages WHERE computer_id=$1 AND package_id=$2 AND version=$3", + ) + .bind(&computer.id) + .bind(ECHO_ID) + .bind(&version) + .fetch_one(state.pool()) + .await + .map_err(internal)?; + let existing: Option<(String, String, String)> = sqlx::query_as( + "SELECT b.id, b.package_row_id, p.version FROM tool_bindings b + JOIN tool_packages p ON p.id=b.package_row_id + WHERE b.bot_id=$1 AND p.package_id=$2 AND b.status='ready' + ORDER BY b.created_at DESC LIMIT 1", + ) + .bind(&bot_id) + .bind(ECHO_ID) + .fetch_optional(state.pool()) + .await + .map_err(internal)?; + let jobs: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM computer_jobs WHERE computer_id=$1 AND status='running'", + ) + .bind(&computer.id) + .fetch_one(state.pool()) + .await + .unwrap_or(0); + let current_version = existing + .as_ref() + .map(|row| row.2.as_str()) + .unwrap_or(&version); + let plan = plan_version_switch(current_version, &version, jobs.max(0) as usize); + // Only this bot's jobs were started with the version being replaced; other + // bots sharing the Computer keep their own bindings. + if let Some(pin) = pin_running_jobs(current_version, &version, jobs.max(0) as usize) { + let _ = sqlx::query( + "UPDATE computer_jobs SET pin_version=$2 + WHERE computer_id=$1 AND bot_id=$3 AND status='running' AND pin_version IS NULL", + ) + .bind(&computer.id) + .bind(&pin) + .bind(&bot_id) + .execute(state.pool()) + .await; + } + let ready = existing + .as_ref() + .map(|(id, package_row_id, ver)| ReadyBinding { + id: id.clone(), + package_row_id: package_row_id.clone(), + version: ver.clone(), + }); + let change = upsert_one_ready_binding( + ready.as_ref(), + &row_id, + &version, + &Uuid::new_v4().to_string(), + ); + let binding_id = match &change { + ReadyBindingChange::Insert { + id, package_row_id, .. + } => { + // (package_row_id, bot_id) is unique: reinstalling after a revoke + // revives the old row instead of failing on the constraint. + let binding_id: String = sqlx::query_scalar( + "INSERT INTO tool_bindings (id, package_row_id, bot_id, status) + VALUES ($1,$2,$3,'ready') + ON CONFLICT (package_row_id, bot_id) DO UPDATE SET status='ready' + RETURNING id", + ) + .bind(id) + .bind(package_row_id) + .bind(&bot_id) + .fetch_one(state.pool()) + .await + .map_err(internal)?; + binding_id + } + ReadyBindingChange::Switch { + id, package_row_id, .. + } => { + switch_binding(state.pool(), id, package_row_id, &bot_id) + .await + .map_err(internal)?; + id.clone() + } + ReadyBindingChange::Keep { id } => id.clone(), + }; + Ok(Json(json!({ + "packageId": ECHO_ID, + "version": version, + "bindingId": binding_id, + "sha256": digest, + "path": relative, + "installStatus": "installed", + "bindingStatus": "ready", + "jobPin": plan.job_pin, + "newRuns": plan.new_runs, + "executionLocation": "assigned_computer", + }))) +} + +/// Point a ready binding at another package row. A revoked leftover for the +/// same (package_row, bot) would trip the unique index, so it is dropped first. +async fn switch_binding( + pool: &PgPool, + binding_id: &str, + package_row_id: &str, + bot_id: &str, +) -> Result<(), sqlx::Error> { + let mut tx = pool.begin().await?; + sqlx::query( + "DELETE FROM tool_bindings + WHERE package_row_id=$1 AND bot_id=$2 AND status<>'ready' AND id<>$3", + ) + .bind(package_row_id) + .bind(bot_id) + .bind(binding_id) + .execute(&mut *tx) + .await?; + sqlx::query("UPDATE tool_bindings SET package_row_id=$2, status='ready' WHERE id=$1") + .bind(binding_id) + .bind(package_row_id) + .execute(&mut *tx) + .await?; + tx.commit().await +} + +#[derive(Deserialize)] +struct CallBody { + pub text: String, + #[serde(default, rename = "packageId")] + pub package_id: Option, +} + +async fn call_installed( + State(state): State, + actor: Actor, + Path(bot_id): Path, + Json(body): Json, +) -> Result, ApiError> { + let package_id = body.package_id.as_deref().unwrap_or(ECHO_ID); + if package_id != ECHO_ID { + return Err(( + StatusCode::BAD_REQUEST, + Json(json!({"message":"only lazyboy.example.echo can be called via this fixture"})), + )); + } + if !package_is_bound(state.pool(), &actor, &bot_id, package_id) + .await + .map_err(internal)? + { + return Err(( + StatusCode::FORBIDDEN, + Json(json!({"message":"package not bound to this agent"})), + )); + } + let bot = state + .db + .get_bot(&actor, &bot_id) + .await + .map_err(internal)? + .ok_or(( + StatusCode::NOT_FOUND, + Json(json!({"message":"bot not found"})), + ))?; + let computer = state + .db + .get_computer(bot.computer_id.as_deref().unwrap_or("")) + .await + .map_err(internal)? + .ok_or(( + StatusCode::NOT_FOUND, + Json(json!({"message":"computer not found"})), + ))?; + let computer_ref = computer::computer_ref(&computer).ok_or(( + StatusCode::CONFLICT, + Json(json!({"message":"computer is not running"})), + ))?; + let bound = bound_echo_relative(state.pool(), &actor, &bot_id, parse_mode(&computer.scope)) + .await + .map_err(internal)? + .ok_or(( + StatusCode::FORBIDDEN, + Json(json!({"message":"package not bound to this agent"})), + ))?; + let result = exec_echo( + state.sandbox.as_ref(), + &computer_ref, + &adapter_context(&actor, &bot_id, "computer-mcp"), + &bound, + &body.text, + ) + .await + .map_err(|error| (StatusCode::BAD_GATEWAY, Json(json!({"message": error}))))?; + Ok(Json(json!({ + "exitCode": result.code, + "stdout": result.stdout, + "stderr": result.stderr, + "executionLocation": "assigned_computer", + }))) +} + +/// The bound package as it must exist on the Computer: where it lives and the +/// digest the installer recorded for it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BoundPackage { + pub relative: String, + pub sha256: String, +} + +pub async fn bound_echo_relative( + pool: &PgPool, + actor: &Actor, + bot_id: &str, + mode: ComputerMode, +) -> Result, sqlx::Error> { + let row: Option<(String, String)> = sqlx::query_as( + "SELECT p.version, p.sha256 FROM tool_bindings b + JOIN tool_packages p ON p.id=b.package_row_id + JOIN bots bot ON bot.id=b.bot_id + WHERE b.bot_id=$1 AND bot.space_id=$2 AND bot.user_id=$3 + AND p.package_id=$4 AND b.status='ready' AND p.status='installed' + ORDER BY b.created_at DESC LIMIT 1", + ) + .bind(bot_id) + .bind(&actor.space_id) + .bind(&actor.user_id) + .bind(ECHO_ID) + .fetch_optional(pool) + .await?; + Ok(row.map(|(version, sha256)| BoundPackage { + relative: echo_relative_path(mode, &version), + sha256, + })) +} + +/// Run the package only if the file on the Computer still hashes to what was +/// installed. The check and the run are one command so nothing can swap the +/// file in between; the digest and path travel as argv, never interpolated. +pub fn verified_run_argv(absolute_path: &str, sha256: &str) -> Vec { + vec![ + "bash".into(), + "-c".into(), + r#"printf '%s %s\n' "$1" "$2" | sha256sum -c --status || { echo "package digest mismatch: $2" >&2; exit 97; }; exec python3 "$2""#.into(), + "verify".into(), + sha256.into(), + absolute_path.into(), + ] +} + +pub async fn package_is_bound( + pool: &PgPool, + actor: &Actor, + bot_id: &str, + package_id: &str, +) -> Result { + let bound: Option<(String,)> = sqlx::query_as( + "SELECT p.id FROM tool_bindings b + JOIN tool_packages p ON p.id=b.package_row_id + JOIN bots bot ON bot.id=b.bot_id + WHERE b.bot_id=$1 AND bot.space_id=$2 AND bot.user_id=$3 + AND p.package_id=$4 AND b.status='ready' AND p.status='installed' + LIMIT 1", + ) + .bind(bot_id) + .bind(&actor.space_id) + .bind(&actor.user_id) + .bind(package_id) + .fetch_optional(pool) + .await?; + Ok(bound.is_some()) +} + +pub async fn exec_echo( + sandbox: &dyn SandboxProvider, + computer_ref: &ComputerRef, + adapter: &AdapterContext, + bound: &BoundPackage, + text: &str, +) -> Result { + let rpc = json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"arguments": {"text": text}}, + }); + sandbox + .execute( + computer_ref, + CommandRequest { + argv: verified_run_argv( + &format!("/home/lazyboy/{}", bound.relative), + &bound.sha256, + ), + cwd: None, + timeout_ms: Some(10_000), + stdin: Some(format!("{rpc}\n")), + ..CommandRequest::default() + }, + adapter, + ) + .await + .map_err(|error| error.to_string()) +} + +async fn revoke_binding( + State(state): State, + actor: Actor, + Path((bot_id, binding_id)): Path<(String, String)>, +) -> Result, ApiError> { + let done = sqlx::query( + "UPDATE tool_bindings b SET status='revoked' + FROM bots bot + WHERE b.id=$1 AND b.bot_id=$2 AND bot.id=b.bot_id + AND bot.space_id=$3 AND bot.user_id=$4", + ) + .bind(&binding_id) + .bind(&bot_id) + .bind(&actor.space_id) + .bind(&actor.user_id) + .execute(state.pool()) + .await + .map_err(internal)?; + if done.rows_affected() == 0 { + return Err(( + StatusCode::NOT_FOUND, + Json(json!({"message":"binding not found"})), + )); + } + Ok(Json(json!({"ok": true}))) +} + +async fn rollback_binding( + State(state): State, + actor: Actor, + Path((bot_id, binding_id)): Path<(String, String)>, +) -> Result, ApiError> { + let row: Option<(String, String, String, String)> = sqlx::query_as( + "SELECT p.computer_id, p.package_id, p.version, b.status + FROM tool_bindings b + JOIN tool_packages p ON p.id=b.package_row_id + JOIN bots bot ON bot.id=b.bot_id + WHERE b.id=$1 AND b.bot_id=$2 AND bot.space_id=$3 AND bot.user_id=$4", + ) + .bind(&binding_id) + .bind(&bot_id) + .bind(&actor.space_id) + .bind(&actor.user_id) + .fetch_optional(state.pool()) + .await + .map_err(internal)?; + let Some((computer_id, package_id, current, binding_status)) = row else { + return Err(( + StatusCode::NOT_FOUND, + Json(json!({"message":"binding not found"})), + )); + }; + // Rolling back a revoked binding would silently re-enable the package. + if binding_status != "ready" { + return Err(( + StatusCode::CONFLICT, + Json(json!({"message": format!("binding is {binding_status}; install again instead")})), + )); + } + let versions: Vec = sqlx::query_scalar( + "SELECT version FROM tool_packages + WHERE computer_id=$1 AND package_id=$2 AND status='installed' + ORDER BY created_at", + ) + .bind(&computer_id) + .bind(&package_id) + .fetch_all(state.pool()) + .await + .map_err(internal)?; + let refs: Vec<&str> = versions.iter().map(String::as_str).collect(); + let Some(prev) = previous_version(&refs, ¤t) else { + return Err(( + StatusCode::CONFLICT, + Json(json!({"message":"no previous version to rollback"})), + )); + }; + let prev_row: String = sqlx::query_scalar( + "SELECT id FROM tool_packages WHERE computer_id=$1 AND package_id=$2 AND version=$3", + ) + .bind(&computer_id) + .bind(&package_id) + .bind(prev) + .fetch_one(state.pool()) + .await + .map_err(internal)?; + let leftover: Option<(String,)> = sqlx::query_as( + "SELECT id FROM tool_bindings + WHERE package_row_id=$1 AND bot_id=$2 AND status='ready' AND id<>$3", + ) + .bind(&prev_row) + .bind(&bot_id) + .bind(&binding_id) + .fetch_optional(state.pool()) + .await + .map_err(internal)?; + let current_binding = ReadyBinding { + id: binding_id.clone(), + package_row_id: String::new(), + version: current.clone(), + }; + match rollback_one_ready_binding( + ¤t_binding, + leftover.is_some(), + Some(&prev_row), + Some(prev), + ) { + RollbackChange::NoPrevious => { + return Err(( + StatusCode::CONFLICT, + Json(json!({"message":"no previous version to rollback"})), + )); + } + RollbackChange::Conflict => { + sqlx::query( + "UPDATE tool_bindings SET status='revoked' + WHERE package_row_id=$1 AND bot_id=$2 AND status='ready' AND id<>$3", + ) + .bind(&prev_row) + .bind(&bot_id) + .bind(&binding_id) + .execute(state.pool()) + .await + .map_err(internal)?; + switch_binding(state.pool(), &binding_id, &prev_row, &bot_id) + .await + .map_err(internal)?; + } + RollbackChange::Switch { + id, package_row_id, .. + } => { + switch_binding(state.pool(), &id, &package_row_id, &bot_id) + .await + .map_err(internal)?; + } + } + Ok(Json(json!({ + "ok": true, + "version": prev, + "previous": current, + "executionLocation": "assigned_computer", + }))) +} + +async fn remove_binding( + State(state): State, + actor: Actor, + Path((bot_id, binding_id)): Path<(String, String)>, +) -> Result, ApiError> { + let row: Option<(String, String)> = sqlx::query_as( + "SELECT p.id, p.computer_id + FROM tool_bindings b + JOIN tool_packages p ON p.id=b.package_row_id + JOIN bots bot ON bot.id=b.bot_id + WHERE b.id=$1 AND b.bot_id=$2 AND bot.space_id=$3 AND bot.user_id=$4", + ) + .bind(&binding_id) + .bind(&bot_id) + .bind(&actor.space_id) + .bind(&actor.user_id) + .fetch_optional(state.pool()) + .await + .map_err(internal)?; + let Some((package_row, computer_id)) = row else { + return Err(( + StatusCode::NOT_FOUND, + Json(json!({"message":"binding not found"})), + )); + }; + sqlx::query( + "UPDATE tool_bindings b SET status='revoked' + FROM bots bot + WHERE b.id=$1 AND b.bot_id=$2 AND bot.id=b.bot_id + AND bot.space_id=$3 AND bot.user_id=$4", + ) + .bind(&binding_id) + .bind(&bot_id) + .bind(&actor.space_id) + .bind(&actor.user_id) + .execute(state.pool()) + .await + .map_err(internal)?; + let ready: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM tool_bindings WHERE package_row_id=$1 AND status='ready'", + ) + .bind(&package_row) + .fetch_one(state.pool()) + .await + .unwrap_or(0); + let jobs: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM computer_jobs WHERE computer_id=$1 AND status='running'", + ) + .bind(&computer_id) + .fetch_one(state.pool()) + .await + .unwrap_or(0); + let removed = package_gc_allowed(ready.max(0) as usize, jobs.max(0) as usize); + if removed { + sqlx::query("UPDATE tool_packages SET status='removed' WHERE id=$1") + .bind(&package_row) + .execute(state.pool()) + .await + .map_err(internal)?; + } + Ok(Json(json!({ + "ok": true, + "packageRemoved": removed, + "executionLocation": "assigned_computer", + }))) +} + +fn internal(error: E) -> ApiError { + tracing::error!("tool install: {error}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"message":"internal error"})), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn team_packages_live_on_the_shared_tree() { + assert_eq!( + echo_relative_path(ComputerMode::Team, ECHO_VERSION), + "shared/tools/lazyboy.example.echo/0.0.1/echo_server.py" + ); + assert_eq!( + echo_relative_path(ComputerMode::Dedicated, ECHO_VERSION_NEXT), + "tools/lazyboy.example.echo/0.0.2/echo_server.py" + ); + } + + /// A package whose bytes no longer match the installed digest must not run. + #[test] + fn tampered_package_is_refused_before_it_runs() { + let dir = std::env::temp_dir().join(format!("lazyboy-verify-{}", Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("echo_server.py"); + std::fs::write(&path, "print('ok')\n").unwrap(); + let path = path.to_string_lossy().into_owned(); + let good = artifact_digest(b"print('ok')\n"); + + let run = |digest: &str| { + let argv = verified_run_argv(&path, digest); + std::process::Command::new(&argv[0]) + .args(&argv[1..]) + .output() + .unwrap() + }; + let ok = run(&good); + assert_eq!(ok.status.code(), Some(0), "{ok:?}"); + assert_eq!(String::from_utf8_lossy(&ok.stdout).trim(), "ok"); + + let bad = run(&artifact_digest(b"something else")); + assert_eq!(bad.status.code(), Some(97), "{bad:?}"); + assert!(bad.stdout.is_empty(), "tampered package must not execute"); + assert!(String::from_utf8_lossy(&bad.stderr).contains("digest mismatch")); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn agent_computer_migration_does_not_unique_bot_computer_id() { + let sql = include_str!("../../../migrations/022_agent_computer.sql"); + assert!(sql.contains("do NOT add UNIQUE(bots.computer_id)")); + let statements: String = sql + .lines() + .filter(|line| !line.trim_start().starts_with("--")) + .collect::>() + .join("\n") + .to_ascii_lowercase(); + assert!(!statements.contains("unique (bots.computer_id)")); + assert!(!statements.contains("unique(bots.computer_id)")); + } +} diff --git a/crates/api/src/tools.rs b/crates/api/src/tools.rs index 3a2510e..be1c936 100644 --- a/crates/api/src/tools.rs +++ b/crates/api/src/tools.rs @@ -1,14 +1,20 @@ use std::sync::Mutex; +use base64::Engine; use lazyboy_contracts::{ ComputerAction, ComputerMode, ComputerObservation, PointerType, UiElement, }; +use sha2::{Digest, Sha256}; + use lazyboy_control::{ ActionDecision, ActionError, ActionRequest, ActionVerdict, AdapterContext, BrowserPage, - BrowserRequest, ComputerRef, SandboxProvider, ScreenChange, apply_element_targets, - click_fingerprint, element_id, format_ui_element_lines, format_ui_elements, frame_signature, - merge_ui_elements, overlay_elements, parse_computer_actions, resolve_bot_workspace_cwd, - resolve_bot_workspace_path, screen_change_between, should_block_stale_click, + BrowserRequest, CommandRequest, ComputerRef, ERROR_TARGET_NOT_FOUND, ERROR_TARGET_STALE, + FileEntry, FormField, SandboxProvider, ScreenChange, apply_element_targets, + check_expected_hash, click_fingerprint, element_id, format_ui_element_lines, + format_ui_elements, frame_signature, job_quota_exceeded, merge_ui_elements, + native_job_concurrency, overlay_elements, parse_computer_actions, resolve_bot_workspace_cwd, + resolve_bot_workspace_path, run_form_macro, screen_change_between, should_block_stale_click, + should_deliver_observation_image, }; use rig_core::completion::ToolDefinition; use serde_json::{Value, json}; @@ -26,12 +32,22 @@ pub struct ToolCtx { pub mode: ComputerMode, pub bot_id: String, pub vision: bool, + /// Provider model id for this run. Compared against `delivered_model_id` + /// so a vision-model switch re-delivers the current frame. + pub model_id: String, pub gui_block: Mutex>, pub previous_frame: Mutex>, - /// Coarse signature of the previous capture. `frame_id` is a sha256, so on + /// Coarse signature of the previous *capture*. `frame_id` is a sha256, so on /// a live desktop every panel clock tick is a "new" frame; the signature is /// what makes "nothing actually happened" detectable. pub previous_signature: Mutex>>, + /// Last frame that was actually attached to a model request. Independent of + /// capture: a background observe must not pretend the model has seen it. + pub delivered_frame: Mutex>, + pub delivered_signature: Mutex>>, + pub delivered_model_id: Mutex>, + pub force_image: Mutex, + pub(crate) pending_image_delivery: Mutex>)>>, pub elements: Mutex>, pub miss_streak: Mutex, pub last_click_key: Mutex>, @@ -45,6 +61,7 @@ pub struct ToolCtx { pub run_id: String, pub memory_enabled: bool, pub mcp: McpHub, + pub last_operation_id: Mutex>, } impl ToolCtx { @@ -59,13 +76,29 @@ impl ToolCtx { pub fn adapter(&self) -> AdapterContext { self.context.lock().unwrap().clone() } + + pub fn request_force_image(&self) { + *self.force_image.lock().unwrap() = true; + } + + /// Record that `outcome.image` was attached to the model request. Capture + /// state is updated on every observe; delivered state only moves here. + pub fn commit_image_delivery(&self) { + let Some((frame_id, signature)) = self.pending_image_delivery.lock().unwrap().take() else { + return; + }; + *self.delivered_frame.lock().unwrap() = Some(frame_id); + *self.delivered_signature.lock().unwrap() = signature; + *self.delivered_model_id.lock().unwrap() = Some(self.model_id.clone()); + *self.force_image.lock().unwrap() = false; + } } pub fn tool_definitions(memory_enabled: bool) -> Vec { let mut definitions = vec![ ToolDefinition { name: "computer_observe".into(), - description: "Capture a fresh desktop screenshot plus numbered targets (page DOM when Chromium is open, otherwise AT-SPI buttons/fields, otherwise windows). Only when the user asked you to do something on the computer. Not for greetings or chat. File tools already return their visible terminal screenshot. The image attaches only if the screen changed.".into(), + description: "Capture a fresh desktop screenshot plus numbered targets (page DOM when Chromium is open, otherwise AT-SPI buttons/fields, otherwise windows). Only when the user asked you to do something on the computer. Not for greetings or chat. Native file/exec tools return text, not screenshots. A vision model receives the image when the frame is new to it (or after takeover / model change / history compression); a text-only model never receives an image.".into(), parameters: json!({"type":"object","properties":{}}), }, ToolDefinition { @@ -106,7 +139,7 @@ pub fn tool_definitions(memory_enabled: bool) -> Vec { }, ToolDefinition { name: "shell".into(), - description: "Use Cua to type in a visible persistent terminal on the shared VNC desktop. The same session keeps directory, environment and background jobs. Results are screenshots: inspect the prompt before sending another command; do not type while a command is busy. Omit command to inspect the terminal, or send keys C-c to interrupt. Use computer_act to scroll through output. Not for greetings or questions that need no computer.".into(), + description: "Type in a visible persistent terminal on the shared VNC desktop when the user asked to watch a TUI or to collaborate in the GUI terminal. Prefer exec for ordinary commands (real stdout/stderr/exit, no screenshot). The same GUI session keeps directory, environment and background jobs. Results are screenshots: inspect the prompt before sending another command. Omit command to inspect the terminal, or send keys C-c to interrupt.".into(), parameters: json!({ "type":"object", "properties":{ @@ -121,23 +154,44 @@ pub fn tool_definitions(memory_enabled: bool) -> Vec { }, ToolDefinition { name: "list_files".into(), - description: "List files in this bot's home when the user asked about workspace files. Not for greetings or chat — do not ls the desktop to start a conversation.".into(), + description: "List files in this bot's workspace on the bound Computer via the native file API (text result, no screenshot). Team computers prefix relative paths into bots//; shared/ is the shared tree. Not for greetings or chat.".into(), parameters: json!({"type":"object","properties":{"path":{"type":"string"}}}), }, ToolDefinition { name: "read_file".into(), - description: "Read a page of a UTF-8 file through Cua in the visible terminal. Output is a screenshot; use start_line and lines for further pages or scroll the terminal.".into(), + description: "Read a workspace file on the bound Computer via the native file API. UTF-8 files support start_line and lines; binary files return size, hash and a short base64 prefix instead of being decoded as text. No screenshot.".into(), parameters: json!({"type":"object","properties":{"path":{"type":"string"},"start_line":{"type":"integer"},"lines":{"type":"integer"}},"required":["path"]}), }, ToolDefinition { name: "write_file".into(), - description: "Write a UTF-8 file by typing a quoted command through Cua in the visible terminal. Inspect the returned screenshot for errors and the written byte count.".into(), + description: "Write a workspace file on the bound Computer via the native file API (bytes, atomic enough for this path). Returns size and hash. No screenshot.".into(), parameters: json!({ "type":"object", - "properties":{"path":{"type":"string"},"content":{"type":"string"}}, + "properties":{ + "path":{"type":"string"}, + "content":{"type":"string"}, + "expectedHash":{"type":"string","description":"sha256 of the file as last read; mismatch returns CONFLICT instead of overwriting"} + }, "required":["path","content"] }), }, + ToolDefinition { + name: "exec".into(), + description: "Run a command on the bound Computer and return real stdout, stderr and exit code. Prefer argv. Use mode=shell only when you need a pipeline. Not a screenshot and not the visible GUI terminal — use shell for TUI collaboration. Timeout does not prove the process stopped.".into(), + parameters: json!({ + "type":"object", + "properties":{ + "argv":{"type":"array","items":{"type":"string"},"description":"Argument vector; preferred. Avoids a host shell."}, + "command":{"type":"string","description":"Used only when argv is omitted and mode is shell."}, + "mode":{"type":"string","enum":["argv","shell"],"description":"Default argv when argv is set, otherwise shell."}, + "cwd":{"type":"string"}, + "timeout_ms":{"type":"number","description":"Request deadline in milliseconds, default 30000, max 120000. Separate from the job runtime."}, + "background":{"type":"boolean","description":"If true, return jobId immediately and keep the process as a Runner job."}, + "jobId":{"type":"string","description":"With action status or cancel, the job to inspect."}, + "action":{"type":"string","enum":["run","status","cancel"]} + } + }), + }, ToolDefinition { name: "open_path".into(), description: "Open a workspace file or http(s) URL on the desktop when the user asked you to open it. Not for greetings or chat.".into(), @@ -145,13 +199,13 @@ pub fn tool_definitions(memory_enabled: bool) -> Vec { }, ToolDefinition { name: "browser".into(), - description: "Control Chromium through the page DOM when the user asked you to use the browser. Prefer this over computer_act for anything in the page. Every action returns fresh numbered elements and visible text; screenshots are opt-in with observe:true for visual ambiguity. click/type/navigate by element id or CSS selector. click scrolls off-screen elements into view and, if the control is [disabled], waits up to 45s (waitMs to change) for it to enable before clicking. Ids are renumbered after every page change. The human still sees the live window.".into(), + description: "Control Chromium through the page DOM when the user asked you to use the browser. Prefer this over computer_act for anything in the page. DOM snapshot/click/type work for text-only models (no screenshot). click/type by element id or snapshot ref (p1:0) from the last snapshot — not CSS. click waits up to 45s (waitMs to change) for a [disabled] control, then returns TARGET_DISABLED if it stays disabled. Ids are renumbered after every page change.".into(), parameters: json!({ "type":"object", "properties":{ "action":{"type":"string","enum":["snapshot","click","type","press","navigate","wait"]}, "element":{"type":"number"}, - "selector":{"type":"string"}, + "selector":{"type":"string","description":"Snapshot ref (p1:0) or unused when element id is set. CSS selectors are rejected as SELECTOR_UNSUPPORTED."}, "text":{"type":"string"}, "key":{"type":"string"}, "url":{"type":"string"}, @@ -250,6 +304,42 @@ pub fn tool_definitions(memory_enabled: bool) -> Vec { description: "Load the playbook of a skill the human taught this bot by demonstration (see 'Taught skills' in your instructions). Returns intent, inputs and semantic steps to follow with the normal tools.".into(), parameters: json!({"type":"object","properties":{"name":{"type":"string"}},"required":["name"]}), }, + ToolDefinition { + name: "form_fill".into(), + description: "Fill a visible web form from the latest browser snapshot. Each field is re-located; the first miss or type failure stops the rest (no blasting stale ids). Prefer this over a chain of browser type calls for multi-field forms. locators are snapshot refs (p1:0) or element ids.".into(), + parameters: json!({ + "type":"object", + "properties":{ + "fields":{ + "type":"array", + "items":{ + "type":"object", + "properties":{ + "name":{"type":"string"}, + "locator":{"type":"string"}, + "element":{"type":"number"}, + "value":{"type":"string"} + }, + "required":["name","value"] + } + }, + "submit":{"type":"string","description":"Optional snapshot ref or element id to click after fields succeed"} + }, + "required":["fields"] + }), + }, + ToolDefinition { + name: "computer_mcp".into(), + description: "Call a Tool Manager package installed on this agent's bound Computer. The process runs inside the Computer, not the API. Currently only the reviewed sample lazyboy.example.echo is callable.".into(), + parameters: json!({ + "type":"object", + "properties":{ + "packageId":{"type":"string"}, + "text":{"type":"string"} + }, + "required":["text"] + }), + }, ]; if memory_enabled { definitions.extend([ @@ -293,16 +383,68 @@ pub struct ToolOutcome { pub image: Option>, pub pause: bool, pub blocks: Vec, + pub error_code: Option, } pub async fn dispatch(ctx: &ToolCtx, name: &str, args: &Value) -> ToolOutcome { + let operation_id = args + .get("operationId") + .or_else(|| args.get("operation_id")) + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| Uuid::new_v4().to_string()); + *ctx.last_operation_id.lock().unwrap() = Some(operation_id.clone()); + match crate::operations::begin(ctx, name, args, &operation_id).await { + crate::operations::Begin::Proceed => {} + crate::operations::Begin::Replay { + text, + error_code, + pause, + } => { + return ToolOutcome { + text, + image: None, + pause, + blocks: Vec::new(), + error_code, + }; + } + crate::operations::Begin::InProgress => { + return error_outcome( + "UNKNOWN_EFFECT", + "an earlier attempt with this operationId started but never reported a result; its effect is unknown. Read the current state back before deciding, and use a new operationId to retry", + ); + } + crate::operations::Begin::PayloadMismatch => { + return error_outcome( + "PAYLOAD_MISMATCH", + "same operationId with a different payload; refused", + ); + } + crate::operations::Begin::JournalUnavailable => { + return error_outcome( + "JOURNAL_UNAVAILABLE", + "operation journal is unavailable; mutation refused", + ); + } + } + let outcome = dispatch_inner(ctx, name, args).await; + crate::operations::finish(ctx, &operation_id, name, args, &outcome).await; + outcome +} + +async fn dispatch_inner(ctx: &ToolCtx, name: &str, args: &Value) -> ToolOutcome { match name { "computer_observe" => observe(ctx).await, "computer_act" => act(ctx, args).await, "wait" => wait_then_observe(ctx, args).await, "browser" => browser(ctx, args).await, + "form_fill" => form_fill(ctx, args).await, + "computer_mcp" => computer_mcp(ctx, args).await, "connection_check" => connection_check(ctx, args).await, "shell" => shell(ctx, args).await, + "exec" => exec_run(ctx, args).await, "list_files" => list_files(ctx, args).await, "read_file" => read_file(ctx, args).await, "write_file" => write_file(ctx, args).await, @@ -322,6 +464,7 @@ pub async fn dispatch(ctx: &ToolCtx, name: &str, args: &Value) -> ToolOutcome { image: None, pause: true, blocks: login_blocks(args), + error_code: None, } } "list_accounts" => list_saved_accounts(ctx).await, @@ -348,15 +491,18 @@ pub async fn dispatch(ctx: &ToolCtx, name: &str, args: &Value) -> ToolOutcome { )), } } - other if other.starts_with("mcp_") => match ctx.mcp.call(other, args).await { - Ok(text) => text_outcome(text), - Err(error) => text_outcome(format!("MCP 工具失敗：{error}")), - }, + other if other.starts_with("mcp_") => { + match ctx.mcp.call_for(Some(&ctx.actor), other, args).await { + Ok(text) => text_outcome(text), + Err(error) => text_outcome(format!("MCP 工具失敗：{error}")), + } + } other => ToolOutcome { text: format!("unknown tool {other}"), image: None, pause: false, blocks: Vec::new(), + error_code: None, }, } } @@ -432,6 +578,17 @@ fn text_outcome(text: impl Into) -> ToolOutcome { image: None, pause: false, blocks: Vec::new(), + error_code: None, + } +} + +fn error_outcome(code: &str, text: impl Into) -> ToolOutcome { + ToolOutcome { + text: text.into(), + image: None, + pause: false, + blocks: Vec::new(), + error_code: Some(code.to_string()), } } @@ -447,21 +604,42 @@ fn gui_blocked(ctx: &ToolCtx) -> Option { image: None, pause: false, blocks: Vec::new(), + error_code: None, }) } -/// Everything that acts on pixels. Observation deliberately does not use this: -/// an element tree is text, so a text-only model can still read the desktop. +/// Pixel actions (coordinates, GUI terminal, opening apps). Observation and +/// DOM/browser tools deliberately do not use this: an element tree is text, so +/// a text-only model can still read the desktop and drive the page by id. fn vision_guard(ctx: &ToolCtx) -> Option { gui_blocked(ctx).or_else(|| { (!ctx.vision).then(|| { text_outcome( - "This model cannot see the shared desktop, so it cannot drive it. computer_observe still reports the element tree as text; pick a vision model to click, type, or browse.", + "This model cannot see the shared desktop, so it cannot drive it with pixel coordinates. computer_observe still reports the element tree as text; browser snapshot/click/type by element id still work; pick a vision model to click pixels, type in the visible terminal, or open GUI apps.", ) }) }) } +fn pixel_actions_require_vision(args: &Value) -> bool { + let Some(actions) = args.get("actions").and_then(Value::as_array) else { + return true; + }; + if actions.is_empty() { + return true; + } + actions.iter().any(|action| { + action.get("x").is_some() + || action.get("y").is_some() + || action.get("x2").is_some() + || action.get("y2").is_some() + || matches!( + action.get("kind").and_then(Value::as_str), + Some("move" | "drag" | "down" | "up" | "hover" | "scroll") + ) + }) +} + /// Controls listed in one observation. A busy native desktop lands near this /// number and the tail is scrolled-off controls and duplicated windows. const MAX_LISTED_ELEMENTS: usize = 120; @@ -523,6 +701,7 @@ async fn observe(ctx: &ToolCtx) -> ToolOutcome { image: None, pause: false, blocks: Vec::new(), + error_code: None, }, } } @@ -650,6 +829,7 @@ fn connection_takeover(ctx: &ToolCtx, reason: &str) -> ToolOutcome { pause: true, blocks: login_blocks(&json!({"reason":reason,"site":"網站連線驗證", "why":"完成驗證後，繼續原本的瀏覽任務。"})), + error_code: None, } } @@ -727,7 +907,7 @@ async fn connection_check(ctx: &ToolCtx, args: &Value) -> ToolOutcome { } async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome { - if let Some(blocked) = vision_guard(ctx) { + if let Some(blocked) = gui_blocked(ctx) { return blocked; } let action = args @@ -784,7 +964,15 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome { *ctx.elements.lock().unwrap() = page.elements.clone(); } if !page.ok { - let mut text = page.error.unwrap_or_else(|| "browser failed".into()); + let mut text = page + .error + .clone() + .unwrap_or_else(|| "browser failed".into()); + if let Some(code) = page.error_code.as_deref() + && !text.starts_with(code) + { + text = format!("{code}: {text}"); + } if !page.elements.is_empty() { text.push_str(&format!( "\nPage: {} {}\nClickable page elements: {}", @@ -793,7 +981,10 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome { format_ui_elements(&page.elements) )); } - return text_outcome(text); + return error_outcome( + page.error_code.as_deref().unwrap_or(ERROR_TARGET_NOT_FOUND), + text, + ); } let mut text = browser_result_text(action, &page); if is_connection_check(&page) { @@ -810,6 +1001,7 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome { image: None, pause: false, blocks: Vec::new(), + error_code: page.error_code.clone(), }; } match ctx @@ -826,10 +1018,251 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome { image: None, pause: false, blocks: Vec::new(), + error_code: page.error_code.clone(), }, } } +fn resolve_form_locator(elements: &[UiElement], locator: &str) -> Option { + let trimmed = locator.trim(); + if trimmed.is_empty() { + return None; + } + if let Ok(id) = trimmed.parse::() { + return elements + .iter() + .find(|element| u64::from(element.id) == id) + .and_then(|element| element.selector.clone()); + } + if let Some(found) = elements + .iter() + .find(|element| element.selector.as_deref() == Some(trimmed)) + { + return found.selector.clone(); + } + if let Some(found) = elements + .iter() + .find(|element| element.title.eq_ignore_ascii_case(trimmed)) + { + return found.selector.clone(); + } + if trimmed.contains(':') { + return Some(trimmed.to_string()); + } + None +} + +async fn form_fill(ctx: &ToolCtx, args: &Value) -> ToolOutcome { + if let Some(blocked) = gui_blocked(ctx) { + return blocked; + } + let Some(fields) = args.get("fields").and_then(Value::as_array) else { + return error_outcome( + "INVALID_ARGUMENT", + "form_fill needs fields: [{name, locator|element, value}]", + ); + }; + if ctx.elements.lock().unwrap().is_empty() { + let page = browser_call(ctx, json!({"action":"snapshot","ensure":true})).await; + if page.ok || !page.elements.is_empty() { + *ctx.elements.lock().unwrap() = page.elements.clone(); + } + } + let elements = ctx.elements.lock().unwrap().clone(); + // Silently dropping a malformed or excess field would submit an incomplete + // form and report success; refuse the whole call instead. + const MAX_FORM_FIELDS: usize = 20; + if fields.len() > MAX_FORM_FIELDS { + return error_outcome( + "INVALID_ARGUMENT", + format!( + "form_fill accepts at most {MAX_FORM_FIELDS} fields per call (got {}); split the form", + fields.len() + ), + ); + } + let mut parsed: Vec = Vec::with_capacity(fields.len()); + for (index, field) in fields.iter().enumerate() { + let Some(name) = field.get("name").and_then(Value::as_str) else { + return error_outcome( + "INVALID_ARGUMENT", + format!("form_fill fields[{index}] is missing name"), + ); + }; + let value = field + .get("value") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let Some(locator) = field + .get("locator") + .and_then(Value::as_str) + .map(str::to_string) + .or_else(|| element_id(field.get("element")).map(|id| id.to_string())) + else { + return error_outcome( + "INVALID_ARGUMENT", + format!("form_fill fields[{index}] ({name}) needs locator or element"), + ); + }; + parsed.push(FormField { + name: name.to_string(), + value, + locator, + }); + } + if parsed.is_empty() { + return error_outcome("INVALID_ARGUMENT", "form_fill fields were empty"); + } + let planned = run_form_macro( + &parsed, + |locator| resolve_form_locator(&elements, locator), + |selector, _value| Ok(selector.to_string()), + || true, + ); + if !planned.ok { + return error_outcome( + "TARGET_STALE", + json!({"ok":false,"steps":planned.steps.iter().map(|step| json!({ + "field": step.field, + "ok": step.ok, + "evidence": step.evidence, + })).collect::>()}) + .to_string(), + ); + } + let mut steps = Vec::new(); + for field in &parsed { + let Some(selector) = + resolve_form_locator(&ctx.elements.lock().unwrap().clone(), &field.locator) + else { + steps.push(json!({"field": field.name, "ok": false, "evidence": "TARGET_STALE"})); + return error_outcome( + "TARGET_STALE", + json!({"ok":false,"steps":steps}).to_string(), + ); + }; + let page = browser_call( + ctx, + json!({"action":"type","ensure":true,"selector":selector,"text":field.value}), + ) + .await; + if page.ok || !page.elements.is_empty() { + *ctx.elements.lock().unwrap() = page.elements.clone(); + } + if !page.ok { + steps.push(json!({ + "field": field.name, + "ok": false, + "evidence": page.error_code.clone().unwrap_or_else(|| "type failed".into()), + })); + return error_outcome( + page.error_code.as_deref().unwrap_or(ERROR_TARGET_NOT_FOUND), + json!({"ok":false,"steps":steps}).to_string(), + ); + } + steps.push(json!({"field": field.name, "ok": true, "evidence": "typed"})); + } + if let Some(submit) = args.get("submit").and_then(Value::as_str) { + let Some(selector) = resolve_form_locator(&ctx.elements.lock().unwrap().clone(), submit) + else { + steps.push(json!({"field":"submit","ok":false,"evidence":"TARGET_STALE"})); + return error_outcome( + "TARGET_STALE", + json!({"ok":false,"steps":steps}).to_string(), + ); + }; + let page = browser_call( + ctx, + json!({"action":"click","ensure":true,"selector":selector}), + ) + .await; + if !page.ok { + steps.push(json!({ + "field":"submit", + "ok": false, + "evidence": page.error_code.clone().unwrap_or_else(|| "click failed".into()), + })); + return error_outcome( + page.error_code.as_deref().unwrap_or(ERROR_TARGET_NOT_FOUND), + json!({"ok":false,"steps":steps}).to_string(), + ); + } + steps.push(json!({"field":"submit","ok":true,"evidence":"clicked"})); + } + text_outcome(json!({"ok":true,"steps":steps}).to_string()) +} + +async fn computer_mcp(ctx: &ToolCtx, args: &Value) -> ToolOutcome { + let package_id = args + .get("packageId") + .and_then(Value::as_str) + .unwrap_or(crate::tool_install::ECHO_ID); + if package_id != crate::tool_install::ECHO_ID { + return error_outcome( + "UNSUPPORTED_PACKAGE", + "only lazyboy.example.echo is callable without a reviewed artifact", + ); + } + let text = args.get("text").and_then(Value::as_str).unwrap_or(""); + match crate::tool_install::package_is_bound(&ctx.pool, &ctx.actor, &ctx.bot_id, package_id) + .await + { + Ok(true) => {} + Ok(false) => { + return error_outcome( + "NOT_BOUND", + "package is not bound to this agent; install it from Plugins first", + ); + } + Err(error) => return text_outcome(format!("computer_mcp failed: {error}")), + } + let bound = match crate::tool_install::bound_echo_relative( + &ctx.pool, + &ctx.actor, + &ctx.bot_id, + ctx.mode, + ) + .await + { + Ok(Some(bound)) => bound, + Ok(None) => { + return error_outcome( + "NOT_BOUND", + "package is not bound to this agent; install it from Plugins first", + ); + } + Err(error) => return text_outcome(format!("computer_mcp failed: {error}")), + }; + match crate::tool_install::exec_echo( + ctx.sandbox.as_ref(), + &ctx.computer_ref(), + &ctx.adapter(), + &bound, + text, + ) + .await + { + Ok(result) if result.code == 97 => error_outcome( + "PACKAGE_TAMPERED", + format!( + "installed package no longer matches its recorded digest; refusing to run. {}", + result.stderr.trim() + ), + ), + Ok(result) => text_outcome( + json!({ + "exitCode": result.code, + "stdout": result.stdout, + "stderr": result.stderr, + "executionLocation": "assigned_computer", + }) + .to_string(), + ), + Err(error) => text_outcome(format!("computer_mcp failed: {error}")), + } +} + fn browser_result_text(action: &str, page: &BrowserPage) -> String { format!( "browser {action}\nPage: {} {}\nClickable page elements: {}\nVisible text:\n{}", @@ -841,7 +1274,12 @@ fn browser_result_text(action: &str, page: &BrowserPage) -> String { } async fn act(ctx: &ToolCtx, args: &Value) -> ToolOutcome { - if let Some(blocked) = vision_guard(ctx) { + if let Some(blocked) = gui_blocked(ctx) { + return blocked; + } + if pixel_actions_require_vision(args) + && let Some(blocked) = vision_guard(ctx) + { return blocked; } let mut args = args.clone(); @@ -876,6 +1314,7 @@ async fn act(ctx: &ToolCtx, args: &Value) -> ToolOutcome { image: None, pause: false, blocks: Vec::new(), + error_code: None, }; } }; @@ -930,6 +1369,7 @@ async fn act(ctx: &ToolCtx, args: &Value) -> ToolOutcome { image: None, pause: false, blocks: Vec::new(), + error_code: None, }; with_verdict(&mut outcome, verdict); outcome @@ -940,6 +1380,7 @@ async fn act(ctx: &ToolCtx, args: &Value) -> ToolOutcome { image: None, pause: false, blocks: Vec::new(), + error_code: None, }, } } @@ -1038,12 +1479,13 @@ async fn apply_semantic_actions(ctx: &ToolCtx, items: &mut [Value], elements: &[ fn pause_unknown_element(_ctx: &ToolCtx, id: u32, elements: &[UiElement]) -> ToolOutcome { ToolOutcome { text: format!( - "element {id} is not on screen any more. Visible now: {}. Call browser snapshot or computer_observe to get fresh ids, then retry.", + "{ERROR_TARGET_STALE}: element {id} is not on screen any more. Visible now: {}. Call browser snapshot or computer_observe to get fresh ids, then retry.", format_ui_elements(elements) ), image: None, pause: false, blocks: Vec::new(), + error_code: Some(ERROR_TARGET_STALE.into()), } } @@ -1103,7 +1545,7 @@ fn pack_observation(ctx: &ToolCtx, note: &str, observation: ComputerObservation) let change = screen_change(ctx, &observation); let signature = frame_signature(&observation.image); *ctx.previous_frame.lock().unwrap() = Some(observation.frame_id.clone()); - *ctx.previous_signature.lock().unwrap() = signature; + *ctx.previous_signature.lock().unwrap() = signature.clone(); *ctx.elements.lock().unwrap() = observation.elements.clone(); let mut text = observation_text(note, &observation, change); if !ctx.vision { @@ -1111,17 +1553,34 @@ fn pack_observation(ctx: &ToolCtx, note: &str, observation: ComputerObservation) // it keeps waiting for a picture that is never coming. text.push_str("\n(elements only: this model cannot see the screen)"); } + let force = *ctx.force_image.lock().unwrap(); + let delivered_frame = ctx.delivered_frame.lock().unwrap().clone(); + let delivered_signature = ctx.delivered_signature.lock().unwrap().clone(); + let delivered_model_id = ctx.delivered_model_id.lock().unwrap().clone(); + let deliver = should_deliver_observation_image( + ctx.vision, + force, + Some(ctx.model_id.as_str()).filter(|id| !id.is_empty()), + delivered_frame.as_deref(), + delivered_signature.as_deref(), + delivered_model_id.as_deref(), + &observation, + ); + *ctx.pending_image_delivery.lock().unwrap() = if deliver { + Some((observation.frame_id.clone(), signature)) + } else { + None + }; ToolOutcome { text, - // Only a byte-identical frame drops the picture. A change too small to - // move the signature is still one the model gets to look at. - image: if ctx.vision && change != ScreenChange::Identical { - None - } else { + image: if deliver { Some(overlay_elements(&observation.image, &observation.elements)) + } else { + None }, pause: false, blocks: Vec::new(), + error_code: None, } } @@ -1307,27 +1766,401 @@ async fn shell(ctx: &ToolCtx, args: &Value) -> ToolOutcome { } } -fn visible_file_path(ctx: &ToolCtx, args: &Value, default: &str) -> Result { +fn workspace_rel_path(ctx: &ToolCtx, args: &Value, default: &str) -> Result { let requested = args.get("path").and_then(Value::as_str).unwrap_or(default); - let stored = resolve_bot_workspace_path(ctx.mode, &ctx.bot_id, requested) - .map_err(|error| error.to_string())?; - Ok(format!("/home/lazyboy/{stored}")) + resolve_bot_workspace_path(ctx.mode, &ctx.bot_id, requested).map_err(|error| error.to_string()) +} + +fn native_file_payload(bytes: &[u8], start_line: u64, lines: u64) -> Value { + let digest = hex::encode(Sha256::digest(bytes)); + match std::str::from_utf8(bytes) { + Ok(text) => { + let all: Vec<&str> = text.lines().collect(); + let start = (start_line.max(1) as usize).saturating_sub(1); + let count = lines.clamp(1, 200) as usize; + let end = (start + count).min(all.len()); + let slice = all.get(start..end).unwrap_or(&[]); + json!({ + "encoding": "utf-8", + "size": bytes.len(), + "sha256": digest, + "startLine": start + 1, + "endLine": start + slice.len(), + "totalLines": all.len(), + "truncated": start > 0 || end < all.len(), + "content": slice.join("\n"), + }) + } + Err(_) => { + const MAX_BINARY_PREVIEW: usize = 4096; + let preview = if bytes.len() <= MAX_BINARY_PREVIEW { + Some(base64::engine::general_purpose::STANDARD.encode(bytes)) + } else { + None + }; + json!({ + "encoding": "binary", + "size": bytes.len(), + "sha256": digest, + "contentBase64": preview, + "note": "binary file; not decoded as text. contentBase64 is included only for files ≤ 4096 bytes.", + }) + } + } +} + +async fn bot_computer_id(ctx: &ToolCtx) -> Option { + sqlx::query_scalar("SELECT computer_id FROM bots WHERE id=$1") + .bind(&ctx.bot_id) + .fetch_optional(&ctx.pool) + .await + .ok() + .flatten() + .flatten() +} + +async fn db_running_jobs(ctx: &ToolCtx, computer_id: &str) -> usize { + let count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM computer_jobs WHERE computer_id=$1 AND status='running'", + ) + .bind(computer_id) + .fetch_one(&ctx.pool) + .await + .unwrap_or(0); + count.max(0) as usize +} + +/// `computer_jobs.status` only changes when someone asks the Computer, so jobs +/// that finished unobserved would otherwise count against the quota forever +/// and keep the idle reaper from ever seeing the Computer as idle. Ask the +/// Computer about the oldest `running` rows before deciding a quota is full. +async fn refresh_running_jobs(ctx: &ToolCtx, computer_id: &str) { + let ids: Vec = sqlx::query_scalar( + "SELECT id FROM computer_jobs WHERE computer_id=$1 AND status='running' + ORDER BY updated_at LIMIT 16", + ) + .bind(computer_id) + .fetch_all(&ctx.pool) + .await + .unwrap_or_default(); + for id in ids { + let outcome = ctx + .sandbox + .execute( + &ctx.computer_ref(), + CommandRequest { + argv: Vec::new(), + timeout_ms: Some(5_000), + job_id: Some(id.clone()), + job_op: Some("status".into()), + ..CommandRequest::default() + }, + &ctx.adapter(), + ) + .await; + match outcome { + Ok(result) => { + if let Some(status) = result.status.filter(|status| status != "running") { + persist_job(ctx, &id, &status, &[]).await; + } + } + // The Computer (or its supervisor) no longer knows the job: it is + // not running there, but we cannot claim how it ended. + Err(error) if error.to_string().contains("unknown job") => { + persist_job(ctx, &id, "interrupted", &[]).await; + } + Err(_) => {} + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum JobOwnership { + /// Started by this bot on the current Computer generation. + Mine, + /// Started by another bot: not this bot's to inspect or cancel. + Foreign, + /// Started before the Computer was rebuilt, or already marked interrupted. + Interrupted, + /// No row (e.g. no Computer bound when it started): defer to the Computer. + Unrecorded, +} + +async fn job_ownership(ctx: &ToolCtx, job_id: &str) -> JobOwnership { + let row: Option<(String, i32, String)> = + sqlx::query_as("SELECT bot_id, generation, status FROM computer_jobs WHERE id=$1") + .bind(job_id) + .fetch_optional(&ctx.pool) + .await + .ok() + .flatten(); + let Some((bot_id, generation, status)) = row else { + return JobOwnership::Unrecorded; + }; + if bot_id != ctx.bot_id { + return JobOwnership::Foreign; + } + let current = ctx.adapter().computer_generation.unwrap_or(1); + if status == "interrupted" || generation < current { + return JobOwnership::Interrupted; + } + JobOwnership::Mine +} + +async fn persist_job(ctx: &ToolCtx, job_id: &str, status: &str, argv: &[String]) { + let Some(computer_id) = bot_computer_id(ctx).await else { + return; + }; + let generation = ctx.adapter().computer_generation.unwrap_or(1); + // A job the reaper marked `interrupted` (Computer restarted) never comes + // back to `running`; only the ledger row that started it may claim that. + let _ = sqlx::query( + "INSERT INTO computer_jobs (id, computer_id, generation, bot_id, status, argv, pin_version) + VALUES ($1,$2,$3,$4,$5,$6,$7) + ON CONFLICT (id) DO UPDATE SET + status=CASE + WHEN computer_jobs.status='interrupted' AND EXCLUDED.status='running' + THEN computer_jobs.status + ELSE EXCLUDED.status END, + updated_at=now(), + pin_version=COALESCE(computer_jobs.pin_version, EXCLUDED.pin_version)", + ) + .bind(job_id) + .bind(computer_id) + .bind(generation) + .bind(&ctx.bot_id) + .bind(status) + .bind(json!(argv)) + .bind(Option::::None) + .execute(&ctx.pool) + .await; +} + +fn exec_result_payload(result: &lazyboy_control::CommandResult) -> Value { + json!({ + "stdout": result.stdout, + "stderr": result.stderr, + "exitCode": result.code, + "truncated": false, + }) +} + +async fn exec_run(ctx: &ToolCtx, args: &Value) -> ToolOutcome { + let action = args.get("action").and_then(Value::as_str).unwrap_or("run"); + if action == "status" || action == "cancel" { + let Some(job_id) = args.get("jobId").and_then(Value::as_str) else { + return text_outcome("jobId is required"); + }; + // The Computer only knows job ids; ownership and the Computer generation + // that started the job live in our table. Another bot's job is invisible, + // and a job from before a restart is reported interrupted without asking + // the (new) Computer, which would only say "unknown job". + match job_ownership(ctx, job_id).await { + JobOwnership::Foreign => { + return error_outcome("UNKNOWN_JOB", format!("unknown job {job_id}")); + } + JobOwnership::Interrupted => { + persist_job(ctx, job_id, "interrupted", &[]).await; + return text_outcome( + json!({ + "jobId": job_id, + "status": "interrupted", + "stdout": "", + "stderr": "", + "exitCode": null, + "executionLocation": "assigned_computer", + "note": "the Computer restarted after this job started; its output is gone. Start it again if still needed.", + }) + .to_string(), + ); + } + JobOwnership::Mine | JobOwnership::Unrecorded => {} + } + let result = match ctx + .sandbox + .execute( + &ctx.computer_ref(), + CommandRequest { + argv: Vec::new(), + cwd: None, + timeout_ms: Some(5_000), + stdin: None, + job_id: Some(job_id.to_string()), + job_op: Some(action.to_string()), + ..CommandRequest::default() + }, + &ctx.adapter(), + ) + .await + { + Ok(result) => result, + Err(error) => return text_outcome(format!("{action} failed: {error}")), + }; + let Some(status) = result.status.clone() else { + return error_outcome( + "TRANSPORT", + "Computer exec result missing status; refusing to invent running/cancelled", + ); + }; + persist_job( + ctx, + result.job_id.as_deref().unwrap_or(job_id), + &status, + &[], + ) + .await; + return text_outcome( + json!({ + "jobId": result.job_id.as_deref().unwrap_or(job_id), + "status": status, + "stdout": result.stdout, + "stderr": result.stderr, + "exitCode": result.code, + "executionLocation": "assigned_computer", + }) + .to_string(), + ); + } + let cwd = match resolve_bot_workspace_cwd( + ctx.mode, + &ctx.bot_id, + args.get("cwd").and_then(Value::as_str), + ) { + Ok(cwd) => cwd, + Err(error) => return text_outcome(error.to_string()), + }; + let timeout_ms = args + .get("timeout_ms") + .and_then(Value::as_u64) + .unwrap_or(30_000) + .clamp(100, 120_000); + let argv = if let Some(items) = args.get("argv").and_then(Value::as_array) { + let argv: Vec = items + .iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect(); + if argv.is_empty() { + return text_outcome("argv must contain at least one argument"); + } + argv + } else if let Some(command) = args + .get("command") + .and_then(Value::as_str) + .filter(|text| !text.trim().is_empty()) + { + vec!["bash".into(), "-lc".into(), command.to_string()] + } else { + return text_outcome("exec needs argv or command"); + }; + let background = args + .get("background") + .and_then(Value::as_bool) + .unwrap_or(false); + if background { + let computer_id = bot_computer_id(ctx).await; + let limit = native_job_concurrency(); + let mut running = match computer_id.as_deref() { + Some(id) => db_running_jobs(ctx, id).await, + None => 0, + }; + if job_quota_exceeded(running, limit) + && let Some(id) = computer_id.as_deref() + { + refresh_running_jobs(ctx, id).await; + running = db_running_jobs(ctx, id).await; + } + if job_quota_exceeded(running, limit) { + return error_outcome( + "QUOTA_EXCEEDED", + format!("native job concurrency is {limit}; wait or cancel a running job"), + ); + } + let job_id = format!("job-{}", Uuid::new_v4()); + let generation = ctx.adapter().computer_generation.unwrap_or(1); + let argv_record = argv.clone(); + return match ctx + .sandbox + .execute( + &ctx.computer_ref(), + CommandRequest { + argv, + cwd, + timeout_ms: None, + stdin: None, + background: true, + job_id: Some(job_id.clone()), + operation_id: Some(job_id.clone()), + ..CommandRequest::default() + }, + &ctx.adapter(), + ) + .await + { + Ok(result) => { + let id = result.job_id.clone().unwrap_or(job_id); + persist_job(ctx, &id, "running", &argv_record).await; + text_outcome( + json!({ + "jobId": id, + "status": "running", + "generation": generation, + "executionLocation": "assigned_computer", + }) + .to_string(), + ) + } + Err(error) => text_outcome(format!("exec failed: {error}")), + }; + } + match ctx + .sandbox + .execute( + &ctx.computer_ref(), + CommandRequest { + argv, + cwd, + timeout_ms: Some(timeout_ms), + stdin: None, + ..CommandRequest::default() + }, + &ctx.adapter(), + ) + .await + { + Ok(result) => text_outcome(exec_result_payload(&result).to_string()), + Err(error) => text_outcome(format!("exec failed: {error}")), + } } async fn list_files(ctx: &ToolCtx, args: &Value) -> ToolOutcome { - let path = match visible_file_path(ctx, args, "") { + let path = match workspace_rel_path(ctx, args, "") { Ok(path) => path, Err(error) => return text_outcome(error), }; - shell( - ctx, - &json!({"session":"files", "command":format!("ls -la -- {}", shell_quote(&path))}), - ) - .await + match ctx + .sandbox + .list_files(&ctx.computer_ref(), &path, &ctx.adapter()) + .await + { + Ok(entries) => text_outcome(native_list_payload(&path, &entries).to_string()), + Err(error) => text_outcome(format!("list_files failed: {error}")), + } +} + +fn native_list_payload(path: &str, entries: &[FileEntry]) -> Value { + json!({ + "path": path, + "entries": entries.iter().map(|entry| json!({ + "path": entry.path, + "kind": entry.kind, + "size": entry.size, + })).collect::>(), + }) } async fn read_file(ctx: &ToolCtx, args: &Value) -> ToolOutcome { - let path = match visible_file_path(ctx, args, "") { + let path = match workspace_rel_path(ctx, args, "") { Ok(path) => path, Err(error) => return text_outcome(error), }; @@ -1339,30 +2172,76 @@ async fn read_file(ctx: &ToolCtx, args: &Value) -> ToolOutcome { let count = args .get("lines") .and_then(Value::as_u64) - .unwrap_or(25) + .unwrap_or(80) .clamp(1, 200); - let end = start.saturating_add(count - 1); - shell(ctx, &json!({"session":"files", "command":format!("sed -n '{start},{end}p' -- {}", shell_quote(&path))})).await + match ctx + .sandbox + .read_file(&ctx.computer_ref(), &path, &ctx.adapter()) + .await + { + Ok(bytes) => text_outcome(native_file_payload(&bytes, start, count).to_string()), + Err(error) => text_outcome(format!("read_file failed: {error}")), + } } async fn write_file(ctx: &ToolCtx, args: &Value) -> ToolOutcome { - let path = match visible_file_path(ctx, args, "notes.txt") { + let path = match workspace_rel_path(ctx, args, "notes.txt") { Ok(path) => path, Err(error) => return text_outcome(error), }; let content = args.get("content").and_then(Value::as_str).unwrap_or(""); - let parent = std::path::Path::new(&path) - .parent() - .and_then(|path| path.to_str()) - .unwrap_or("/home/lazyboy"); - let command = format!( - "mkdir -p -- {} && printf %s {} > {} && wc -c -- {}", - shell_quote(parent), - shell_quote(content), - shell_quote(&path), - shell_quote(&path) - ); - shell(ctx, &json!({"session":"files", "command":command})).await + let expected = args.get("expectedHash").and_then(Value::as_str); + if expected.is_some() { + match ctx + .sandbox + .read_file(&ctx.computer_ref(), &path, &ctx.adapter()) + .await + { + Ok(current) => { + if let Err(error) = check_expected_hash(Some(¤t), expected) { + return error_outcome( + "CONFLICT", + format!("CONFLICT: file changed since last read ({error:?})"), + ); + } + } + Err(_) => { + if let Err(error) = check_expected_hash(None, expected) { + return error_outcome( + "CONFLICT", + format!("CONFLICT: file changed since last read ({error:?})"), + ); + } + } + } + } + match ctx + .sandbox + .write_file( + &ctx.computer_ref(), + &path, + content.as_bytes(), + &ctx.adapter(), + ) + .await + { + Ok(()) => { + let digest = hex::encode(Sha256::digest(content.as_bytes())); + let operation_id = ctx.last_operation_id.lock().unwrap().clone(); + crate::artifacts::record_file(ctx, &path, content.as_bytes(), operation_id.as_deref()) + .await; + text_outcome( + json!({ + "ok": true, + "path": path, + "size": content.len(), + "sha256": digest, + }) + .to_string(), + ) + } + Err(error) => text_outcome(format!("write_file failed: {error}")), + } } async fn open_path(ctx: &ToolCtx, args: &Value) -> ToolOutcome { @@ -1394,6 +2273,7 @@ async fn open_path(ctx: &ToolCtx, args: &Value) -> ToolOutcome { image: None, pause: false, blocks: Vec::new(), + error_code: None, } } } @@ -1402,6 +2282,7 @@ async fn open_path(ctx: &ToolCtx, args: &Value) -> ToolOutcome { image: None, pause: false, blocks: Vec::new(), + error_code: None, }, } } @@ -1442,6 +2323,7 @@ async fn launch_app(ctx: &ToolCtx, args: &Value) -> ToolOutcome { image: None, pause: false, blocks: Vec::new(), + error_code: None, } } } @@ -1450,6 +2332,7 @@ async fn launch_app(ctx: &ToolCtx, args: &Value) -> ToolOutcome { image: None, pause: false, blocks: Vec::new(), + error_code: None, }, } } @@ -1502,7 +2385,7 @@ async fn list_saved_accounts(ctx: &ToolCtx) -> ToolOutcome { } async fn use_saved_login(ctx: &ToolCtx, args: &Value) -> ToolOutcome { - if let Some(blocked) = vision_guard(ctx) { + if let Some(blocked) = gui_blocked(ctx) { return blocked; } let Some(account_id) = args.get("accountId").and_then(Value::as_str) else { @@ -1661,6 +2544,7 @@ async fn create_schedule_tool(ctx: &ToolCtx, args: &Value) -> ToolOutcome { "cron": row.cron, "human": human, })], + error_code: None, } } Err(error) => text_outcome(error), @@ -2073,4 +2957,115 @@ mod tool_schema_tests { }; assert_eq!(desktop(false), desktop(true)); } + + #[test] + fn native_exec_is_in_the_schema_after_write_file() { + let tools = tool_definitions(false); + let names: Vec<&str> = tools.iter().map(|tool| tool.name.as_str()).collect(); + assert!(names.contains(&"exec")); + let write = names.iter().position(|name| *name == "write_file").unwrap(); + let exec = names.iter().position(|name| *name == "exec").unwrap(); + assert!(exec > write); + } + + #[test] + fn exec_background_does_not_spawn_on_the_api() { + let src = include_str!("tools.rs"); + let start = format!("{}{}", "JOBS.", "start"); + assert!( + !src.contains(&start), + "background exec must go through sandbox.execute on the Computer" + ); + } + + #[test] + fn form_fill_and_computer_mcp_are_in_the_schema() { + let tools = tool_definitions(false); + let names: Vec<&str> = tools.iter().map(|tool| tool.name.as_str()).collect(); + assert!(names.contains(&"form_fill")); + assert!(names.contains(&"computer_mcp")); + } +} + +#[cfg(test)] +mod form_locator_tests { + use super::*; + + fn el(id: u32, title: &str, selector: &str) -> UiElement { + UiElement { + id, + title: title.into(), + x: 0, + y: 0, + w: 10, + h: 10, + selector: Some(selector.into()), + kind: Some("dom".into()), + role: None, + disabled: false, + } + } + + #[test] + fn form_locator_resolves_id_selector_and_snapshot_ref() { + let elements = vec![el(3, "Email", "p1:3")]; + assert_eq!( + resolve_form_locator(&elements, "3").as_deref(), + Some("p1:3") + ); + assert_eq!( + resolve_form_locator(&elements, "Email").as_deref(), + Some("p1:3") + ); + assert_eq!( + resolve_form_locator(&elements, "p1:3").as_deref(), + Some("p1:3") + ); + assert!(resolve_form_locator(&elements, "missing").is_none()); + } +} + +#[cfg(test)] +mod native_file_payload_tests { + use super::*; + use lazyboy_control::{ERROR_SELECTOR_UNSUPPORTED, ERROR_TARGET_DISABLED}; + + #[test] + fn utf8_range_does_not_rewrite_the_file() { + let payload = native_file_payload(b"a\nb\nc\n", 2, 1); + assert_eq!(payload["encoding"], "utf-8"); + assert_eq!(payload["content"], "b"); + assert_eq!(payload["startLine"], 2); + assert_eq!(payload["totalLines"], 3); + assert_eq!(payload["truncated"], true); + } + + #[test] + fn invalid_utf8_is_binary_not_lossy_text() { + let payload = native_file_payload(&[0xff, 0xfe, 0x00], 1, 25); + assert_eq!(payload["encoding"], "binary"); + assert_eq!(payload["size"], 3); + assert!(payload["sha256"].as_str().unwrap().len() == 64); + assert!(payload.get("content").is_none()); + assert!(payload["contentBase64"].as_str().is_some()); + } + + #[test] + fn pixel_actions_need_vision_element_ids_do_not() { + assert!(pixel_actions_require_vision( + &json!({"actions":[{"kind":"click","x":10,"y":20}]}) + )); + assert!(!pixel_actions_require_vision( + &json!({"actions":[{"kind":"click","element":3}]}) + )); + assert!(pixel_actions_require_vision(&json!({"actions":[]}))); + } + + #[test] + fn typed_browser_errors_keep_stable_codes() { + assert_eq!(ERROR_SELECTOR_UNSUPPORTED, "SELECTOR_UNSUPPORTED"); + assert_eq!(ERROR_TARGET_DISABLED, "TARGET_DISABLED"); + assert_eq!(ERROR_TARGET_STALE, "TARGET_STALE"); + assert_eq!(ERROR_TARGET_NOT_FOUND, "TARGET_NOT_FOUND"); + } } diff --git a/crates/contracts/src/action.rs b/crates/contracts/src/action.rs index 99aabff..7207d36 100644 --- a/crates/contracts/src/action.rs +++ b/crates/contracts/src/action.rs @@ -112,6 +112,10 @@ pub struct UiElement { pub kind: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub role: Option, + /// True when the snapshot marked the control disabled. Clicks wait, then + /// return TARGET_DISABLED instead of guessing that time will fix it. + #[serde(default)] + pub disabled: bool, } impl UiElement { diff --git a/crates/contracts/src/computer.rs b/crates/contracts/src/computer.rs index 326e2cb..8d275ea 100644 --- a/crates/contracts/src/computer.rs +++ b/crates/contracts/src/computer.rs @@ -220,6 +220,18 @@ pub const PROFILE_LOCKED: &str = mod tests { use super::*; + #[test] + fn team_computers_share_one_scope_across_bots() { + assert_eq!( + computer_scope_key(ComputerMode::Team, "space-1", Some("bot-a")).unwrap(), + computer_scope_key(ComputerMode::Team, "space-1", Some("bot-b")).unwrap() + ); + assert_ne!( + computer_scope_key(ComputerMode::Dedicated, "space-1", Some("bot-a")).unwrap(), + computer_scope_key(ComputerMode::Dedicated, "space-1", Some("bot-b")).unwrap() + ); + } + #[test] fn team_and_dedicated_keys_differ() { assert_eq!( diff --git a/crates/contracts/src/lib.rs b/crates/contracts/src/lib.rs index 186c203..8a2b2c6 100644 --- a/crates/contracts/src/lib.rs +++ b/crates/contracts/src/lib.rs @@ -6,6 +6,7 @@ mod model; mod room; mod run; mod session; +mod tool; mod voice; pub use action::*; @@ -16,6 +17,7 @@ pub use model::*; pub use room::*; pub use run::*; pub use session::*; +pub use tool::*; pub use voice::*; pub type Id = String; diff --git a/crates/contracts/src/tool.rs b/crates/contracts/src/tool.rs new file mode 100644 index 0000000..f6a1308 --- /dev/null +++ b/crates/contracts/src/tool.rs @@ -0,0 +1,141 @@ +use serde::{Deserialize, Serialize}; + +/// Transport / operation / task are three different success layers. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolStatus { + Accepted, + Running, + Succeeded, + Failed, + Cancelled, + TimedOut, + NeedsAuth, + NeedsApproval, + NeedsHuman, + PolicyDenied, + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolEffect { + None, + Confirmed, + Partial, + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionPlane { + Computer, + Control, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ToolExecution { + pub computer_id: String, + #[serde(default)] + pub computer_generation: i32, + pub executor: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub job_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ToolResult { + pub schema_version: u32, + pub operation_id: String, + pub status: ToolStatus, + pub effect: ToolEffect, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub execution: Option, + #[serde(default)] + pub data: serde_json::Value, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_code: Option, +} + +impl ToolResult { + pub fn failed(operation_id: impl Into, code: &str, error: impl Into) -> Self { + Self { + schema_version: 1, + operation_id: operation_id.into(), + status: ToolStatus::Failed, + effect: ToolEffect::Unknown, + execution: None, + data: serde_json::Value::Null, + error: Some(error.into()), + error_code: Some(code.into()), + } + } + + pub fn succeeded(operation_id: impl Into, data: serde_json::Value) -> Self { + Self { + schema_version: 1, + operation_id: operation_id.into(), + status: ToolStatus::Succeeded, + effect: ToolEffect::Confirmed, + execution: None, + data, + error: None, + error_code: None, + } + } + + /// HTTP 200 / exit 0 is not task completion. + pub fn task_complete(&self) -> bool { + false + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PauseScope { + Agent, + Display, + Computer, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RouteKind { + NativeApi, + NativeExec, + NativeFiles, + Dom, + Accessibility, + Vision, + Human, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CompletionLayer { + Transport, + OperationEffect, + TaskSuccess, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_successful_tool_is_not_a_finished_task() { + let result = ToolResult::succeeded("op-1", serde_json::json!({"exitCode": 0})); + assert_eq!(result.status, ToolStatus::Succeeded); + assert!(!result.task_complete()); + } + + #[test] + fn pause_scopes_are_distinct() { + assert_ne!(PauseScope::Agent, PauseScope::Computer); + assert_ne!(PauseScope::Display, PauseScope::Agent); + } +} diff --git a/crates/control/Cargo.toml b/crates/control/Cargo.toml index bcd51c0..ff89eb9 100644 --- a/crates/control/Cargo.toml +++ b/crates/control/Cargo.toml @@ -19,6 +19,7 @@ image.workspace = true tokio.workspace = true tracing.workspace = true base64.workspace = true +regex = "1" [lints] workspace = true diff --git a/crates/control/src/a11y.rs b/crates/control/src/a11y.rs index 21f692a..bee45aa 100644 --- a/crates/control/src/a11y.rs +++ b/crates/control/src/a11y.rs @@ -76,6 +76,7 @@ mod tests { kind: Some("a11y".into()), selector: Some(format!("0/{id}")), role: Some("push button".into()), + ..UiElement::default() } } diff --git a/crates/control/src/actions.rs b/crates/control/src/actions.rs index 637bd00..c284b2a 100644 --- a/crates/control/src/actions.rs +++ b/crates/control/src/actions.rs @@ -697,6 +697,7 @@ mod tests { y: 20, w: 80, h: 24, + ..UiElement::default() } } diff --git a/crates/control/src/browser_page.rs b/crates/control/src/browser_page.rs index b3b8946..28907e6 100644 --- a/crates/control/src/browser_page.rs +++ b/crates/control/src/browser_page.rs @@ -35,6 +35,16 @@ pub struct BrowserPage { /// Seconds the click waited for a disabled control to become enabled. #[serde(default, skip_serializing_if = "Option::is_none")] pub waited_seconds: Option, + /// Typed tool error: SELECTOR_UNSUPPORTED, TARGET_STALE, TARGET_DISABLED, + /// TARGET_NOT_FOUND. Distinct from a free-text `error` so recovery can + /// branch without parsing English. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_code: Option, #[serde(default)] pub elements: Vec, } + +pub const ERROR_SELECTOR_UNSUPPORTED: &str = "SELECTOR_UNSUPPORTED"; +pub const ERROR_TARGET_STALE: &str = "TARGET_STALE"; +pub const ERROR_TARGET_DISABLED: &str = "TARGET_DISABLED"; +pub const ERROR_TARGET_NOT_FOUND: &str = "TARGET_NOT_FOUND"; diff --git a/crates/control/src/capability.rs b/crates/control/src/capability.rs new file mode 100644 index 0000000..5f326b6 --- /dev/null +++ b/crates/control/src/capability.rs @@ -0,0 +1,87 @@ +use lazyboy_contracts::RouteKind; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RouteDecision { + pub kind: RouteKind, + pub rationale: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RouteInputs { + pub has_native_api: bool, + pub policy_denied: bool, + pub account_mismatch: bool, + pub local_files: bool, + pub needs_pixels: bool, + pub has_dom: bool, +} + +/// Capability-aware preference. Policy denial is never a reason to try a +/// slower/weaker path that would bypass the grant. +pub fn choose_route(inputs: RouteInputs) -> Result { + if inputs.policy_denied || inputs.account_mismatch { + return Err("POLICY_DENIED"); + } + if inputs.has_native_api { + return Ok(RouteDecision { + kind: RouteKind::NativeApi, + rationale: "authorized structured API is available".into(), + }); + } + if inputs.local_files { + return Ok(RouteDecision { + kind: RouteKind::NativeFiles, + rationale: "task files are on the bound Computer".into(), + }); + } + if inputs.has_dom && !inputs.needs_pixels { + return Ok(RouteDecision { + kind: RouteKind::Dom, + rationale: "page has semantic targets".into(), + }); + } + if inputs.needs_pixels { + return Ok(RouteDecision { + kind: RouteKind::Vision, + rationale: "pixel-only control; requires a fresh screenshot".into(), + }); + } + Ok(RouteDecision { + kind: RouteKind::NativeExec, + rationale: "fall through to computer-local exec".into(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gmail_with_grant_does_not_use_the_browser() { + let decision = choose_route(RouteInputs { + has_native_api: true, + policy_denied: false, + account_mismatch: false, + local_files: false, + needs_pixels: false, + has_dom: true, + }) + .unwrap(); + assert_eq!(decision.kind, RouteKind::NativeApi); + } + + #[test] + fn policy_denied_does_not_fall_back_to_ui() { + assert_eq!( + choose_route(RouteInputs { + has_native_api: false, + policy_denied: true, + account_mismatch: false, + local_files: false, + needs_pixels: false, + has_dom: true, + }), + Err("POLICY_DENIED") + ); + } +} diff --git a/crates/control/src/controller.rs b/crates/control/src/controller.rs index 5b41227..0ccdeab 100644 --- a/crates/control/src/controller.rs +++ b/crates/control/src/controller.rs @@ -51,6 +51,10 @@ pub enum ControlError { TargetNotFound, #[error("stale UI reference; take a fresh observation")] StaleReference, + #[error("selector type is not supported by this browser backend")] + SelectorUnsupported, + #[error("target is disabled")] + TargetDisabled, #[error("permission denied")] PermissionDenied, /// Fail closed, in the same words the tool-layer timeout uses @@ -80,6 +84,8 @@ impl ControlError { self, Self::TargetNotFound | Self::StaleReference + | Self::SelectorUnsupported + | Self::TargetDisabled | Self::Unsupported | Self::InvalidAction(_) | Self::PermissionDenied diff --git a/crates/control/src/cua/browser.rs b/crates/control/src/cua/browser.rs index d464a4e..f9b687e 100644 --- a/crates/control/src/cua/browser.rs +++ b/crates/control/src/cua/browser.rs @@ -79,6 +79,7 @@ pub fn page_from_semantic(value: &Value) -> BrowserPage { text: outline.to_string(), restarted: false, waited_seconds: None, + error_code: None, elements, } } @@ -99,6 +100,16 @@ fn element_from_ref(id: u32, item: &Value) -> Option { .and_then(Value::as_str) .filter(|role| !role.is_empty()) .map(str::to_string); + let disabled = item + .get("disabled") + .and_then(Value::as_bool) + .unwrap_or(false) + || item + .get("states") + .and_then(Value::as_array) + .into_iter() + .flatten() + .any(|state| state.as_str() == Some("disabled")); let visibility = item .get("visibility") .and_then(Value::as_str) @@ -131,9 +142,71 @@ fn element_from_ref(id: u32, item: &Value) -> Option { selector: Some(selector), kind: Some("dom".into()), role, + disabled, }) } +/// How the model addressed a page control. Cua's semantic backend understands +/// snapshot refs (`p1:0`) and element ids from the last snapshot. CSS is +/// advertised in older schemas but is not implemented — callers must get a +/// typed SELECTOR_UNSUPPORTED rather than a fuzzy title match. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BrowserLocatorKind { + SnapshotRef, + ElementId, + Css, + RoleName, +} + +pub fn classify_browser_locator(selector: &str) -> BrowserLocatorKind { + let selector = selector.trim(); + if is_cua_ref(selector) { + BrowserLocatorKind::SnapshotRef + } else if selector.parse::().is_ok() { + BrowserLocatorKind::ElementId + } else if looks_like_css(selector) { + BrowserLocatorKind::Css + } else { + BrowserLocatorKind::RoleName + } +} + +fn looks_like_css(selector: &str) -> bool { + let trimmed = selector.trim(); + trimmed.starts_with('#') + || trimmed.starts_with('.') + || trimmed.contains('[') + || trimmed.contains('>') + || trimmed.contains('+') + || trimmed.contains('~') + || (trimmed.contains('.') && !trimmed.contains(' ')) +} + +fn locator_miss_code(kind: BrowserLocatorKind) -> &'static str { + match kind { + BrowserLocatorKind::SnapshotRef | BrowserLocatorKind::ElementId => { + crate::ERROR_TARGET_STALE + } + BrowserLocatorKind::Css => crate::ERROR_SELECTOR_UNSUPPORTED, + BrowserLocatorKind::RoleName => crate::ERROR_TARGET_NOT_FOUND, + } +} + +fn locator_miss_message(kind: BrowserLocatorKind) -> String { + let code = locator_miss_code(kind); + match kind { + BrowserLocatorKind::Css => format!( + "{code}: this backend accepts snapshot refs (p1:0), element ids, or an exact element title, not CSS selectors" + ), + BrowserLocatorKind::RoleName => format!( + "{code}: no element has exactly that title (or several do). Use the element id or snapshot ref from the element list in this result" + ), + BrowserLocatorKind::SnapshotRef | BrowserLocatorKind::ElementId => format!( + "{code}: element gone: the page changed and ids were renumbered. Use the fresh element list in this result." + ), + } +} + fn number(value: &Value, key: &str) -> Option { value .get(key) @@ -366,6 +439,13 @@ pub async fn run( sleep(Duration::from_millis(ms)).await; snapshot(client, display, &attached).await } + "click" => { + let mut current = attached; + let page = click(client, display, &mut current, request).await; + *bind = Some(current); + page + } + "type" => type_into(client, display, &attached, request).await, "navigate" => { let url = request.url.as_deref().unwrap_or(""); if !allowed_navigate_url(url) { @@ -387,8 +467,6 @@ pub async fn run( .await?; snapshot_when_document_ready(client, display, &attached).await } - "click" => click(client, display, &attached, request).await, - "type" => type_into(client, display, &attached, request).await, "press" => { let key = map_press_key(request.key.as_deref().unwrap_or("return")); client @@ -415,43 +493,93 @@ pub async fn run( async fn click( client: &CuaClient, display: &str, - bind: &BrowserBind, + bind: &mut BrowserBind, request: &BrowserRequest, ) -> Result { let selector = request .selector .as_deref() .ok_or_else(|| ControlError::InvalidAction("browser click needs a selector".into()))?; - let page = bind.page.as_ref().ok_or(ControlError::StaleReference)?; - let Some(r#ref) = find_ref(page, selector) else { - return Ok(BrowserPage { + // Exact, unique titles always win (a link named "v1.2 Release" is not a + // CSS selector). Only when nothing matches does the shape of the locator + // decide which error the model gets. + let kind = classify_browser_locator(selector); + let wait_ms = request.wait_ms.unwrap_or(45_000).min(120_000); + let started = Instant::now(); + loop { + if bind.page.is_none() { + let page = snapshot(client, display, bind).await?; + bind.page = Some(page); + } + let (r#ref, disabled) = { + let page = bind.page.as_ref().ok_or(ControlError::StaleReference)?; + match find_ref(page, selector) { + Some(r#ref) => { + let disabled = page + .elements + .iter() + .find(|element| element.selector.as_deref() == Some(r#ref)) + .is_some_and(|element| element.disabled); + (r#ref.to_string(), disabled) + } + None => { + let code = locator_miss_code(kind); + let page = page.clone(); + return Ok(BrowserPage { + ok: false, + error: Some(locator_miss_message(kind)), + error_code: Some(code.into()), + url: page.url, + title: page.title, + text: page.text, + elements: page.elements, + ..BrowserPage::default() + }); + } + } + }; + if !disabled { + client + .call( + display, + "browser_click", + &json!({ + "target_id": bind.target_id, + "tab_id": bind.tab_id, + "ref": r#ref, + "input_route": "dom_event", + }), + &[], + ) + .await?; + let mut page = snapshot(client, display, bind).await?; + let waited = started.elapsed().as_secs_f64(); + if waited >= 1.0 { + page.waited_seconds = Some(waited); + } + bind.page = Some(page.clone()); + return Ok(page); + } + if started.elapsed() >= Duration::from_millis(wait_ms) { + let page = bind.page.clone().unwrap_or_default(); + return Ok(BrowserPage { ok: false, error: Some( - "element gone: the page changed and ids were renumbered. Use the fresh element list in this result." - .into(), + "TARGET_DISABLED: the control stayed disabled until the wait deadline; no click was sent".into(), ), - url: page.url.clone(), - title: page.title.clone(), - text: page.text.clone(), - elements: page.elements.clone(), + error_code: Some(crate::ERROR_TARGET_DISABLED.into()), + waited_seconds: Some(started.elapsed().as_secs_f64()), + url: page.url, + title: page.title, + text: page.text, + elements: page.elements, ..BrowserPage::default() }); - }; - let r#ref = r#ref.to_string(); - client - .call( - display, - "browser_click", - &json!({ - "target_id": bind.target_id, - "tab_id": bind.tab_id, - "ref": r#ref, - "input_route": "dom_event", - }), - &[], - ) - .await?; - snapshot(client, display, bind).await + } + sleep(Duration::from_millis(200)).await; + let page = snapshot(client, display, bind).await?; + bind.page = Some(page); + } } async fn snapshot_when_document_ready( @@ -512,9 +640,12 @@ async fn type_into( if let Some(selector) = request.selector.as_deref() { let page = bind.page.as_ref().ok_or(ControlError::StaleReference)?; let Some(r#ref) = find_ref(page, selector) else { + let kind = classify_browser_locator(selector); + let code = locator_miss_code(kind); return Ok(BrowserPage { ok: false, - error: Some("target field is unavailable; no text inserted".into()), + error: Some(format!("{}; no text inserted", locator_miss_message(kind))), + error_code: Some(code.into()), url: page.url.clone(), title: page.title.clone(), text: page.text.clone(), @@ -656,6 +787,62 @@ mod tests { assert!(!is_cua_ref("p:1")); } + #[test] + fn classify_browser_locator_does_not_treat_css_as_a_title() { + assert_eq!( + classify_browser_locator("p1:1"), + BrowserLocatorKind::SnapshotRef + ); + assert_eq!( + classify_browser_locator("12"), + BrowserLocatorKind::ElementId + ); + assert_eq!(classify_browser_locator("#submit"), BrowserLocatorKind::Css); + assert_eq!( + classify_browser_locator("button.primary"), + BrowserLocatorKind::Css + ); + assert_eq!( + classify_browser_locator("Smoke Entry"), + BrowserLocatorKind::RoleName + ); + assert_eq!( + locator_miss_code(BrowserLocatorKind::Css), + crate::ERROR_SELECTOR_UNSUPPORTED + ); + assert_eq!( + locator_miss_code(BrowserLocatorKind::SnapshotRef), + crate::ERROR_TARGET_STALE + ); + } + + /// A title that happens to contain `.` or `+` is still a title when an + /// element carries it exactly; the CSS refusal only applies on a miss. + #[test] + fn exact_title_with_css_shaped_characters_still_resolves() { + let raw = json!({ + "status": "ok", + "page": { "title": "t", "url": "http://x" }, + "refs": [ + { "name": "v1.2 Release", "ref": "p1:1", "role": "link", "visibility": "in_viewport" }, + { "name": "C++ Guide", "ref": "p1:2", "role": "link", "visibility": "in_viewport" } + ] + }); + let page = page_from_semantic(&raw); + assert_eq!( + classify_browser_locator("v1.2 Release"), + BrowserLocatorKind::RoleName + ); + assert_eq!( + classify_browser_locator("C++ Guide"), + BrowserLocatorKind::Css + ); + assert_eq!(find_ref(&page, "v1.2 Release"), Some("p1:1")); + assert_eq!(find_ref(&page, "C++ Guide"), Some("p1:2")); + assert_eq!(find_ref(&page, "button.primary"), None); + assert!(locator_miss_message(BrowserLocatorKind::Css).contains("SELECTOR_UNSUPPORTED")); + } + #[test] fn semantic_snapshot_becomes_browser_page() { let raw = json!({ @@ -684,6 +871,18 @@ mod tests { assert_eq!(find_ref(&page, "Smoke Entry"), Some("p1:2")); } + #[test] + fn semantic_snapshot_marks_disabled_controls() { + let raw = json!({ + "status": "ok", + "refs": [ + { "name": "Next", "ref": "p1:1", "role": "button", "disabled": true, "visibility": "in_viewport" } + ] + }); + let page = page_from_semantic(&raw); + assert!(page.elements[0].disabled); + } + #[test] fn navigation_snapshot_is_ready_once_a_url_is_present() { let mut page = BrowserPage { diff --git a/crates/control/src/cua/mod.rs b/crates/control/src/cua/mod.rs index de91f29..0b102a2 100644 --- a/crates/control/src/cua/mod.rs +++ b/crates/control/src/cua/mod.rs @@ -29,6 +29,7 @@ use crate::{ }; use client::{action_verdict, first_array_of_objects}; +pub use browser::{BrowserLocatorKind, classify_browser_locator, is_cua_ref}; pub use client::CuaClient; /// Last resort when neither the screenshot nor the driver reports a mode. @@ -547,6 +548,7 @@ impl CuaController { selector: None, kind: Some("window".into()), role: None, + disabled: false, }) .collect::>(), ); diff --git a/crates/control/src/cua/native.rs b/crates/control/src/cua/native.rs index b793139..cfda1c4 100644 --- a/crates/control/src/cua/native.rs +++ b/crates/control/src/cua/native.rs @@ -101,6 +101,7 @@ pub(super) async fn observe( selector: Some(selector.clone()), kind: Some("a11y".into()), role: Some(role.into()), + disabled: false, }); observed.targets.insert( selector, diff --git a/crates/control/src/display_backend.rs b/crates/control/src/display_backend.rs new file mode 100644 index 0000000..aba0f3b --- /dev/null +++ b/crates/control/src/display_backend.rs @@ -0,0 +1,110 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DisplayBackend { + XvfbX11vnc, + TigerVncXvnc, +} + +impl DisplayBackend { + pub const ENV: &'static str = "LAZYBOY_DISPLAY_BACKEND"; + + pub fn from_env() -> Self { + Self::parse(&std::env::var(Self::ENV).unwrap_or_default()) + } + + pub fn parse(value: &str) -> Self { + match value.trim() { + "tigervnc_xvnc" => Self::TigerVncXvnc, + _ => Self::XvfbX11vnc, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::XvfbX11vnc => "xvfb_x11vnc", + Self::TigerVncXvnc => "tigervnc_xvnc", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DisplaySlot { + pub slot: u32, + pub display_number: u32, + pub rfb_port: u16, + pub view_port: u16, +} + +/// Slot 0 is DISPLAY :1 / RFB 5900 / web 6080. Do not use Xvnc's default +/// 5900+display-number mapping, which would shift every existing port. +pub fn slot_endpoints(slot: u32) -> DisplaySlot { + DisplaySlot { + slot, + display_number: slot + 1, + rfb_port: 5900 + slot as u16, + view_port: 6080 + slot as u16, + } +} + +pub fn display_name(slot: u32) -> String { + format!(":{}", slot_endpoints(slot).display_number) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum X11Access { + /// Intra-container X clients (Cua, AT-SPI, x11vnc) currently rely on + /// `Xvfb -ac`. T59 still holds: the public VNC/noVNC path is the + /// authenticated screen proxy, not a naked 5900. Shipping Xauthority + /// requires giving every in-image X client the cookie; deleting `-ac` + /// alone would break those clients. + KeepAcUntilXauthority, +} + +pub fn x11_access_control() -> X11Access { + X11Access::KeepAcUntilXauthority +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn slot_zero_keeps_the_existing_port_map() { + let slot = slot_endpoints(0); + assert_eq!(slot.display_number, 1); + assert_eq!(slot.rfb_port, 5900); + assert_eq!(slot.view_port, 6080); + assert_eq!(display_name(0), ":1"); + } + + #[test] + fn slot_one_is_display_two_rfb_5901() { + let slot = slot_endpoints(1); + assert_eq!(slot.display_number, 2); + assert_eq!(slot.rfb_port, 5901); + assert_eq!(slot.view_port, 6081); + } + + #[test] + fn t59_keeps_ac_until_xauthority_is_wired() { + assert_eq!(x11_access_control(), X11Access::KeepAcUntilXauthority); + let script = include_str!("../../../image/computer/lazyboy-screen"); + assert!( + script.contains(" -ac "), + "removing -ac without Xauthority breaks Cua/AT-SPI inside the Computer" + ); + assert!( + !script.contains("xauth generate"), + "do not claim Xauthority is wired while -ac remains" + ); + } + + #[test] + fn default_backend_stays_xvfb_until_tigervnc_is_chosen() { + assert_eq!(DisplayBackend::parse(""), DisplayBackend::XvfbX11vnc); + assert_eq!( + DisplayBackend::parse("tigervnc_xvnc"), + DisplayBackend::TigerVncXvnc + ); + assert_eq!(DisplayBackend::XvfbX11vnc.as_str(), "xvfb_x11vnc"); + } +} diff --git a/crates/control/src/file_cas.rs b/crates/control/src/file_cas.rs new file mode 100644 index 0000000..23a443c --- /dev/null +++ b/crates/control/src/file_cas.rs @@ -0,0 +1,48 @@ +use sha2::{Digest, Sha256}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FileCasError { + Conflict { expected: String, actual: String }, +} + +pub fn content_hash(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} + +/// Optimistic write: if the caller read `expected` and the file has moved, +/// refuse rather than overwrite. +pub fn check_expected_hash( + current: Option<&[u8]>, + expected: Option<&str>, +) -> Result<(), FileCasError> { + let Some(expected) = expected.filter(|value| !value.is_empty()) else { + return Ok(()); + }; + let actual = current.map(content_hash).unwrap_or_default(); + if actual != expected { + return Err(FileCasError::Conflict { + expected: expected.to_string(), + actual, + }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn matching_hash_allows_the_write() { + let bytes = b"hello"; + let hash = content_hash(bytes); + assert!(check_expected_hash(Some(bytes), Some(&hash)).is_ok()); + } + + #[test] + fn stale_hash_is_a_conflict_not_an_overwrite() { + let hash = content_hash(b"old"); + let error = check_expected_hash(Some(b"new"), Some(&hash)).unwrap_err(); + assert!(matches!(error, FileCasError::Conflict { .. })); + } +} diff --git a/crates/control/src/form.rs b/crates/control/src/form.rs new file mode 100644 index 0000000..0fe9fab --- /dev/null +++ b/crates/control/src/form.rs @@ -0,0 +1,94 @@ +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FormField { + pub name: String, + pub value: String, + pub locator: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FormStepResult { + pub field: String, + pub ok: bool, + pub evidence: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FormMacroResult { + pub ok: bool, + pub steps: Vec, +} + +/// Bounded form fill: each field is re-located and checked. Failure stops the +/// rest of the macro instead of blasting stale ids. +pub fn run_form_macro( + fields: &[FormField], + locate: impl Fn(&str) -> Option, + fill: impl Fn(&str, &str) -> Result, + submit_ready: impl Fn() -> bool, +) -> FormMacroResult { + let mut steps = Vec::new(); + for field in fields.iter().take(20) { + let Some(current) = locate(&field.locator) else { + steps.push(FormStepResult { + field: field.name.clone(), + ok: false, + evidence: "TARGET_STALE".into(), + }); + return FormMacroResult { ok: false, steps }; + }; + match fill(¤t, &field.value) { + Ok(evidence) => steps.push(FormStepResult { + field: field.name.clone(), + ok: true, + evidence, + }), + Err(error) => { + steps.push(FormStepResult { + field: field.name.clone(), + ok: false, + evidence: error, + }); + return FormMacroResult { ok: false, steps }; + } + } + } + if !submit_ready() { + steps.push(FormStepResult { + field: "submit".into(), + ok: false, + evidence: "precondition failed".into(), + }); + return FormMacroResult { ok: false, steps }; + } + FormMacroResult { ok: true, steps } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_missing_field_stops_the_macro() { + let fields = vec![ + FormField { + name: "a".into(), + value: "1".into(), + locator: "p1:1".into(), + }, + FormField { + name: "b".into(), + value: "2".into(), + locator: "p1:2".into(), + }, + ]; + let result = run_form_macro( + &fields, + |locator| (locator == "p1:1").then(|| locator.to_string()), + |_, value| Ok(value.to_string()), + || true, + ); + assert!(!result.ok); + assert_eq!(result.steps.len(), 2); + assert!(!result.steps[1].ok); + } +} diff --git a/crates/control/src/gmail.rs b/crates/control/src/gmail.rs new file mode 100644 index 0000000..3b13684 --- /dev/null +++ b/crates/control/src/gmail.rs @@ -0,0 +1,236 @@ +const GMAIL_BATCH_MODIFY_MAX: usize = 1000; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GmailMessage { + pub id: String, + pub labels: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LabelDelta { + pub add: Vec, + pub remove: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ModifyBatch { + pub ids: Vec, + pub delta: LabelDelta, +} + +/// Group messages that share the same add/remove set. Each batch is at most +/// 1000 ids, matching users.messages.batchModify. +pub fn plan_batch_modify( + messages: &[GmailMessage], + delta_for: impl Fn(&GmailMessage) -> LabelDelta, +) -> Vec { + let mut groups: Vec<(LabelDelta, Vec)> = Vec::new(); + for message in messages { + let delta = delta_for(message); + if let Some((_, ids)) = groups.iter_mut().find(|(existing, _)| existing == &delta) { + ids.push(message.id.clone()); + } else { + groups.push((delta, vec![message.id.clone()])); + } + } + let mut batches = Vec::new(); + for (delta, ids) in groups { + for chunk in ids.chunks(GMAIL_BATCH_MODIFY_MAX) { + batches.push(ModifyBatch { + ids: chunk.to_vec(), + delta: delta.clone(), + }); + } + } + batches +} + +pub fn apply_delta(labels: &[String], delta: &LabelDelta) -> Vec { + let mut next: Vec = labels + .iter() + .filter(|label| !delta.remove.contains(label)) + .cloned() + .collect(); + for label in &delta.add { + if !next.contains(label) { + next.push(label.clone()); + } + } + next +} + +/// Inverse of a plan, computed against the labels each message had *before* +/// the plan ran. Only what this plan actually changed is reverted: a label the +/// message already carried is not stripped, one it never had is not re-added, +/// and labels a human added in the meantime are left alone. +pub fn undo_plan(before: &[GmailMessage], batches: &[ModifyBatch]) -> Vec { + let mut deltas: Vec<(String, LabelDelta)> = Vec::new(); + for batch in batches { + for id in &batch.ids { + let Some(message) = before.iter().find(|message| &message.id == id) else { + continue; + }; + let inverse = LabelDelta { + add: batch + .delta + .remove + .iter() + .filter(|label| message.labels.contains(label)) + .cloned() + .collect(), + remove: batch + .delta + .add + .iter() + .filter(|label| !message.labels.contains(label)) + .cloned() + .collect(), + }; + if inverse.add.is_empty() && inverse.remove.is_empty() { + continue; + } + deltas.push((id.clone(), inverse)); + } + } + let messages: Vec = deltas + .iter() + .map(|(id, _)| GmailMessage { + id: id.clone(), + labels: Vec::new(), + }) + .collect(); + plan_batch_modify(&messages, |message| { + deltas + .iter() + .find(|(id, _)| id == &message.id) + .map(|(_, delta)| delta.clone()) + .unwrap_or(LabelDelta { + add: Vec::new(), + remove: Vec::new(), + }) + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VerifyStatus { + Matched, + Conflict, + Missing, +} + +pub fn verify_labels(actual: &[String], expected: &[String]) -> VerifyStatus { + let mut left = actual.to_vec(); + let mut right = expected.to_vec(); + left.sort(); + right.sort(); + if left == right { + VerifyStatus::Matched + } else { + VerifyStatus::Conflict + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn batches_split_on_delta_and_the_1000_cap() { + let messages: Vec<_> = (0..1005) + .map(|index| GmailMessage { + id: format!("m{index}"), + labels: vec!["INBOX".into()], + }) + .collect(); + let batches = plan_batch_modify(&messages, |_| LabelDelta { + add: vec!["work".into()], + remove: vec![], + }); + assert_eq!(batches.len(), 2); + assert_eq!(batches[0].ids.len(), 1000); + assert_eq!(batches[1].ids.len(), 5); + } + + #[test] + fn undo_only_reverts_this_plans_delta() { + let after = apply_delta( + &["INBOX".into(), "work".into(), "human".into()], + &LabelDelta { + add: vec![], + remove: vec!["work".into()], + }, + ); + assert!(after.contains(&"human".into())); + assert!(!after.contains(&"work".into())); + } + + #[test] + fn undo_plan_reverts_only_what_the_plan_changed() { + let before = vec![ + // Already had "work": the plan's add was a no-op here, so undo must + // not remove it. + GmailMessage { + id: "m1".into(), + labels: vec!["INBOX".into(), "work".into()], + }, + // Gained "work" and lost "INBOX": both get reverted. + GmailMessage { + id: "m2".into(), + labels: vec!["INBOX".into()], + }, + // Never had "INBOX": the plan's remove was a no-op, nothing to re-add. + GmailMessage { + id: "m3".into(), + labels: vec![], + }, + ]; + let plan = plan_batch_modify(&before, |_| LabelDelta { + add: vec!["work".into()], + remove: vec!["INBOX".into()], + }); + let undo = undo_plan(&before, &plan); + let delta_for = |id: &str| { + undo.iter() + .find(|batch| batch.ids.iter().any(|item| item == id)) + .map(|batch| batch.delta.clone()) + }; + assert_eq!( + delta_for("m1"), + Some(LabelDelta { + add: vec!["INBOX".into()], + remove: vec![], + }) + ); + assert_eq!( + delta_for("m2"), + Some(LabelDelta { + add: vec!["INBOX".into()], + remove: vec!["work".into()], + }) + ); + assert_eq!( + delta_for("m3"), + Some(LabelDelta { + add: vec![], + remove: vec!["work".into()], + }) + ); + // Applying plan then undo lands back where each message started. + for message in &before { + let after_plan = apply_delta( + &message.labels, + &LabelDelta { + add: vec!["work".into()], + remove: vec!["INBOX".into()], + }, + ); + let restored = apply_delta(&after_plan, &delta_for(&message.id).unwrap()); + assert_eq!( + verify_labels(&restored, &message.labels), + VerifyStatus::Matched, + "{}", + message.id + ); + } + } +} diff --git a/crates/control/src/jobs.rs b/crates/control/src/jobs.rs new file mode 100644 index 0000000..ab4f3fe --- /dev/null +++ b/crates/control/src/jobs.rs @@ -0,0 +1,729 @@ +use std::collections::HashMap; +use std::process::Stdio; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use tokio::process::Command; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JobStatus { + Running, + Succeeded, + Failed, + Cancelled, + Interrupted, +} + +#[derive(Debug, Clone)] +pub struct JobSnapshot { + pub id: String, + pub status: JobStatus, + pub pid: Option, + pub stdout: String, + pub stderr: String, + pub exit_code: Option, + pub computer_generation: i32, + pub computer_id: Option, +} + +struct LiveJob { + snapshot: JobSnapshot, + child: Option, + pgid: Option, + started: Instant, +} + +pub struct JobSupervisor { + inner: Mutex>, +} + +impl Default for JobSupervisor { + fn default() -> Self { + Self { + inner: Mutex::new(HashMap::new()), + } + } +} + +impl JobSupervisor { + pub fn get(&self, id: &str) -> Option { + self.inner + .lock() + .unwrap() + .get(id) + .map(|job| job.snapshot.clone()) + } + + pub fn running_count(&self) -> usize { + self.inner + .lock() + .unwrap() + .values() + .filter(|job| job.snapshot.status == JobStatus::Running) + .count() + } + + pub fn running_count_for(&self, computer_id: &str) -> usize { + self.inner + .lock() + .unwrap() + .values() + .filter(|job| { + job.snapshot.status == JobStatus::Running + && job.snapshot.computer_id.as_deref() == Some(computer_id) + }) + .count() + } + + /// After a container recreate, leftover job ids must not look alive. + pub fn interrupt_generation(&self, generation: i32) { + let mut map = self.inner.lock().unwrap(); + for job in map.values_mut() { + if job.snapshot.computer_generation != generation + && job.snapshot.status == JobStatus::Running + { + job.snapshot.status = JobStatus::Interrupted; + job.child = None; + } + } + } + + pub fn interrupt_computer(&self, computer_id: &str) { + let mut map = self.inner.lock().unwrap(); + for job in map.values_mut() { + if job.snapshot.computer_id.as_deref() == Some(computer_id) + && job.snapshot.status == JobStatus::Running + { + job.snapshot.status = JobStatus::Interrupted; + job.child = None; + } + } + } + + pub async fn start( + &self, + id: &str, + argv: &[String], + generation: i32, + computer_id: Option, + ) -> Result { + if argv.is_empty() { + return Err("argv required".into()); + } + let mut command = Command::new(&argv[0]); + if argv.len() > 1 { + command.args(&argv[1..]); + } + command.stdout(Stdio::piped()).stderr(Stdio::piped()); + #[cfg(unix)] + { + command.process_group(0); + } + let child = command.spawn().map_err(|error| error.to_string())?; + let pid = child.id(); + let snapshot = JobSnapshot { + id: id.to_string(), + status: JobStatus::Running, + pid, + stdout: String::new(), + stderr: String::new(), + exit_code: None, + computer_generation: generation, + computer_id, + }; + self.inner.lock().unwrap().insert( + id.to_string(), + LiveJob { + snapshot: snapshot.clone(), + child: Some(child), + pgid: pid.map(|pid| pid as i32), + started: Instant::now(), + }, + ); + Ok(snapshot) + } + + pub async fn wait(&self, id: &str, timeout: Duration) -> Result { + let started = Instant::now(); + loop { + self.poll(id).await?; + let snapshot = self.get(id).ok_or_else(|| "unknown job".to_string())?; + if snapshot.status != JobStatus::Running { + return Ok(snapshot); + } + if started.elapsed() >= timeout { + return Ok(snapshot); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + + pub async fn poll(&self, id: &str) -> Result { + let mut child = { + let mut map = self.inner.lock().unwrap(); + let job = map.get_mut(id).ok_or_else(|| "unknown job".to_string())?; + job.child.take() + }; + if let Some(mut live) = child.take() { + match live.try_wait() { + Ok(Some(status)) => { + let stdout = child_output(&mut live, true).await; + let stderr = child_output(&mut live, false).await; + let mut map = self.inner.lock().unwrap(); + if let Some(job) = map.get_mut(id) { + job.snapshot.status = if status.success() { + JobStatus::Succeeded + } else { + JobStatus::Failed + }; + job.snapshot.exit_code = status.code(); + job.snapshot.stdout = stdout; + job.snapshot.stderr = stderr; + job.child = None; + return Ok(job.snapshot.clone()); + } + } + Ok(None) => { + let mut map = self.inner.lock().unwrap(); + if let Some(job) = map.get_mut(id) { + job.child = Some(live); + return Ok(job.snapshot.clone()); + } + } + Err(error) => return Err(error.to_string()), + } + } + self.get(id).ok_or_else(|| "unknown job".to_string()) + } + + /// TERM the process group, then KILL after grace. Never claims cancel until + /// the tree is actually gone. + pub async fn cancel(&self, id: &str) -> Result { + let (pid, pgid) = { + let map = self.inner.lock().unwrap(); + let job = map.get(id).ok_or_else(|| "unknown job".to_string())?; + (job.snapshot.pid, job.pgid) + }; + if let Some(pgid) = pgid { + let _ = Command::new("kill") + .args(["-TERM", "--", &format!("-{pgid}")]) + .status() + .await; + tokio::time::sleep(Duration::from_millis(50)).await; + let still = self.poll(id).await?; + if still.status == JobStatus::Running { + let _ = Command::new("kill") + .args(["-KILL", "--", &format!("-{pgid}")]) + .status() + .await; + } + } else if let Some(pid) = pid { + let _ = Command::new("kill") + .args(["-TERM", &pid.to_string()]) + .status() + .await; + } + let _ = pid; + let mut snapshot = self.poll(id).await?; + if snapshot.status == JobStatus::Running { + // Process did not exit in time: be honest rather than claim cancelled. + return Ok(snapshot); + } + if snapshot.status != JobStatus::Succeeded { + snapshot.status = JobStatus::Cancelled; + if let Some(job) = self.inner.lock().unwrap().get_mut(id) { + job.snapshot.status = JobStatus::Cancelled; + } + } + Ok(self.get(id).unwrap_or(snapshot)) + } + + pub fn age(&self, id: &str) -> Option { + self.inner + .lock() + .unwrap() + .get(id) + .map(|job| job.started.elapsed()) + } +} + +/// `LAZYBOY_NATIVE_JOB_CONCURRENCY`, default 2, hard-capped. +pub fn native_job_concurrency() -> usize { + std::env::var("LAZYBOY_NATIVE_JOB_CONCURRENCY") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(2) + .clamp(1, 32) +} + +pub fn job_quota_exceeded(running: usize, limit: usize) -> bool { + running >= limit +} + +/// Tracker for jobs that already run on the Computer. Does not spawn host processes. +#[derive(Default)] +pub struct ComputerJobTable { + inner: Mutex>, +} + +impl ComputerJobTable { + pub fn insert(&self, snapshot: JobSnapshot) { + self.inner + .lock() + .unwrap() + .insert(snapshot.id.clone(), snapshot); + } + + pub fn get(&self, id: &str) -> Option { + self.inner.lock().unwrap().get(id).cloned() + } + + pub fn update(&self, id: &str, mutate: impl FnOnce(&mut JobSnapshot)) -> Option { + let mut map = self.inner.lock().unwrap(); + let job = map.get_mut(id)?; + mutate(job); + Some(job.clone()) + } + + pub fn running_count(&self) -> usize { + self.inner + .lock() + .unwrap() + .values() + .filter(|job| job.status == JobStatus::Running) + .count() + } +} + +pub fn sanitize_job_id(id: &str) -> String { + let cleaned: String = id + .chars() + .filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_') + .take(80) + .collect(); + if cleaned.is_empty() { + "job".into() + } else { + cleaned + } +} + +pub fn posix_shell_join(argv: &[String]) -> String { + argv.iter() + .map(|arg| { + if arg + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_./:@+".contains(&b)) + { + arg.clone() + } else { + format!("'{}'", arg.replace('\'', "'\\''")) + } + }) + .collect::>() + .join(" ") +} + +/// Launch argv inside the Computer so the process is not a child of the API. +/// The wrapper writes `$?` to `/tmp/lazyboy/jobs/{id}/exit` when the job exits +/// and prints the pid of the `setsid` session leader (the `sh` wrapper), which +/// is also the process-group id used by [`computer_cancel_script`]. +/// +/// The joined command is a separate argv element (`$1`), not interpolated into +/// `sh -c '…'`. Nested single quotes from `posix_shell_join` must survive. +/// +/// `mkdir` is a separate statement on purpose: `a && b &` backgrounds the whole +/// list in a subshell and `$!` would then be that subshell, not the job. The +/// wrapper also records its own `$$` so the reported pid is right even when +/// `setsid` had to fork. +pub fn computer_background_launch(argv: &[String], job_id: &str) -> Vec { + let id = sanitize_job_id(job_id); + let dir = format!("/tmp/lazyboy/jobs/{id}"); + let cmd = posix_shell_join(argv); + let script = format!( + r#"mkdir -p {dir} || exit 1 +rm -f {dir}/pid {dir}/exit +setsid nohup sh -c 'printf %s "$$" >{dir}/pid; eval "$1"; echo $? > {dir}/exit' _ "$1" >{dir}/out 2>{dir}/err /dev/null || break + sleep 0.02 +done +if [ -s {dir}/pid ]; then cat {dir}/pid; else echo "$launcher"; fi"# + ); + vec!["bash".into(), "-c".into(), script, "bash".into(), cmd] +} + +/// Shell run inside the Computer to stop a background job started by +/// [`computer_background_launch`]. The wrapper is a `setsid` session leader, so +/// `-pid` addresses its whole process group. Exits 0 only when nothing in the +/// group is left; exit 1 means the tree survived TERM and KILL and the caller +/// must keep reporting the job as running. +pub fn computer_cancel_script(pid: u32) -> String { + format!( + "kill -TERM -- -{pid} 2>/dev/null || kill -TERM {pid} 2>/dev/null; \ + for _ in 1 2 3 4 5 6 7 8 9 10; do kill -0 -- -{pid} 2>/dev/null || kill -0 {pid} 2>/dev/null || exit 0; sleep 0.1; done; \ + kill -KILL -- -{pid} 2>/dev/null || kill -KILL {pid} 2>/dev/null; sleep 0.1; \ + if kill -0 -- -{pid} 2>/dev/null || kill -0 {pid} 2>/dev/null; then exit 1; fi; exit 0" + ) +} + +/// Liveness probe for a Computer-side job. `kill -0` alone is not enough: a +/// wrapper whose parent never reaped it is a zombie that still answers, so the +/// process state is checked too. Exit 0 = something in the group is genuinely +/// running. +pub fn computer_alive_script(pid: u32) -> String { + format!( + "for p in $(pgrep -g {pid} 2>/dev/null) {pid}; do \ + s=$(awk '/^State:/ {{print $2}}' /proc/$p/status 2>/dev/null); \ + case \"$s\" in ''|Z|X) ;; *) exit 0;; esac; \ + done; exit 1" + ) +} + +/// Reap a Computer-side background job from pid liveness and the exit file. +/// Missing or unreadable `$?` is Failed/1, never Succeeded/0. +pub fn reap_background_job(pid_alive: bool, exit_file: Option<&str>) -> (JobStatus, i32) { + if pid_alive { + return (JobStatus::Running, 0); + } + let trimmed = exit_file.map(str::trim).filter(|text| !text.is_empty()); + match trimmed { + Some("0") => (JobStatus::Succeeded, 0), + Some(raw) if raw.eq_ignore_ascii_case("false") => (JobStatus::Failed, 1), + Some(raw) => { + let code = raw.parse::().unwrap_or(1); + if code == 0 { + (JobStatus::Succeeded, 0) + } else { + (JobStatus::Failed, code) + } + } + None => (JobStatus::Failed, 1), + } +} + +pub fn job_snapshot_result(job: &JobSnapshot) -> crate::CommandResult { + crate::CommandResult { + stdout: job.stdout.clone(), + stderr: job.stderr.clone(), + code: job.exit_code.unwrap_or(0), + job_id: Some(job.id.clone()), + signal: None, + status: Some(format!("{:?}", job.status).to_ascii_lowercase()), + } +} + +/// True when this process has a direct child whose cmdline contains `needle`. +pub fn current_process_has_child_cmd(needle: &str) -> bool { + let self_pid = std::process::id(); + let Ok(entries) = std::fs::read_dir("/proc") else { + return false; + }; + for entry in entries.flatten() { + let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { + continue; + }; + let status = std::fs::read_to_string(format!("/proc/{pid}/status")).unwrap_or_default(); + let ppid = status.lines().find_map(|line| { + line.strip_prefix("PPid:") + .and_then(|rest| rest.trim().parse::().ok()) + }); + if ppid != Some(self_pid) { + continue; + } + let cmd = std::fs::read(format!("/proc/{pid}/cmdline")).unwrap_or_default(); + let cmd: Vec = cmd + .iter() + .map(|byte| if *byte == 0 { b' ' } else { *byte }) + .collect(); + if String::from_utf8_lossy(&cmd).contains(needle) { + return true; + } + } + false +} + +async fn child_output(child: &mut tokio::process::Child, stdout: bool) -> String { + use tokio::io::AsyncReadExt; + let mut buf = String::new(); + if stdout { + if let Some(pipe) = child.stdout.as_mut() { + let mut bytes = Vec::new(); + let _ = pipe.read_to_end(&mut bytes).await; + buf = String::from_utf8_lossy(&bytes).into_owned(); + } + } else if let Some(pipe) = child.stderr.as_mut() { + let mut bytes = Vec::new(); + let _ = pipe.read_to_end(&mut bytes).await; + buf = String::from_utf8_lossy(&bytes).into_owned(); + } + buf +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn short_command_returns_real_exit_status() { + let jobs = JobSupervisor::default(); + jobs.start("j1", &["/bin/true".into()], 1, None) + .await + .unwrap(); + let done = jobs.wait("j1", Duration::from_secs(2)).await.unwrap(); + assert_eq!(done.status, JobStatus::Succeeded); + assert_eq!(done.exit_code, Some(0)); + } + + #[tokio::test] + async fn cancel_stops_a_sleeping_process_tree() { + let jobs = JobSupervisor::default(); + jobs.start( + "j2", + &["/bin/sh".into(), "-c".into(), "sleep 30".into()], + 1, + None, + ) + .await + .unwrap(); + let cancelled = jobs.cancel("j2").await.unwrap(); + assert_ne!(cancelled.status, JobStatus::Running); + assert!(matches!( + cancelled.status, + JobStatus::Cancelled | JobStatus::Failed | JobStatus::Succeeded + )); + } + + #[test] + fn old_generation_jobs_are_interrupted_not_alive() { + let jobs = JobSupervisor::default(); + jobs.inner.lock().unwrap().insert( + "old".into(), + LiveJob { + snapshot: JobSnapshot { + id: "old".into(), + status: JobStatus::Running, + pid: Some(1), + stdout: String::new(), + stderr: String::new(), + exit_code: None, + computer_generation: 1, + computer_id: Some("comp-a".into()), + }, + child: None, + pgid: None, + started: Instant::now(), + }, + ); + jobs.interrupt_generation(2); + assert_eq!(jobs.get("old").unwrap().status, JobStatus::Interrupted); + } + + #[test] + fn computer_background_launch_runs_inside_the_computer() { + let argv = computer_background_launch(&["/bin/sleep".into(), "30".into()], "job-abc"); + assert_eq!(argv[0], "bash"); + let script = &argv[2]; + assert!(script.contains("/tmp/lazyboy/jobs/job-abc")); + assert!(script.contains("/tmp/lazyboy/jobs/job-abc/exit")); + assert!(script.contains("echo $?")); + assert!(script.contains("eval \"$1\"")); + assert!(script.contains("setsid")); + assert!(!script.contains("Command::new")); + assert!(argv.iter().any(|part| part.contains("/bin/sleep 30"))); + } + + #[test] + fn background_launch_keeps_posix_quoted_arguments() { + let args = vec!["echo".into(), "hello world".into()]; + let quoted = posix_shell_join(&args); + assert_eq!(quoted, "echo 'hello world'"); + let launch = computer_background_launch(&args, "job-q"); + assert!( + launch.iter().any(|part| part == "ed), + "quoted argv must be a separate argument, not spliced into sh -c '...'" + ); + let script = &launch[2]; + assert!( + !script.contains("sh -c 'echo "), + "must not wrap posix_shell_join output in another layer of single quotes" + ); + assert!(script.contains("eval \"$1\"")); + let ran = std::process::Command::new("sh") + .args(["-c", "eval \"$1\"", "_", "ed]) + .output() + .unwrap(); + assert!(ran.status.success()); + assert_eq!(String::from_utf8_lossy(&ran.stdout).trim(), "hello world"); + } + + #[test] + fn computer_cancel_script_kills_the_whole_process_tree() { + // Same launcher the supervisor uses inside the Computer, run locally. + let job_id = format!("cancel-{}", std::process::id()); + let launch = computer_background_launch( + &["/bin/sh".into(), "-c".into(), "sleep 300; sleep 300".into()], + &job_id, + ); + let launched = std::process::Command::new(&launch[0]) + .args(&launch[1..]) + .output() + .unwrap(); + assert!(launched.status.success(), "{launched:?}"); + let pid: u32 = String::from_utf8_lossy(&launched.stdout) + .trim() + .lines() + .last() + .unwrap() + .trim() + .parse() + .unwrap(); + std::thread::sleep(Duration::from_millis(200)); + let find_sleep = || { + std::process::Command::new("pgrep") + .args(["-g", &pid.to_string(), "-x", "sleep"]) + .output() + .map(|out| out.status.success()) + .unwrap_or(false) + }; + assert!( + find_sleep(), + "a sleep must be running in the job's process group" + ); + + let alive = |pid: u32| { + std::process::Command::new("bash") + .args(["-c", &computer_alive_script(pid)]) + .status() + .map(|status| status.success()) + .unwrap_or(false) + }; + assert!(alive(pid), "running job must probe alive"); + + let cancelled = std::process::Command::new("bash") + .args(["-c", &computer_cancel_script(pid)]) + .output() + .unwrap(); + assert!(cancelled.status.success(), "{cancelled:?}"); + assert!( + !find_sleep(), + "the child sleep must be gone, not just the sh wrapper" + ); + assert!(!alive(pid), "cancelled job must not probe alive"); + let _ = std::fs::remove_dir_all(format!("/tmp/lazyboy/jobs/{job_id}")); + } + + /// A finished wrapper nobody reaped is a zombie: `kill -0` still succeeds, + /// but the job is over and must not be reported as running. + #[test] + fn computer_alive_script_treats_a_zombie_as_dead() { + // A child that exits at once while its parent (this sh) keeps living + // without wait(1)-ing leaves a zombie whose pid we can probe. + let mut parent = std::process::Command::new("sh") + .args(["-c", "sh -c 'exit 0' & echo $!; exec sleep 30"]) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let mut stdout = std::io::BufReader::new(parent.stdout.take().unwrap()); + let mut line = String::new(); + std::io::BufRead::read_line(&mut stdout, &mut line).unwrap(); + let zombie: u32 = line.trim().parse().unwrap(); + std::thread::sleep(Duration::from_millis(200)); + let kill0 = std::process::Command::new("kill") + .args(["-0", &zombie.to_string()]) + .status() + .unwrap() + .success(); + let alive = std::process::Command::new("bash") + .args(["-c", &computer_alive_script(zombie)]) + .status() + .unwrap() + .success(); + let _ = parent.kill(); + let _ = parent.wait(); + assert!(kill0, "precondition: the zombie still answers kill -0"); + assert!(!alive, "a zombie must count as finished"); + } + + #[test] + fn reap_background_job_failed_exit_is_not_success() { + assert_eq!(reap_background_job(true, None), (JobStatus::Running, 0)); + assert_eq!( + reap_background_job(false, Some("0")), + (JobStatus::Succeeded, 0) + ); + assert_eq!( + reap_background_job(false, Some("1")), + (JobStatus::Failed, 1) + ); + assert_eq!( + reap_background_job(false, Some("false")), + (JobStatus::Failed, 1) + ); + assert_eq!(reap_background_job(false, None), (JobStatus::Failed, 1)); + assert_eq!( + reap_background_job(false, Some(" ")), + (JobStatus::Failed, 1) + ); + } + + #[test] + fn command_result_json_keeps_job_id_status_and_code() { + let result = crate::CommandResult { + stdout: String::new(), + stderr: String::new(), + code: 1, + job_id: Some("job-1".into()), + signal: None, + status: Some("failed".into()), + }; + let value = serde_json::to_value(&result).unwrap(); + let back: crate::CommandResult = serde_json::from_value(value).unwrap(); + assert_eq!(back.job_id.as_deref(), Some("job-1")); + assert_eq!(back.status.as_deref(), Some("failed")); + assert_eq!(back.code, 1); + } + + #[test] + fn supervisor_exec_returns_full_command_result() { + let src = include_str!("../../../crates/supervisor/src/main.rs"); + assert!( + !src.contains("\"stdout\": result.stdout"), + "exec HTTP must serialize CommandResult, not a stdout/stderr/code subset" + ); + assert!(src.contains("Ok(Json(result))")); + } + + #[test] + fn running_count_and_quota_are_per_computer() { + let jobs = JobSupervisor::default(); + jobs.inner.lock().unwrap().insert( + "a".into(), + LiveJob { + snapshot: JobSnapshot { + id: "a".into(), + status: JobStatus::Running, + pid: None, + stdout: String::new(), + stderr: String::new(), + exit_code: None, + computer_generation: 1, + computer_id: Some("comp-a".into()), + }, + child: None, + pgid: None, + started: Instant::now(), + }, + ); + assert_eq!(jobs.running_count(), 1); + assert_eq!(jobs.running_count_for("comp-a"), 1); + assert_eq!(jobs.running_count_for("comp-b"), 0); + assert!(job_quota_exceeded(2, 2)); + assert!(!job_quota_exceeded(1, 2)); + jobs.interrupt_computer("comp-a"); + assert_eq!(jobs.get("a").unwrap().status, JobStatus::Interrupted); + assert_eq!(jobs.running_count(), 0); + } +} diff --git a/crates/control/src/lib.rs b/crates/control/src/lib.rs index be74198..b467aac 100644 --- a/crates/control/src/lib.rs +++ b/crates/control/src/lib.rs @@ -1,27 +1,60 @@ mod a11y; mod actions; mod browser_page; +mod capability; mod controller; mod cua; +mod display_backend; +mod file_cas; +mod form; +mod gmail; +mod jobs; mod lease; +mod mcp_policy; mod observe; +mod operation; +mod outlook; mod overlay; mod path; +mod pause; +mod readiness; mod sandbox; mod screen; +mod secrets; mod takeover; +mod tool_manager; +mod verifier; +mod wait; mod x11; pub use a11y::*; pub use actions::*; pub use browser_page::*; +pub use capability::*; pub use controller::*; -pub use cua::{CuaClient, CuaController, TranslatedAction, translate_action}; +pub use cua::{ + BrowserLocatorKind, CuaClient, CuaController, TranslatedAction, classify_browser_locator, + is_cua_ref, translate_action, +}; +pub use display_backend::*; +pub use file_cas::*; +pub use form::*; +pub use gmail::*; +pub use jobs::*; pub use lease::*; +pub use mcp_policy::*; pub use observe::*; +pub use operation::*; +pub use outlook::*; pub use overlay::*; pub use path::*; +pub use pause::*; +pub use readiness::*; pub use sandbox::*; pub use screen::*; +pub use secrets::*; pub use takeover::*; +pub use tool_manager::*; +pub use verifier::*; +pub use wait::*; pub use x11::*; diff --git a/crates/control/src/mcp_policy.rs b/crates/control/src/mcp_policy.rs new file mode 100644 index 0000000..559cb62 --- /dev/null +++ b/crates/control/src/mcp_policy.rs @@ -0,0 +1,92 @@ +//! Where MCP runtimes execute. Catalog stdio is never a child of the API. + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum McpExecutionLocation { + AssignedComputer, + RemoteHttp, +} + +pub fn mcp_execution_location(transport: &str) -> McpExecutionLocation { + match transport { + "http" | "sse" => McpExecutionLocation::RemoteHttp, + _ => McpExecutionLocation::AssignedComputer, + } +} + +/// The API/Supervisor process must not `spawn` a catalog stdio MCP child. +pub fn api_host_may_spawn_stdio() -> bool { + false +} + +pub fn catalog_stdio_api_error() -> &'static str { + "stdio MCP runs on the assigned Computer, not the API process. Install a reviewed package onto the Computer (Tool Manager). Unpinned npx -y latest is refused." +} + +/// `npx -y pkg` without a version pin is not an install record. +pub fn refuse_unpinned_npx(command: &str, args: &[String]) -> bool { + let name = command.rsplit('/').next().unwrap_or(command); + if name != "npx" && name != "pnpx" { + return false; + } + let yes = args.iter().any(|arg| arg == "-y" || arg == "--yes"); + let pinned = args.iter().any(|arg| package_is_pinned(arg)); + yes && !pinned +} + +/// `pkg@1.2.3` and `@scope/pkg@1.2.3` are pinned; `@scope/pkg` is not — the +/// leading `@` is the scope, not a version separator. +fn package_is_pinned(arg: &str) -> bool { + let name = arg.strip_prefix('@').unwrap_or(arg); + name.contains('@') +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn catalog_stdio_is_not_an_api_child() { + assert!(!api_host_may_spawn_stdio()); + assert_eq!( + mcp_execution_location("stdio"), + McpExecutionLocation::AssignedComputer + ); + assert_eq!( + mcp_execution_location("http"), + McpExecutionLocation::RemoteHttp + ); + assert_eq!( + mcp_execution_location("sse"), + McpExecutionLocation::RemoteHttp + ); + assert!(catalog_stdio_api_error().contains("assigned Computer")); + } + + #[test] + fn unpinned_npx_yes_is_refused() { + assert!(refuse_unpinned_npx( + "npx", + &["-y".into(), "some-mcp".into()] + )); + assert!(!refuse_unpinned_npx( + "npx", + &["-y".into(), "some-mcp@1.2.3".into()] + )); + assert!(!refuse_unpinned_npx("python3", &["echo_server.py".into()])); + // Scoped packages: the leading `@` is not a version pin. + assert!(refuse_unpinned_npx( + "npx", + &[ + "-y".into(), + "@modelcontextprotocol/server-filesystem".into() + ] + )); + assert!(!refuse_unpinned_npx( + "npx", + &[ + "-y".into(), + "@modelcontextprotocol/server-filesystem@0.6.2".into() + ] + )); + } +} diff --git a/crates/control/src/observe.rs b/crates/control/src/observe.rs index 6d20b39..3064b6b 100644 --- a/crates/control/src/observe.rs +++ b/crates/control/src/observe.rs @@ -111,7 +111,11 @@ pub fn format_ui_element_lines(elements: &[UiElement], max: usize) -> String { } else { "" }; - format!("[{}] {kind} \"{title}{ellipsis}\" @ {x},{y}", element.id) + let disabled = if element.disabled { " [disabled]" } else { "" }; + format!( + "[{}] {kind} \"{title}{ellipsis}\"{disabled} @ {x},{y}", + element.id + ) }) .collect(); if let Some(dropped) = elements.len().checked_sub(max) { @@ -200,6 +204,43 @@ pub fn screen_change_between( } } +/// Whether a captured frame should be attached to the *model request*. +/// +/// Delivery is independent of capture: a background observe may update the +/// last captured frame without the model having seen it. Text-only models +/// never receive an image. Vision models get a frame when nothing has been +/// delivered yet, the bytes differ from the last delivered frame, the vision +/// model identity changed, history compression dropped the previous picture, +/// or the caller forced delivery (takeover resume). +pub fn should_deliver_observation_image( + vision: bool, + force_delivery: bool, + current_model_id: Option<&str>, + delivered_frame: Option<&str>, + delivered_signature: Option<&[u8]>, + delivered_model_id: Option<&str>, + observation: &ComputerObservation, +) -> bool { + if !vision { + return false; + } + if force_delivery { + return true; + } + if vision_model_changed(current_model_id, delivered_model_id) { + return true; + } + screen_change_between(delivered_frame, delivered_signature, observation) + != ScreenChange::Identical +} + +fn vision_model_changed(current: Option<&str>, delivered: Option<&str>) -> bool { + match (current, delivered) { + (Some(current), Some(delivered)) => current != delivered, + _ => false, + } +} + #[cfg(test)] mod tests { use super::*; @@ -338,6 +379,117 @@ mod tests { ); } + #[test] + fn should_deliver_observation_image_follows_the_vision_truth_table() { + fn png(paint: impl Fn(u32, u32) -> Rgb) -> Vec { + let img = RgbImage::from_fn(320, 180, paint); + let mut out = Vec::new(); + PngEncoder::new(&mut out) + .write_image(img.as_raw(), 320, 180, ExtendedColorType::Rgb8) + .unwrap(); + out + } + let plain = png(|_, _| Rgb([240, 240, 240])); + let clock = png(|x, y| { + if x < 6 && y < 6 { + Rgb([0, 0, 0]) + } else { + Rgb([240, 240, 240]) + } + }); + let dialog = png(|x, y| { + if x < 160 && y < 90 { + Rgb([20, 20, 20]) + } else { + Rgb([240, 240, 240]) + } + }); + let same = observation_from_png(plain.clone(), 320, 180, None, None); + let tick = observation_from_png(clock, 320, 180, None, None); + let covered = observation_from_png(dialog, 320, 180, None, None); + let signature = frame_signature(&plain); + let frame = same.frame_id.as_str(); + let sig = signature.as_deref(); + + // Text-only models never receive an image, even on a first capture. + assert!(!should_deliver_observation_image( + false, false, None, None, None, None, &same + )); + assert!(!should_deliver_observation_image( + false, + true, + Some("vision-model"), + None, + None, + None, + &covered + )); + + // First visual capture (nothing delivered yet) always attaches. + assert!(should_deliver_observation_image( + true, + false, + Some("m1"), + None, + None, + None, + &same + )); + + // Identical to the last *delivered* frame is skipped. + assert!(!should_deliver_observation_image( + true, + false, + Some("m1"), + Some(frame), + sig, + Some("m1"), + &same + )); + + // Similar and Changed still attach: the model may need to look. + assert!(should_deliver_observation_image( + true, + false, + Some("m1"), + Some(frame), + sig, + Some("m1"), + &tick + )); + assert!(should_deliver_observation_image( + true, + false, + Some("m1"), + Some(frame), + sig, + Some("m1"), + &covered + )); + + // Force delivery (takeover / history compression) even when identical. + assert!(should_deliver_observation_image( + true, + true, + Some("m1"), + Some(frame), + sig, + Some("m1"), + &same + )); + + // Switching vision model identity re-delivers the current frame. + assert!(should_deliver_observation_image( + true, + false, + Some("m2"), + Some(frame), + sig, + Some("m1"), + &same + )); + } + #[test] fn element_lines_cap_the_count_and_long_labels() { let elements: Vec = (1..=5) diff --git a/crates/control/src/operation.rs b/crates/control/src/operation.rs new file mode 100644 index 0000000..93b6d69 --- /dev/null +++ b/crates/control/src/operation.rs @@ -0,0 +1,173 @@ +use std::collections::HashMap; +use std::sync::Mutex; + +use sha2::{Digest, Sha256}; + +/// Durable-enough in-process operation ledger. Same operation_id + payload +/// hash returns the original result. Same id with a different payload is +/// rejected. Journal failure (lock poisoned / quota) refuses new mutations. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OperationRecord { + pub operation_id: String, + pub payload_hash: String, + pub status: String, + pub result: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OperationError { + PayloadMismatch, + JournalUnavailable, +} + +pub struct OperationLedger { + inner: Mutex>, + fail_journal: Mutex, +} + +impl Default for OperationLedger { + fn default() -> Self { + Self { + inner: Mutex::new(HashMap::new()), + fail_journal: Mutex::new(false), + } + } +} + +impl OperationLedger { + pub fn payload_hash(payload: &[u8]) -> String { + hex::encode(Sha256::digest(payload)) + } + + pub fn set_journal_available(&self, available: bool) { + *self.fail_journal.lock().unwrap() = !available; + } + + pub fn accept( + &self, + operation_id: &str, + payload: &[u8], + result: &str, + ) -> Result { + let record = self.begin(operation_id, payload)?; + if !record.result.is_empty() { + return Ok(record); + } + self.complete(operation_id, payload, result, "succeeded") + } + + /// Insert an accepted row. A finished result is returned as-is (replay). + pub fn begin( + &self, + operation_id: &str, + payload: &[u8], + ) -> Result { + if *self.fail_journal.lock().unwrap() { + return Err(OperationError::JournalUnavailable); + } + let hash = Self::payload_hash(payload); + let mut map = self.inner.lock().unwrap(); + if let Some(existing) = map.get(operation_id) { + if existing.payload_hash != hash { + return Err(OperationError::PayloadMismatch); + } + return Ok(existing.clone()); + } + let record = OperationRecord { + operation_id: operation_id.to_string(), + payload_hash: hash, + status: "accepted".into(), + result: String::new(), + }; + map.insert(operation_id.to_string(), record.clone()); + Ok(record) + } + + /// Store the result of an accepted operation. A completed row is not overwritten. + pub fn complete( + &self, + operation_id: &str, + payload: &[u8], + result: &str, + status: &str, + ) -> Result { + if *self.fail_journal.lock().unwrap() { + return Err(OperationError::JournalUnavailable); + } + let hash = Self::payload_hash(payload); + let mut map = self.inner.lock().unwrap(); + if let Some(existing) = map.get_mut(operation_id) { + if existing.payload_hash != hash { + return Err(OperationError::PayloadMismatch); + } + if !existing.result.is_empty() { + return Ok(existing.clone()); + } + existing.result = result.to_string(); + existing.status = status.to_string(); + return Ok(existing.clone()); + } + let record = OperationRecord { + operation_id: operation_id.to_string(), + payload_hash: hash, + status: status.into(), + result: result.to_string(), + }; + map.insert(operation_id.to_string(), record.clone()); + Ok(record) + } + + pub fn get(&self, operation_id: &str) -> Option { + self.inner.lock().unwrap().get(operation_id).cloned() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn same_operation_is_not_rerun() { + let ledger = OperationLedger::default(); + let first = ledger.accept("op-1", b"{\"a\":1}", "ok").unwrap(); + let second = ledger.accept("op-1", b"{\"a\":1}", "different").unwrap(); + assert_eq!(first.result, "ok"); + assert_eq!(second.result, "ok"); + } + + #[test] + fn same_id_different_payload_is_rejected() { + let ledger = OperationLedger::default(); + ledger.accept("op-1", b"a", "ok").unwrap(); + assert_eq!( + ledger.accept("op-1", b"b", "ok"), + Err(OperationError::PayloadMismatch) + ); + } + + #[test] + fn journal_failure_refuses_new_mutations() { + let ledger = OperationLedger::default(); + ledger.set_journal_available(false); + assert_eq!( + ledger.accept("op-2", b"x", "ok"), + Err(OperationError::JournalUnavailable) + ); + } + + #[test] + fn begin_then_complete_does_not_keep_an_empty_result() { + let ledger = OperationLedger::default(); + let started = ledger.begin("op-3", b"payload").unwrap(); + assert_eq!(started.status, "accepted"); + assert!(started.result.is_empty()); + let done = ledger + .complete("op-3", b"payload", "wrote", "succeeded") + .unwrap(); + assert_eq!(done.result, "wrote"); + let replay = ledger + .complete("op-3", b"payload", "other", "succeeded") + .unwrap(); + assert_eq!(replay.result, "wrote"); + } +} diff --git a/crates/control/src/outlook.rs b/crates/control/src/outlook.rs new file mode 100644 index 0000000..07984a3 --- /dev/null +++ b/crates/control/src/outlook.rs @@ -0,0 +1,94 @@ +pub const GRAPH_BATCH_MAX: usize = 20; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GraphSubResult { + pub id: String, + pub status: u16, + pub retry_after: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GraphItemOutcome { + Ok, + Retry { after_ms: u64 }, + NeedsAuth, + Failed(u16), +} + +/// HTTP 200 on the batch envelope is not success. Each sub-request is judged +/// on its own status. 429 honours Retry-After; 401 stops mutation. +pub fn classify_graph_item(item: &GraphSubResult) -> GraphItemOutcome { + match item.status { + 200..=299 => GraphItemOutcome::Ok, + 401 | 403 => GraphItemOutcome::NeedsAuth, + 429 => GraphItemOutcome::Retry { + after_ms: item.retry_after.unwrap_or(1) * 1000, + }, + other => GraphItemOutcome::Failed(other), + } +} + +pub fn chunk_graph_batch(items: &[T]) -> Vec<&[T]> { + items.chunks(GRAPH_BATCH_MAX).collect() +} + +/// Merge categories without wiping ones this plan did not mention. +pub fn merge_categories(current: &[String], add: &[String], remove: &[String]) -> Vec { + let mut next: Vec = current + .iter() + .filter(|item| !remove.contains(item)) + .cloned() + .collect(); + for item in add { + if !next.contains(item) { + next.push(item.clone()); + } + } + next +} + +pub fn mail_readwrite_allows_send() -> bool { + false +} + +pub fn mailbox_settings_required_for_master_categories() -> bool { + true +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn batch_http_200_still_classifies_child_429() { + let item = GraphSubResult { + id: "1".into(), + status: 429, + retry_after: Some(2), + }; + assert_eq!( + classify_graph_item(&item), + GraphItemOutcome::Retry { after_ms: 2000 } + ); + } + + #[test] + fn graph_batches_are_at_most_twenty() { + let items: Vec = (0..41).collect(); + let chunks = chunk_graph_batch(&items); + assert_eq!(chunks.len(), 3); + assert!(chunks.iter().all(|chunk| chunk.len() <= 20)); + } + + #[test] + fn categories_merge_does_not_clobber_unrelated_labels() { + let merged = merge_categories( + &["keep".into(), "old".into()], + &["new".into()], + &["old".into()], + ); + assert_eq!(merged, vec!["keep".to_string(), "new".to_string()]); + assert!(!mail_readwrite_allows_send()); + assert!(mailbox_settings_required_for_master_categories()); + } +} diff --git a/crates/control/src/pause.rs b/crates/control/src/pause.rs new file mode 100644 index 0000000..5f403d1 --- /dev/null +++ b/crates/control/src/pause.rs @@ -0,0 +1,66 @@ +use lazyboy_contracts::PauseScope; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct ControlEpochs { + pub computer: u64, + pub agent: u64, + pub display: u64, +} + +impl ControlEpochs { + pub fn bump(&mut self, scope: PauseScope) { + match scope { + PauseScope::Computer => { + self.computer = self.computer.saturating_add(1); + self.agent = self.agent.saturating_add(1); + self.display = self.display.saturating_add(1); + } + PauseScope::Agent => { + self.agent = self.agent.saturating_add(1); + } + PauseScope::Display => { + self.display = self.display.saturating_add(1); + } + } + } + + /// A token from before the bump is stale for that scope. + pub fn allows(&self, scope: PauseScope, token: ControlEpochs) -> bool { + match scope { + PauseScope::Computer => token.computer == self.computer, + PauseScope::Agent => token.computer == self.computer && token.agent == self.agent, + PauseScope::Display => token.computer == self.computer && token.display == self.display, + } + } +} + +/// Pausing agent A must not freeze unrelated work of agent B on the same Team +/// computer. Computer-wide pause is explicit. +pub fn freeze_peer_on_agent_pause() -> bool { + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pausing_agent_a_does_not_invalidate_b_display() { + let mut epochs = ControlEpochs::default(); + let b_before = epochs; + epochs.bump(PauseScope::Agent); + assert!(epochs.allows(PauseScope::Display, b_before)); + assert!(!epochs.allows(PauseScope::Agent, b_before)); + assert!(!freeze_peer_on_agent_pause()); + } + + #[test] + fn computer_pause_invalidates_every_scope() { + let mut epochs = ControlEpochs::default(); + let before = epochs; + epochs.bump(PauseScope::Computer); + assert!(!epochs.allows(PauseScope::Agent, before)); + assert!(!epochs.allows(PauseScope::Display, before)); + assert!(!epochs.allows(PauseScope::Computer, before)); + } +} diff --git a/crates/control/src/readiness.rs b/crates/control/src/readiness.rs new file mode 100644 index 0000000..f1cf682 --- /dev/null +++ b/crates/control/src/readiness.rs @@ -0,0 +1,88 @@ +//! Layered readiness: runner / browser / desktop / viewer are independent. + +use crate::{ + AdapterContext, ComputerRef, EnsureScreenRequest, EnsureScreenResult, SandboxError, + SandboxProvider, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct ComponentReadiness { + pub runner: bool, + pub browser: bool, + pub desktop: bool, + pub viewer: bool, +} + +/// Native exec, files, and Computer-local MCP need the Runner, not XFCE or noVNC. +pub fn native_work_allowed(ready: ComponentReadiness) -> bool { + ready.runner +} + +pub fn viewer_blocks_native() -> bool { + false +} + +pub fn desktop_required_for_native() -> bool { + false +} + +/// Production gate: native tools must not attach a display/viewer. +pub fn attach_display_for_tool(need_gui: bool) -> bool { + need_gui +} + +/// Call `ensure_screen` only when the tool actually needs a display. +pub async fn maybe_ensure_screen( + sandbox: &dyn SandboxProvider, + computer: &ComputerRef, + need_gui: bool, + request: EnsureScreenRequest, + context: &AdapterContext, +) -> Result, SandboxError> { + if !attach_display_for_tool(need_gui) { + return Ok(None); + } + Ok(Some( + sandbox.ensure_screen(computer, request, context).await?, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn native_exec_does_not_need_desktop_or_viewer() { + let ready = ComponentReadiness { + runner: true, + browser: false, + desktop: false, + viewer: false, + }; + assert!(native_work_allowed(ready)); + assert!(!viewer_blocks_native()); + assert!(!desktop_required_for_native()); + assert!(!attach_display_for_tool(false)); + assert!(attach_display_for_tool(true)); + } + + #[test] + fn start_sh_skips_desktop_when_runner_only() { + let script = include_str!("../../../image/computer/start.sh"); + assert!(script.contains("LAZYBOY_RUNNER_ONLY")); + assert!(script.contains("exec sleep infinity")); + assert!( + script.contains("lazyboy-screen boot-primary"), + "GUI boot still starts the desktop" + ); + } + + #[test] + fn viewer_not_ready_does_not_block_native() { + assert!(native_work_allowed(ComponentReadiness { + runner: true, + ..ComponentReadiness::default() + })); + assert!(!native_work_allowed(ComponentReadiness::default())); + } +} diff --git a/crates/control/src/sandbox.rs b/crates/control/src/sandbox.rs index fc065f9..3379b8d 100644 --- a/crates/control/src/sandbox.rs +++ b/crates/control/src/sandbox.rs @@ -18,6 +18,10 @@ pub struct AdapterContext { pub display: Option, #[serde(default)] pub profile_path: Option, + #[serde(default)] + pub computer_generation: Option, + #[serde(default)] + pub control_epoch: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -29,11 +33,15 @@ pub struct ComputerRef { pub fresh: bool, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] pub struct ProvisionRequest { pub home_key: String, pub home_path: String, pub provider_ref: Option, + /// When true the container stays up for exec/files without XFCE/noVNC. + /// GUI later starts via `ensure_screen` / `lazyboy-screen ensure`. + #[serde(default)] + pub runner_only: bool, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] @@ -45,13 +53,30 @@ pub struct CommandRequest { /// Used so fill-login never puts a password on the argv of `ps`. #[serde(default, skip_serializing_if = "Option::is_none")] pub stdin: Option, + /// When true the Computer keeps the process as a job and returns job_id. + #[serde(default)] + pub background: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operation_id: Option, + /// Existing Computer-side job for `job_op` status/cancel, or the id to assign on start. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub job_id: Option, + /// `"status"` or `"cancel"` for an existing Computer-side job. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub job_op: Option, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] pub struct CommandResult { pub stdout: String, pub stderr: String, pub code: i32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub job_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub signal: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub status: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/crates/control/src/secrets.rs b/crates/control/src/secrets.rs new file mode 100644 index 0000000..e9c73af --- /dev/null +++ b/crates/control/src/secrets.rs @@ -0,0 +1,129 @@ +const SECRET_KEYS: &[&str] = &[ + "password", + "passwd", + "secret", + "token", + "api_key", + "apikey", + "authorization", + "access_token", + "refresh_token", + "private_key", +]; + +/// Redact secret-shaped values before they reach traces, checkpoints, or UI. +pub fn redact_text(text: &str, canaries: &[&str]) -> String { + let mut out = text.to_string(); + for canary in canaries { + if !canary.is_empty() { + out = out.replace(canary, "[redacted]"); + } + } + out +} + +/// Pattern-based redaction for free text that is persisted (ledger results, +/// activity snippets). Catches the common shapes — `Authorization: Bearer …`, +/// `password=…`/`token: …` assignments, and well-known API key prefixes — +/// without needing to know the secret values in advance. Not a guarantee. +pub fn redact_secret_patterns(text: &str) -> String { + static PATTERNS: std::sync::OnceLock> = std::sync::OnceLock::new(); + let patterns = PATTERNS.get_or_init(|| { + [ + r"(?i)\bbearer\s+[A-Za-z0-9\-._~+/]+=*", + r"(?i)\b(?:password|passwd|pwd|secret|token|api[_-]?key|apikey|access[_-]?token|refresh[_-]?token|client[_-]?secret|private[_-]?key|authorization)\b\s*[:=]\s*\S+", + r#"(?i)"(?:password|passwd|secret|token|api_key|apikey|access_token|refresh_token|client_secret|private_key|authorization)"\s*:\s*"[^"]*""#, + r"\b(?:sk|rk|pk)-[A-Za-z0-9\-_]{16,}\b", + r"\bgh[pousr]_[A-Za-z0-9]{20,}\b", + r"\bAKIA[0-9A-Z]{16}\b", + r"\bxox[abprs]-[A-Za-z0-9\-]{10,}\b", + r"\bAIza[0-9A-Za-z\-_]{30,}\b", + r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----", + ] + .iter() + .map(|pattern| regex::Regex::new(pattern).expect("static regex")) + .collect() + }); + let mut out = text.to_string(); + for pattern in patterns { + out = pattern.replace_all(&out, "[redacted]").into_owned(); + } + out +} + +pub fn looks_like_secret_key(key: &str) -> bool { + let lower = key.to_ascii_lowercase(); + SECRET_KEYS + .iter() + .any(|secret| lower == *secret || lower.ends_with(&format!("_{secret}"))) +} + +pub fn redact_json(value: &mut serde_json::Value) { + match value { + serde_json::Value::Object(map) => { + let keys: Vec = map.keys().cloned().collect(); + for key in keys { + if looks_like_secret_key(&key) { + map.insert(key, serde_json::Value::String("[redacted]".into())); + } else if let Some(child) = map.get_mut(&key) { + redact_json(child); + } + } + } + serde_json::Value::Array(items) => { + for item in items { + redact_json(item); + } + } + _ => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn canary_is_stripped_from_success_and_error_text() { + let secret = "super-secret-token-value"; + assert!(!redact_text(&format!("ok {secret}"), &[secret]).contains(secret)); + assert!(!redact_text(&format!("error: {secret}"), &[secret]).contains(secret)); + assert!(redact_text(&format!("error: {secret}"), &[secret]).contains("[redacted]")); + } + + #[test] + fn secret_shaped_text_is_redacted_without_knowing_the_value() { + let text = concat!( + "curl -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.abc.def' ", + "https://x/ password=hunter2 token: abc123 ", + "OPENAI=sk-abcdefghijklmnopqrstuvwxyz0123456789 ", + "ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZ0123 AKIAABCDEFGHIJKLMNOP ", + "{\"access_token\":\"zzz\"} plain words stay" + ); + let out = redact_secret_patterns(text); + for secret in [ + "eyJhbGciOiJIUzI1NiJ9", + "hunter2", + "abc123", + "sk-abcdefghijklmnopqrstuvwxyz0123456789", + "ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZ0123", + "AKIAABCDEFGHIJKLMNOP", + "zzz", + ] { + assert!(!out.contains(secret), "{secret} leaked: {out}"); + } + assert!(out.contains("plain words stay")); + assert!(out.contains("https://x/")); + assert_eq!(redact_secret_patterns("exit 0\nls -la"), "exit 0\nls -la"); + } + + #[test] + fn json_secret_keys_are_redacted() { + let mut value = json!({"password":"hunter2","nested":{"access_token":"abc"},"ok":true}); + redact_json(&mut value); + assert_eq!(value["password"], "[redacted]"); + assert_eq!(value["nested"]["access_token"], "[redacted]"); + assert_eq!(value["ok"], true); + } +} diff --git a/crates/control/src/tool_manager.rs b/crates/control/src/tool_manager.rs new file mode 100644 index 0000000..b1146f1 --- /dev/null +++ b/crates/control/src/tool_manager.rs @@ -0,0 +1,389 @@ +use sha2::{Digest, Sha256}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InstallState { + PendingApproval, + Installing, + Installed, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InstanceState { + Starting, + Connected, + AuthRequired, + Authenticated, + Ready, + Degraded, + Disabled, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ToolManifest { + pub id: String, + pub version: String, + pub sha256: String, + pub entrypoint: Vec, + pub share_immutable_package: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ManifestError { + MissingDigest, + PathEscape, + EmptyEntrypoint, +} + +pub fn validate_manifest(manifest: &ToolManifest) -> Result<(), ManifestError> { + // A digest is what gets verified before every run, so a placeholder such as + // "compute-at-install" must be rejected here, not trusted later. + let digest = manifest.sha256.trim(); + if digest.len() != 64 || !digest.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(ManifestError::MissingDigest); + } + if manifest.entrypoint.is_empty() { + return Err(ManifestError::EmptyEntrypoint); + } + for part in &manifest.entrypoint { + if part.contains("..") { + return Err(ManifestError::PathEscape); + } + } + Ok(()) +} + +pub fn artifact_digest(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} + +pub fn install_dir(root: &Path, id: &str, version: &str) -> PathBuf { + root.join("tools").join(id).join(version) +} + +/// Reject archive members that escape the destination (zip-slip). +pub fn safe_member_path(destination: &Path, member: &str) -> Result { + let cleaned = member.replace('\\', "/"); + if cleaned.split('/').any(|part| part == ".." || part == ".") { + return Err(ManifestError::PathEscape); + } + let path = destination.join(cleaned); + if !path.starts_with(destination) { + return Err(ManifestError::PathEscape); + } + Ok(path) +} + +pub fn installed_is_not_ready(install: InstallState, instance: InstanceState) -> bool { + !matches!( + (install, instance), + (InstallState::Installed, InstanceState::Ready) + ) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VersionPlan { + pub job_pin: String, + pub new_runs: String, +} + +/// Active jobs keep the old bytes; new runs may use a verified new version. +/// Never requires restarting the whole Computer. +pub fn plan_version_switch(current: &str, incoming: &str, active_jobs: usize) -> VersionPlan { + if current == incoming { + return VersionPlan { + job_pin: current.into(), + new_runs: current.into(), + }; + } + if active_jobs > 0 { + VersionPlan { + job_pin: current.into(), + new_runs: incoming.into(), + } + } else { + VersionPlan { + job_pin: incoming.into(), + new_runs: incoming.into(), + } + } +} + +pub fn previous_version<'a>(installed: &'a [&str], current: &str) -> Option<&'a str> { + let pos = installed.iter().position(|version| *version == current)?; + if pos == 0 { + None + } else { + Some(installed[pos - 1]) + } +} + +pub fn package_gc_allowed(remaining_ready_bindings: usize, active_job_pins: usize) -> bool { + remaining_ready_bindings == 0 && active_job_pins == 0 +} + +/// Revoke drops Computer-local MCP from the next run schema immediately. +/// At most one `ready` binding per (bot, package_id). Version changes switch +/// that row instead of inserting a second ready binding (UNIQUE is +/// package_row_id+bot_id, so a second insert would survive and break rollback). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReadyBinding { + pub id: String, + pub package_row_id: String, + pub version: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReadyBindingChange { + Insert { + id: String, + package_row_id: String, + version: String, + }, + Switch { + id: String, + package_row_id: String, + version: String, + }, + Keep { + id: String, + }, +} + +pub fn upsert_one_ready_binding( + existing: Option<&ReadyBinding>, + incoming_row_id: &str, + incoming_version: &str, + new_id: &str, +) -> ReadyBindingChange { + match existing { + None => ReadyBindingChange::Insert { + id: new_id.into(), + package_row_id: incoming_row_id.into(), + version: incoming_version.into(), + }, + Some(current) if current.package_row_id == incoming_row_id => ReadyBindingChange::Keep { + id: current.id.clone(), + }, + Some(current) => ReadyBindingChange::Switch { + id: current.id.clone(), + package_row_id: incoming_row_id.into(), + version: incoming_version.into(), + }, + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RollbackChange { + Switch { + id: String, + package_row_id: String, + version: String, + }, + NoPrevious, + /// A leftover ready row already occupies the previous package_row_id. + Conflict, +} + +pub fn rollback_one_ready_binding( + current: &ReadyBinding, + leftover_on_previous: bool, + previous_row_id: Option<&str>, + previous_version: Option<&str>, +) -> RollbackChange { + let (Some(row_id), Some(version)) = (previous_row_id, previous_version) else { + return RollbackChange::NoPrevious; + }; + if leftover_on_previous { + return RollbackChange::Conflict; + } + RollbackChange::Switch { + id: current.id.clone(), + package_row_id: row_id.into(), + version: version.into(), + } +} + +pub fn pin_running_jobs( + current_version: &str, + incoming_version: &str, + active_jobs: usize, +) -> Option { + let plan = plan_version_switch(current_version, incoming_version, active_jobs); + if plan.job_pin != plan.new_runs { + Some(plan.job_pin) + } else { + None + } +} + +pub fn package_version_for_call<'a>( + ready_version: &'a str, + job_pin: Option<&'a str>, + existing_job: bool, +) -> &'a str { + if existing_job { + job_pin.unwrap_or(ready_version) + } else { + ready_version + } +} + +pub fn filter_run_tool_names<'a>( + names: impl IntoIterator, + computer_mcp_bound: bool, + granted_mcp: &[String], +) -> Vec { + names + .into_iter() + .filter(|name| { + if *name == "computer_mcp" { + computer_mcp_bound + } else if name.starts_with("mcp_") { + granted_mcp.iter().any(|granted| granted == name) + } else { + true + } + }) + .map(str::to_string) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> ToolManifest { + ToolManifest { + id: "lazyboy.example.echo".into(), + version: "0.0.1".into(), + sha256: artifact_digest(b"echo"), + entrypoint: vec!["./echo-adapter".into()], + share_immutable_package: true, + } + } + + #[test] + fn empty_digest_is_rejected() { + let mut manifest = sample(); + manifest.sha256.clear(); + assert_eq!( + validate_manifest(&manifest), + Err(ManifestError::MissingDigest) + ); + manifest.sha256 = "compute-at-install".into(); + assert_eq!( + validate_manifest(&manifest), + Err(ManifestError::MissingDigest) + ); + assert_eq!(validate_manifest(&sample()), Ok(())); + } + + #[test] + fn zip_slip_is_rejected() { + let dest = Path::new("/tmp/pkg"); + assert_eq!( + safe_member_path(dest, "../etc/passwd"), + Err(ManifestError::PathEscape) + ); + assert!(safe_member_path(dest, "bin/echo").is_ok()); + } + + #[test] + fn installed_is_not_the_same_as_authorized() { + assert!(installed_is_not_ready( + InstallState::Installed, + InstanceState::AuthRequired + )); + assert!(!installed_is_not_ready( + InstallState::Installed, + InstanceState::Ready + )); + } + + #[test] + fn update_pins_old_version_while_jobs_run() { + let plan = plan_version_switch("0.0.1", "0.0.2", 1); + assert_eq!(plan.job_pin, "0.0.1"); + assert_eq!(plan.new_runs, "0.0.2"); + let idle = plan_version_switch("0.0.1", "0.0.2", 0); + assert_eq!(idle.job_pin, "0.0.2"); + assert_eq!(idle.new_runs, "0.0.2"); + } + + #[test] + fn rollback_selects_the_previous_side_by_side_version() { + let installed = ["0.0.1", "0.0.2"]; + assert_eq!(previous_version(&installed, "0.0.2"), Some("0.0.1")); + assert_eq!(previous_version(&installed, "0.0.1"), None); + assert!(package_gc_allowed(0, 0)); + assert!(!package_gc_allowed(1, 0)); + } + + #[test] + fn update_switches_the_single_ready_binding_then_rollback() { + let first = upsert_one_ready_binding(None, "row-1", "0.0.1", "bind-a"); + assert_eq!( + first, + ReadyBindingChange::Insert { + id: "bind-a".into(), + package_row_id: "row-1".into(), + version: "0.0.1".into(), + } + ); + let ready = ReadyBinding { + id: "bind-a".into(), + package_row_id: "row-1".into(), + version: "0.0.1".into(), + }; + let second = upsert_one_ready_binding(Some(&ready), "row-2", "0.0.2", "bind-b"); + assert_eq!( + second, + ReadyBindingChange::Switch { + id: "bind-a".into(), + package_row_id: "row-2".into(), + version: "0.0.2".into(), + } + ); + let on_new = ReadyBinding { + id: "bind-a".into(), + package_row_id: "row-2".into(), + version: "0.0.2".into(), + }; + assert_eq!( + rollback_one_ready_binding(&on_new, false, Some("row-1"), Some("0.0.1")), + RollbackChange::Switch { + id: "bind-a".into(), + package_row_id: "row-1".into(), + version: "0.0.1".into(), + } + ); + assert_eq!( + rollback_one_ready_binding(&on_new, true, Some("row-1"), Some("0.0.1")), + RollbackChange::Conflict + ); + assert_eq!(pin_running_jobs("0.0.1", "0.0.2", 2), Some("0.0.1".into())); + assert_eq!(pin_running_jobs("0.0.1", "0.0.2", 0), None); + assert_eq!( + package_version_for_call("0.0.2", Some("0.0.1"), true), + "0.0.1" + ); + assert_eq!( + package_version_for_call("0.0.2", Some("0.0.1"), false), + "0.0.2" + ); + } + + #[test] + fn revoke_drops_computer_mcp_from_the_next_schema() { + let before = filter_run_tool_names( + ["exec", "computer_mcp", "mcp_x_echo"], + true, + &["mcp_x_echo".into()], + ); + assert!(before.contains(&"computer_mcp".into())); + assert!(before.contains(&"mcp_x_echo".into())); + let after = filter_run_tool_names(["exec", "computer_mcp", "mcp_x_echo"], false, &[]); + assert_eq!(after, vec!["exec".to_string()]); + } +} diff --git a/crates/control/src/verifier.rs b/crates/control/src/verifier.rs new file mode 100644 index 0000000..27e5570 --- /dev/null +++ b/crates/control/src/verifier.rs @@ -0,0 +1,62 @@ +use lazyboy_contracts::CompletionLayer; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Verification { + pub layer: CompletionLayer, + pub ok: bool, + pub evidence: String, +} + +pub fn transport_ok(http_ok: bool) -> Verification { + Verification { + layer: CompletionLayer::Transport, + ok: http_ok, + evidence: if http_ok { + "response received".into() + } else { + "no valid response".into() + }, + } +} + +pub fn operation_effect(confirmed: bool, evidence: impl Into) -> Verification { + Verification { + layer: CompletionLayer::OperationEffect, + ok: confirmed, + evidence: evidence.into(), + } +} + +pub fn task_success(all_required: bool, evidence: impl Into) -> Verification { + Verification { + layer: CompletionLayer::TaskSuccess, + ok: all_required, + evidence: evidence.into(), + } +} + +/// Wait / poll heartbeats keep the worker alive but do not count as task progress. +pub fn wait_counts_as_progress() -> bool { + false +} + +pub fn clock_or_cursor_is_progress() -> bool { + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn exit_zero_is_not_task_success() { + let transport = transport_ok(true); + let effect = operation_effect(true, "exit 0"); + let task = task_success(false, "output hash missing"); + assert!(transport.ok); + assert!(effect.ok); + assert!(!task.ok); + assert!(!wait_counts_as_progress()); + assert!(!clock_or_cursor_is_progress()); + } +} diff --git a/crates/control/src/wait.rs b/crates/control/src/wait.rs new file mode 100644 index 0000000..87d6a7e --- /dev/null +++ b/crates/control/src/wait.rs @@ -0,0 +1,42 @@ +use std::time::{Duration, Instant}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WaitOutcome { + Satisfied, + Timeout, + DisabledUnknown, +} + +/// Wait for a predicate with a deadline. Unknown disabled reasons do not spin +/// forever — they surface as DisabledUnknown so the caller can re-observe. +pub fn wait_until(deadline: Duration, mut ready: F) -> WaitOutcome +where + F: FnMut() -> Result, +{ + let start = Instant::now(); + loop { + match ready() { + Ok(true) => return WaitOutcome::Satisfied, + Ok(false) if start.elapsed() >= deadline => return WaitOutcome::Timeout, + Ok(false) => std::thread::sleep(Duration::from_millis(5)), + Err(_) => return WaitOutcome::DisabledUnknown, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deadline_stops_an_unready_condition() { + let outcome = wait_until(Duration::from_millis(20), || Ok(false)); + assert_eq!(outcome, WaitOutcome::Timeout); + } + + #[test] + fn unknown_disabled_does_not_spin() { + let outcome = wait_until(Duration::from_secs(5), || Err("unknown")); + assert_eq!(outcome, WaitOutcome::DisabledUnknown); + } +} diff --git a/crates/control/src/x11.rs b/crates/control/src/x11.rs index e8293f0..96b498c 100644 --- a/crates/control/src/x11.rs +++ b/crates/control/src/x11.rs @@ -68,6 +68,10 @@ pub fn parse_ui_elements(raw: &str) -> Vec { .and_then(serde_json::Value::as_str) .filter(|value| !value.is_empty()) .map(str::to_string), + disabled: item + .get("disabled") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false), }) }) .collect() diff --git a/crates/harness/src/policy.rs b/crates/harness/src/policy.rs index 2427d05..a765dbb 100644 --- a/crates/harness/src/policy.rs +++ b/crates/harness/src/policy.rs @@ -56,9 +56,15 @@ pub const STALE_HALT: u32 = 150; /// Coaching has to stay rare enough that the model actually reads it. pub const MAX_REFLECTIONS: u32 = 8; -/// A run that repeats itself is not a run that waits or polls, so these tools -/// reset the no-progress clock even when their arguments repeat. -const PROGRESS_WHEN_REPEATED: [&str; 2] = ["shell", "wait"]; +/// Running a command again can legitimately produce something new (a build +/// step, a poll that returns different output), so these tools reset the +/// no-progress clock even when their arguments repeat. +const PROGRESS_WHEN_REPEATED: [&str; 2] = ["shell", "exec"]; + +/// Waiting proves the worker is alive, not that the task moved. It keeps a +/// polling run from being halted as a loop, but the stale coaching still fires +/// so the model says what it is waiting for. +const HEARTBEAT_TOOLS: [&str; 1] = ["wait"]; fn parse_u64(value: Option) -> Option { value?.trim().parse::().ok().filter(|value| *value > 0) @@ -218,6 +224,7 @@ pub struct LoopGuard { /// are handled by the nudge limit, not by this guard. acted: bool, last_progress_turn: u32, + last_heartbeat_turn: u32, last_stale_reflect_turn: u32, checkpoint_turn: u32, soft_wall_spoken: bool, @@ -244,6 +251,7 @@ impl LoopGuard { watched_since: watched_from, acted: false, last_progress_turn: watched_from, + last_heartbeat_turn: watched_from, last_stale_reflect_turn: watched_from, checkpoint_turn: 0, soft_wall_spoken: false, @@ -347,7 +355,9 @@ impl LoopGuard { self.failures.remove(&key); self.warned_failure = false; self.acted = true; - if seen == 1 || PROGRESS_WHEN_REPEATED.contains(&action.name) { + if HEARTBEAT_TOOLS.contains(&action.name) { + self.last_heartbeat_turn = action.turn; + } else if seen == 1 || PROGRESS_WHEN_REPEATED.contains(&action.name) { self.last_progress_turn = action.turn; } @@ -389,7 +399,10 @@ impl LoopGuard { if since < STALE_REFLECT { return None; } - if since >= STALE_HALT { + // A run that keeps waiting is alive, not spinning: coach it, never halt it. + let since_alive = + turns.saturating_sub(self.last_progress_turn.max(self.last_heartbeat_turn)); + if since_alive >= STALE_HALT { return Some(Verdict::Halt { reason: StopReason::LoopDetected, note: format!( @@ -624,6 +637,7 @@ mod tests { fn polling_a_build_or_a_queue_is_not_treated_as_spinning() { let mut guard = LoopGuard::new(RunPolicy::default()); let args = json!({"ms": 30000}); + let mut stale_coaching = 0; for turn in 1..=160 { assert_eq!( guard.on_action(&act("wait", &args, turn, true, false)), @@ -632,11 +646,22 @@ mod tests { ); // A checkpoint question is fine; what a polling run must never get // is a halt. + let verdict = guard.on_turn(turn, Duration::from_secs(30)); assert!( - !guard.on_turn(turn, Duration::from_secs(30)).is_halt(), + !verdict.is_halt(), "a polling run was halted at turn {turn}" ); + if matches!(&verdict, Verdict::Reflect(text) if text.contains("nothing new has succeeded")) + { + stale_coaching += 1; + } } + // Waiting is a heartbeat, not progress: the run is kept alive but is + // still asked to say what it is waiting for. + assert!( + stale_coaching >= 1, + "waiting must not count as progress; the stale coaching never fired" + ); } #[test] diff --git a/crates/sandbox/Cargo.toml b/crates/sandbox/Cargo.toml index f7496c9..daf0514 100644 --- a/crates/sandbox/Cargo.toml +++ b/crates/sandbox/Cargo.toml @@ -14,5 +14,8 @@ reqwest.workspace = true serde_json.workspace = true base64.workspace = true +[dev-dependencies] +tokio.workspace = true + [lints] workspace = true diff --git a/crates/sandbox/src/docker.rs b/crates/sandbox/src/docker.rs index c0a3f69..40db571 100644 --- a/crates/sandbox/src/docker.rs +++ b/crates/sandbox/src/docker.rs @@ -101,6 +101,7 @@ impl SandboxProvider for DockerSandbox { "homeKey": request.home_key, "homePath": request.home_path, "spaceId": context.space_id, + "runnerOnly": request.runner_only, })) .send() .await @@ -397,23 +398,18 @@ impl SandboxProvider for DockerSandbox { .send() .await .map_err(|error| SandboxError::message(error.to_string()))?; + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(SandboxError::message(format!( + "list files failed: {status} {body}" + ))); + } let body: Value = response .json() .await .map_err(|error| SandboxError::message(error.to_string()))?; - let Some(items) = body.as_array() else { - return Ok(Vec::new()); - }; - Ok(items - .iter() - .filter_map(|item| { - Some(FileEntry { - path: item.get("path")?.as_str()?.to_string(), - kind: item.get("kind")?.as_str()?.to_string(), - size: item.get("size")?.as_u64().unwrap_or(0), - }) - }) - .collect()) + file_entries_from_json(&body) } async fn read_file( @@ -430,16 +426,18 @@ impl SandboxProvider for DockerSandbox { .send() .await .map_err(|error| SandboxError::message(error.to_string()))?; + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(SandboxError::message(format!( + "read failed: {status} {body}" + ))); + } let body: Value = response .json() .await .map_err(|error| SandboxError::message(error.to_string()))?; - Ok(body - .get("content") - .and_then(Value::as_str) - .unwrap_or_default() - .as_bytes() - .to_vec()) + file_bytes_from_json(&body) } async fn write_file( @@ -455,7 +453,7 @@ impl SandboxProvider for DockerSandbox { .headers(self.headers(context)) .json(&serde_json::json!({ "path": path, - "content": String::from_utf8_lossy(content), + "contentBase64": base64::engine::general_purpose::STANDARD.encode(content), })) .send() .await @@ -576,9 +574,42 @@ fn decode_observation(body: &Value) -> Result Ok(observation) } +fn file_entries_from_json(body: &Value) -> Result, SandboxError> { + let Some(items) = body.as_array() else { + return Err(SandboxError::message( + "list files failed: response was not a JSON array", + )); + }; + Ok(items + .iter() + .filter_map(|item| { + Some(FileEntry { + path: item.get("path")?.as_str()?.to_string(), + kind: item.get("kind")?.as_str()?.to_string(), + size: item.get("size")?.as_u64().unwrap_or(0), + }) + }) + .collect()) +} + +fn file_bytes_from_json(body: &Value) -> Result, SandboxError> { + if let Some(encoded) = body.get("contentBase64").and_then(Value::as_str) { + return base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|error| { + SandboxError::message(format!("read failed: invalid base64: {error}")) + }); + } + if let Some(text) = body.get("content").and_then(Value::as_str) { + return Ok(text.as_bytes().to_vec()); + } + Err(SandboxError::message("read failed: missing file content")) +} + #[cfg(test)] mod tests { use super::decode_observation; + use base64::Engine; use lazyboy_contracts::{DEFAULT_SCREEN_HEIGHT, DEFAULT_SCREEN_WIDTH}; use serde_json::json; @@ -601,4 +632,28 @@ mod tests { assert_eq!(observation.width, DEFAULT_SCREEN_WIDTH); assert_eq!(observation.height, DEFAULT_SCREEN_HEIGHT); } + + #[test] + fn file_bytes_prefer_base64_and_refuse_empty_success() { + use super::file_bytes_from_json; + let raw = [0xff, 0xfe, 0x00]; + let encoded = base64::engine::general_purpose::STANDARD.encode(raw); + let body = json!({ "contentBase64": encoded, "content": "not-the-bytes" }); + assert_eq!(file_bytes_from_json(&body).unwrap(), raw); + assert_eq!( + file_bytes_from_json(&json!({ "content": "hello" })).unwrap(), + b"hello" + ); + assert!(file_bytes_from_json(&json!({})).is_err()); + } + + #[test] + fn file_list_refuses_a_non_array() { + use super::file_entries_from_json; + assert!(file_entries_from_json(&json!({"error":"nope"})).is_err()); + let entries = + file_entries_from_json(&json!([{"path":"a.txt","kind":"file","size":4}])).unwrap(); + assert_eq!(entries[0].path, "a.txt"); + assert_eq!(entries[0].size, 4); + } } diff --git a/crates/sandbox/src/fake.rs b/crates/sandbox/src/fake.rs index 862040b..9880931 100644 --- a/crates/sandbox/src/fake.rs +++ b/crates/sandbox/src/fake.rs @@ -5,8 +5,10 @@ use async_trait::async_trait; use lazyboy_contracts::{ComputerObservation, SandboxKind}; use lazyboy_control::{ ActionRequest, ActionResult, AdapterContext, BrowserPage, BrowserRequest, CommandRequest, - CommandResult, ComputerRef, FileEntry, ProvisionRequest, RecordingRequest, RecordingResult, - RecordingSession, SandboxError, SandboxProvider, ScreenSession, observation_from_png, + CommandResult, ComputerJobTable, ComputerRef, EnsureScreenRequest, EnsureScreenResult, + FileEntry, JobSnapshot, JobStatus, ProvisionRequest, RecordingRequest, RecordingResult, + RecordingSession, SandboxError, SandboxProvider, ScreenSession, job_snapshot_result, + observation_from_png, screen_layout, }; const EMPTY_PNG: &[u8] = &[ @@ -20,12 +22,23 @@ const EMPTY_PNG: &[u8] = &[ #[derive(Default)] pub struct FakeSandbox { files: Mutex>>>, + ensure_screen_calls: Mutex, + jobs: ComputerJobTable, + host_spawns: Mutex, } impl FakeSandbox { pub fn new() -> Self { Self::default() } + + pub fn ensure_screen_calls(&self) -> u32 { + *self.ensure_screen_calls.lock().unwrap() + } + + pub fn host_spawn_count(&self) -> u32 { + *self.host_spawns.lock().unwrap() + } } #[async_trait] @@ -49,17 +62,111 @@ impl SandboxProvider for FakeSandbox { }) } + async fn ensure_screen( + &self, + _computer: &ComputerRef, + request: EnsureScreenRequest, + _context: &AdapterContext, + ) -> Result { + *self.ensure_screen_calls.lock().unwrap() += 1; + let layout = screen_layout(request.slot) + .map_err(|error| SandboxError::message(error.to_string()))?; + Ok(EnsureScreenResult { + slot: layout.slot, + display: layout.display, + view_port: layout.view_port, + }) + } + async fn execute( &self, computer: &ComputerRef, request: CommandRequest, _context: &AdapterContext, ) -> Result { + let _ = computer; + if let Some(op) = request.job_op.as_deref() { + let id = request + .job_id + .as_deref() + .ok_or_else(|| SandboxError::message("job_id required"))?; + match op { + "status" => { + let job = self + .jobs + .get(id) + .ok_or_else(|| SandboxError::message("unknown job"))?; + return Ok(job_snapshot_result(&job)); + } + "cancel" => { + let job = self + .jobs + .update(id, |job| { + if job.status == JobStatus::Running { + job.status = JobStatus::Cancelled; + job.exit_code = Some(143); + } + }) + .ok_or_else(|| SandboxError::message("unknown job"))?; + return Ok(job_snapshot_result(&job)); + } + other => { + return Err(SandboxError::message(format!("unknown job_op {other}"))); + } + } + } + if request.background { + let id = request + .job_id + .clone() + .or(request.operation_id.clone()) + .unwrap_or_else(|| "job".into()); + let short = matches!( + request.argv.first().map(String::as_str), + Some("/bin/true") | Some("true") + ); + let failed = matches!( + request.argv.first().map(String::as_str), + Some("/bin/false") | Some("false") + ); + let snapshot = JobSnapshot { + id: id.clone(), + status: if short { + JobStatus::Succeeded + } else if failed { + JobStatus::Failed + } else { + JobStatus::Running + }, + pid: None, + stdout: String::new(), + stderr: String::new(), + exit_code: if short { + Some(0) + } else if failed { + Some(1) + } else { + None + }, + computer_generation: 1, + computer_id: Some(computer.id.clone()), + }; + self.jobs.insert(snapshot.clone()); + return Ok(CommandResult { + stdout: String::new(), + stderr: String::new(), + code: 0, + job_id: Some(id), + signal: None, + status: Some("running".into()), + }); + } if request.argv.first().map(String::as_str) == Some("mkdir") { return Ok(CommandResult { stdout: String::new(), stderr: String::new(), code: 0, + ..CommandResult::default() }); } if request.argv.first().map(String::as_str) == Some("touch") @@ -72,10 +179,16 @@ impl SandboxProvider for FakeSandbox { .or_default() .insert(path.clone(), Vec::new()); } + let code = match request.argv.first().map(String::as_str) { + Some("/bin/true") | Some("true") => 0, + Some("/bin/false") | Some("false") => 1, + _ => 0, + }; Ok(CommandResult { stdout: request.argv.join(" "), stderr: String::new(), - code: 0, + code, + ..CommandResult::default() }) } @@ -247,3 +360,216 @@ impl SandboxProvider for FakeSandbox { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use lazyboy_control::maybe_ensure_screen; + + #[tokio::test] + async fn fake_sandbox_keeps_invalid_utf8_bytes() { + let sandbox = FakeSandbox::new(); + let context = AdapterContext::default(); + let computer = sandbox + .provision( + ProvisionRequest { + home_key: "home".into(), + home_path: "/tmp/home".into(), + ..ProvisionRequest::default() + }, + &context, + ) + .await + .unwrap(); + let raw = vec![0xff, 0xfe, 0x00, b'A']; + sandbox + .write_file(&computer, "bin.dat", &raw, &context) + .await + .unwrap(); + let got = sandbox + .read_file(&computer, "bin.dat", &context) + .await + .unwrap(); + assert_eq!(got, raw); + let listed = sandbox.list_files(&computer, "", &context).await.unwrap(); + assert_eq!(listed[0].size, 4); + } + + #[tokio::test] + async fn native_exec_and_files_do_not_call_ensure_screen() { + let sandbox = FakeSandbox::new(); + let context = AdapterContext::default(); + let computer = sandbox + .provision( + ProvisionRequest { + home_key: "home".into(), + home_path: "/tmp/home".into(), + provider_ref: None, + runner_only: true, + }, + &context, + ) + .await + .unwrap(); + let attached = maybe_ensure_screen( + &sandbox, + &computer, + false, + EnsureScreenRequest { + slot: 0, + profile_path: String::new(), + bot_id: "bot".into(), + bot_name: String::new(), + bot_color: String::new(), + }, + &context, + ) + .await + .unwrap(); + assert!(attached.is_none()); + assert_eq!(sandbox.ensure_screen_calls(), 0); + + sandbox + .write_file(&computer, "notes.txt", b"hello", &context) + .await + .unwrap(); + assert_eq!( + sandbox + .read_file(&computer, "notes.txt", &context) + .await + .unwrap(), + b"hello" + ); + let listed = sandbox + .list_files(&computer, "notes.txt", &context) + .await + .unwrap(); + assert_eq!(listed[0].size, 5); + let ran = sandbox + .execute( + &computer, + CommandRequest { + argv: vec!["/bin/true".into()], + ..CommandRequest::default() + }, + &context, + ) + .await + .unwrap(); + assert_eq!(ran.code, 0); + assert_eq!(sandbox.ensure_screen_calls(), 0); + + let gui = maybe_ensure_screen( + &sandbox, + &computer, + true, + EnsureScreenRequest { + slot: 0, + profile_path: String::new(), + bot_id: "bot".into(), + bot_name: String::new(), + bot_color: String::new(), + }, + &context, + ) + .await + .unwrap(); + assert!(gui.is_some()); + assert_eq!(sandbox.ensure_screen_calls(), 1); + } + + #[tokio::test] + async fn background_exec_runs_on_the_computer_not_the_api_host() { + use lazyboy_control::current_process_has_child_cmd; + + let sandbox = FakeSandbox::new(); + let context = AdapterContext::default(); + let computer = sandbox + .provision( + ProvisionRequest { + home_key: "home".into(), + home_path: "/tmp/home".into(), + ..ProvisionRequest::default() + }, + &context, + ) + .await + .unwrap(); + + let short = sandbox + .execute( + &computer, + CommandRequest { + argv: vec!["/bin/true".into()], + background: true, + job_id: Some("job-true".into()), + ..CommandRequest::default() + }, + &context, + ) + .await + .unwrap(); + assert_eq!(short.job_id.as_deref(), Some("job-true")); + let status = sandbox + .execute( + &computer, + CommandRequest { + job_id: Some("job-true".into()), + job_op: Some("status".into()), + ..CommandRequest::default() + }, + &context, + ) + .await + .unwrap(); + assert_eq!(status.code, 0); + assert_eq!(status.job_id.as_deref(), Some("job-true")); + + let failed = sandbox + .execute( + &computer, + CommandRequest { + argv: vec!["/bin/false".into()], + ..CommandRequest::default() + }, + &context, + ) + .await + .unwrap(); + assert_eq!(failed.code, 1); + + let started = sandbox + .execute( + &computer, + CommandRequest { + argv: vec!["/bin/sleep".into(), "30".into()], + background: true, + job_id: Some("job-sleep".into()), + ..CommandRequest::default() + }, + &context, + ) + .await + .unwrap(); + assert_eq!(started.job_id.as_deref(), Some("job-sleep")); + assert_eq!(sandbox.host_spawn_count(), 0); + assert!( + !current_process_has_child_cmd("sleep 30"), + "background exec must not spawn a child of the API/test process" + ); + let cancelled = sandbox + .execute( + &computer, + CommandRequest { + job_id: Some("job-sleep".into()), + job_op: Some("cancel".into()), + ..CommandRequest::default() + }, + &context, + ) + .await + .unwrap(); + assert_eq!(cancelled.job_id.as_deref(), Some("job-sleep")); + assert_eq!(sandbox.host_spawn_count(), 0); + } +} diff --git a/crates/supervisor/src/docker.rs b/crates/supervisor/src/docker.rs index 61483c2..72cd1e7 100644 --- a/crates/supervisor/src/docker.rs +++ b/crates/supervisor/src/docker.rs @@ -13,9 +13,10 @@ use bollard::models::{EndpointSettings, HostConfig, HostConfigLogConfig, PortBin use bollard::network::{ConnectNetworkOptions, CreateNetworkOptions}; use futures_util::StreamExt; use lazyboy_control::{ - ActionRequest, BrowserRequest, CommandRequest, CommandResult, EnsureScreenRequest, - EnsureScreenResult, HOME, RecordingRequest, ScreenTarget, TEAM_SCREEN_LIMIT, normalize_display, - normalize_workspace_path, screen_layout, + ActionRequest, BrowserRequest, CommandRequest, CommandResult, ComputerJobTable, + EnsureScreenRequest, EnsureScreenResult, HOME, JobSnapshot, JobStatus, RecordingRequest, + ScreenTarget, TEAM_SCREEN_LIMIT, computer_background_launch, job_snapshot_result, + normalize_display, normalize_workspace_path, sanitize_job_id, screen_layout, }; use tokio::time::{Duration, sleep}; @@ -25,6 +26,7 @@ pub struct DockerHost { docker: Docker, image: String, control_token: String, + jobs: ComputerJobTable, } pub struct Provisioned { @@ -58,6 +60,7 @@ impl DockerHost { docker, image, control_token, + jobs: ComputerJobTable::default(), }) } @@ -66,6 +69,7 @@ impl DockerHost { home_key: &str, home_path: &str, space_id: &str, + runner_only: bool, ) -> Result { let data_dir = std::env::var("DATA_DIR").unwrap_or_else(|_| "./data".into()); if home_key.is_empty() @@ -168,6 +172,12 @@ impl DockerHost { privileged: Some(false), shm_size: Some(512 * 1024 * 1024), network_mode: Some(network), + // PID 1 is `sleep infinity` in runner-only mode and start.sh + // otherwise; neither reaps orphans. Background jobs are launched + // with setsid so they reparent to PID 1 when their wrapper exits + // and would stay zombies (kill -0 keeps succeeding, the job looks + // running forever). docker-init reaps them. + init: Some(true), ..Default::default() }; @@ -181,6 +191,10 @@ impl DockerHost { env: Some(vec![ "DISPLAY=:1".into(), format!("HOME={HOME}"), + format!( + "LAZYBOY_RUNNER_ONLY={}", + if runner_only { "1" } else { "0" } + ), format!( "LAZYBOY_CONTROL_TOKEN={}", scoped_control_token(&self.control_token, home_key) @@ -265,6 +279,11 @@ impl DockerHost { request: CommandRequest, target: &ScreenTarget, ) -> Result { + if let Some(op) = request.job_op.as_deref() { + return self + .job_follow_up(id, request.job_id.as_deref(), op, target) + .await; + } let cwd = match request.cwd { Some(cwd) if PathBuf::from(&cwd).is_absolute() => cwd, Some(cwd) => { @@ -277,12 +296,17 @@ impl DockerHost { } None => HOME.to_string(), }; - let timeout_ms = request.timeout_ms.unwrap_or(30_000).clamp(100, 120_000); let argv = if request.argv.is_empty() { vec!["/bin/echo".into(), "ready".into()] } else { request.argv }; + if request.background { + return self + .start_background_job(id, argv, &cwd, request.job_id.as_deref(), target) + .await; + } + let timeout_ms = request.timeout_ms.unwrap_or(30_000).clamp(100, 120_000); let mut bounded = vec![ "timeout".into(), "--signal=TERM".into(), @@ -294,6 +318,184 @@ impl DockerHost { .await } + async fn start_background_job( + &self, + container: &str, + argv: Vec, + cwd: &str, + job_id: Option<&str>, + target: &ScreenTarget, + ) -> Result { + let job_id = sanitize_job_id(job_id.unwrap_or("job")); + let launch = computer_background_launch(&argv, &job_id); + let launched = self + .exec_raw_cmd(container, &launch, Some(cwd), target, None) + .await?; + if launched.code != 0 { + return Err(launched.stderr); + } + // Without a pid the job could never be reaped or cancelled and would be + // reported running forever; refuse rather than track a ghost. + let pid = launched + .stdout + .trim() + .lines() + .last() + .and_then(|line| line.trim().parse::().ok()) + .ok_or_else(|| { + format!( + "background launch did not report a pid (stdout={:?}, stderr={:?})", + launched.stdout.trim(), + launched.stderr.trim() + ) + })?; + self.jobs.insert(JobSnapshot { + id: job_id.clone(), + status: JobStatus::Running, + pid: Some(pid), + stdout: String::new(), + stderr: String::new(), + exit_code: None, + computer_generation: 1, + computer_id: Some(container.to_string()), + }); + Ok(CommandResult { + stdout: String::new(), + stderr: String::new(), + code: 0, + job_id: Some(job_id), + signal: None, + status: Some("running".into()), + }) + } + + async fn job_follow_up( + &self, + container: &str, + job_id: Option<&str>, + op: &str, + target: &ScreenTarget, + ) -> Result { + let job_id = job_id.ok_or_else(|| "job_id required".to_string())?; + let Some(job) = self.jobs.get(job_id) else { + return Err("unknown job".into()); + }; + // Job ids are only meaningful on the Computer that started them; a pid + // from another container must never be signalled here. + if job.computer_id.as_deref() != Some(container) { + return Err("unknown job".into()); + } + match op { + "status" => { + let mut job = job; + if let Some(pid) = job.pid + && job.status == JobStatus::Running + { + // The exit file is written by the wrapper after the command + // returns, so it settles the question even when the wrapper + // is a zombie that `kill -0` still counts as alive. + let exit = self.read_job_file(container, job_id, "exit", target).await; + let exit = exit.trim().to_string(); + let alive = if exit.is_empty() { + self.exec_raw_cmd( + container, + &[ + "bash".into(), + "-c".into(), + lazyboy_control::computer_alive_script(pid), + ], + None, + target, + None, + ) + .await? + .code + == 0 + } else { + false + }; + if !alive { + let out = self.read_job_file(container, job_id, "out", target).await; + let err = self.read_job_file(container, job_id, "err", target).await; + let (status, code) = lazyboy_control::reap_background_job( + false, + if exit.is_empty() { + None + } else { + Some(exit.as_str()) + }, + ); + job = self + .jobs + .update(job_id, |snap| { + snap.status = status; + snap.exit_code = Some(code); + snap.stdout = out; + snap.stderr = err; + }) + .unwrap_or(job); + } + } + Ok(job_snapshot_result(&job)) + } + "cancel" => { + // The launcher ran the wrapper under `setsid`, so its pid is also + // the process-group id: signalling `-pid` reaches the whole tree, + // not just the `sh -c` wrapper (T13). TERM first, a short grace, + // then KILL; only claim cancelled once nothing in the group answers. + let mut tree_gone = job.pid.is_none(); + if let Some(pid) = job.pid { + let script = lazyboy_control::computer_cancel_script(pid); + let outcome = self + .exec_raw_cmd( + container, + &["bash".into(), "-c".into(), script], + None, + target, + None, + ) + .await; + tree_gone = matches!(outcome, Ok(ref result) if result.code == 0); + } + let job = self + .jobs + .update(job_id, |snap| { + if snap.status == JobStatus::Running && tree_gone { + snap.status = JobStatus::Cancelled; + snap.exit_code = Some(143); + } + }) + .ok_or_else(|| "unknown job".to_string())?; + Ok(job_snapshot_result(&job)) + } + other => Err(format!("unknown job_op {other}")), + } + } + + async fn read_job_file( + &self, + container: &str, + job_id: &str, + which: &str, + target: &ScreenTarget, + ) -> String { + let id = sanitize_job_id(job_id); + let result = self + .exec_raw_cmd( + container, + &[ + "bash".into(), + "-lc".into(), + format!("cat /tmp/lazyboy/jobs/{id}/{which} 2>/dev/null || true"), + ], + None, + target, + None, + ) + .await; + result.map(|r| r.stdout).unwrap_or_default() + } + pub async fn observe_payload( &self, id: &str, @@ -786,6 +988,7 @@ PY"#, stdout: String::from_utf8_lossy(&stdout).into_owned(), stderr: String::from_utf8_lossy(&stderr).into_owned(), code, + ..CommandResult::default() }) } diff --git a/crates/supervisor/src/main.rs b/crates/supervisor/src/main.rs index 001b54e..8be69da 100644 --- a/crates/supervisor/src/main.rs +++ b/crates/supervisor/src/main.rs @@ -6,10 +6,11 @@ use axum::extract::{Path, State}; use axum::http::{HeaderMap, StatusCode}; use axum::routing::{delete, get, post}; use axum::{Json, Router}; +use base64::Engine; use docker::DockerHost; use lazyboy_control::{ - ActionRequest, BrowserRequest, CommandRequest, EnsureScreenRequest, HOME, RecordingRequest, - ScreenTarget, normalize_workspace_path, + ActionRequest, BrowserRequest, CommandRequest, CommandResult, EnsureScreenRequest, HOME, + RecordingRequest, ScreenTarget, normalize_workspace_path, }; use serde::{Deserialize, Serialize}; use tracing_subscriber::EnvFilter; @@ -28,6 +29,8 @@ struct ProvisionBody { home_path: String, #[serde(rename = "spaceId")] space_id: String, + #[serde(rename = "runnerOnly", default)] + runner_only: bool, } #[tokio::main] @@ -135,7 +138,12 @@ async fn provision( require_token(&headers, &app.token)?; let created = app .docker - .provision(&body.home_key, &body.home_path, &body.space_id) + .provision( + &body.home_key, + &body.home_path, + &body.space_id, + body.runner_only, + ) .await .map_err(|error| { tracing::error!("provision: {error}"); @@ -153,7 +161,7 @@ async fn exec( headers: HeaderMap, Path(id): Path, Json(body): Json, -) -> Result, (StatusCode, String)> { +) -> Result, (StatusCode, String)> { require_token(&headers, &app.token).map_err(|status| (status, String::new()))?; let result = app .docker @@ -163,11 +171,7 @@ async fn exec( tracing::error!("exec: {error}"); (StatusCode::INTERNAL_SERVER_ERROR, error) })?; - Ok(Json(serde_json::json!({ - "stdout": result.stdout, - "stderr": result.stderr, - "code": result.code, - }))) + Ok(Json(result)) } async fn observe( @@ -373,6 +377,17 @@ async fn list_files( struct FileBody { path: String, content: Option, + #[serde(rename = "contentBase64")] + content_base64: Option, +} + +fn file_body_bytes(body: &FileBody) -> Result, StatusCode> { + if let Some(encoded) = body.content_base64.as_deref() { + return base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|_| StatusCode::BAD_REQUEST); + } + Ok(body.content.clone().unwrap_or_default().into_bytes()) } async fn write_file( @@ -383,8 +398,9 @@ async fn write_file( ) -> Result { require_token(&headers, &app.token)?; let relative = normalize_workspace_path(&body.path).map_err(|_| StatusCode::BAD_REQUEST)?; + let bytes = file_body_bytes(&body)?; app.docker - .write_file(&id, &relative, body.content.unwrap_or_default().as_bytes()) + .write_file(&id, &relative, &bytes) .await .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; Ok(StatusCode::NO_CONTENT) @@ -405,7 +421,8 @@ async fn read_file( .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; Ok(Json(serde_json::json!({ "path": relative, - "content": String::from_utf8_lossy(&bytes), + "encoding": if std::str::from_utf8(&bytes).is_ok() { "utf-8" } else { "binary" }, + "contentBase64": base64::engine::general_purpose::STANDARD.encode(&bytes), }))) } diff --git a/docs/agent-computer-progress.md b/docs/agent-computer-progress.md new file mode 100644 index 0000000..a3cbe81 --- /dev/null +++ b/docs/agent-computer-progress.md @@ -0,0 +1,361 @@ +# Agent Computer progress (V3) + +基準 commit（計劃）：`e6afa324530e19922909d4692c28fb005cc05a7a` +本機開始 HEAD：`79952b088b394e36d6c3db3952165f1c3fd59961` +未覆蓋使用者未提交修改（`docs/plan/` 仍未追蹤）。 + +驗證（2026-09-10 複查，見文末「複查：Bug 修正」）：`cargo fmt --check`、`cargo clippy --workspace --all-targets -D warnings`、`cargo test --workspace`（含 sqlx，全綠）、`make test-agent-computer`、`apps/web npm run build` 全部 exit 0。 +sqlx 測試用拋棄式 `pgvector/pgvector:pg16` 於 `127.0.0.1:5434`（`docker run --rm --name lazyboy-test-pg …`），不重建正在服務的 `lazyboy-postgres-1`。 +真實 OAuth／主信箱／TigerVNC GUI fixture：`BLOCKED_EXTERNAL`。 +sample MCP 在真實 Computer 容器 `lb-team-space-*` 內 `python3 echo_server.py` JSON-RPC 回 `from-computer`。外掛 UI 已 typecheck；無瀏覽器工具未點擊。 + +--- + +## PR-00 / PR-01 + +狀態：IMPLEMENTED（見前次）。截圖交付、semantic/vision、原生 fs/exec 短路徑。 + +--- + +## PR-02：雙模式身份 + +狀態：IMPLEMENTED（資料＋契約＋idle；沒有強制遷移） + +- `computers.generation`；destroy/recreate `generation + 1`（舊 job 不可裝活）。 +- Team 多 bot 同 `computer_id`：`computer_scope_key(Team, space, bot-a) == bot-b`。Dedicated 互斥 scope。 +- **沒有**對 `bots.computer_id` 加 UNIQUE。 +- `AdapterContext.computer_generation` / `control_epoch`。 +- Pause scopes：`PauseScope::{Agent,Display,Computer}`；停 A 不 bump B 的 display epoch。 +- idle reaper 把 `computer_jobs` running 算進活性。 + +回滾：migration 022 + generation SELECT/UPDATE。 + +--- + +## PR-03：ledger / 脫敏 / activity + +狀態：IMPLEMENTED（Postgres 為 mutation 來源；本環境未跑 sqlx 整合測試） + +- `dispatch`：mutating tool 先 `begin`；journal insert 失敗則 `JOURNAL_UNAVAILABLE`，不開做。 +- 同 `operationId` + payload hash replay 已存結果；不同 hash → `PAYLOAD_MISMATCH`。 +- `operationId` 不進入 payload hash。 +- `finish` 寫 `computer_operations` + `operation_outbox`；`worker_loop` `flush_outbox` 只補尚未出現在 `run_activity` 的 `operationId`。 +- 結果與 outbox payload 經過 `redact_text` / `redact_json`。 + +--- + +## PR-04：jobs / cancel / CAS + +狀態：IMPLEMENTED（Runner 在 API 行程內；container recreate 以 generation 中斷） + +- `JobSupervisor`：`process_group(0)`；cancel TERM→KILL process group。 +- `exec`：`action=run|status|cancel`、`background` 回 `jobId`。 +- `write_file.expectedHash` → `CONFLICT`，不覆蓋。 +- 測試：`/bin/true` 真 exit；`sleep 30` 可 cancel。 + +- API 重啟：`computer_jobs` running/accepted → `interrupted`（不假裝收回 Docker PID）。 +- `LAZYBOY_NATIVE_JOB_CONCURRENCY`（預設 2）超額 → `QUOTA_EXCEEDED`。 +- Artifact：`write_file` 成功寫 `computer_artifacts`；`GET /api/bots/{id}/artifacts` 列表；下載從 Computer `read_file` 串回，sha256 不符 → `CONFLICT`。 + +O04 PTY `terminal.*`：DEFERRED（見文末）。跨重啟回收仍在跑的容器 PID：標 interrupted，不是 resume。 + +--- + +## PR-05：Tool Manager / MCP + +狀態：IMPLEMENTED（stdio 不在 API spawn；sample 在 Computer；無第三方商店） + +- Manifest：空 digest / zip-slip / 空 entrypoint 拒絕。 +- 安裝≠Ready（`AuthRequired` ≠ 可用）。 +- `scripts/sample-mcp/echo_server.py` + `manifest.yaml`（本機範例，不是虛構 npm）。 +- REST：`GET/POST /api/bots/{id}/tools`、`POST .../tools/call`、`POST .../revoke`。 +- 安裝把 echo 寫進 Computer：Dedicated `tools/...`，Team `shared/tools/...`；`python3` 在該容器執行 JSON-RPC。 +- Agent 工具 `computer_mcp` 走同一條路徑；未 bind → `NOT_BOUND`。 +- 外掛頁「Computer 套件」可安裝／呼叫／撤銷 echo；市集 MCP 文案不再假裝 stdio 都在 Computer。 +- MCP `call_for` 先複製 client 再 await；`definitions_for(actor)` 不把別人的工具塞進這次 run。 + +未做：任意第三方套件商店、OAuth broker。`/api/mcp-servers` stdio **不再** `spawn` 子行程，只存 metadata 並指向 Computer Tool Manager。HTTP/SSE 仍為遠端 client。真實 Microsoft/Google OAuth：`BLOCKED_EXTERNAL`。 + +--- + +## PR-06：路由 / verifier / wait + +狀態：IMPLEMENTED（決策層） + +- `choose_route`：有授權 API 不走 browser；`POLICY_DENIED` 不繞路。 +- `CompletionLayer`：transport ≠ effect ≠ task；`wait_counts_as_progress() == false`。 +- policy：`wait` 不再刷新無進展時鐘；`exec` 可。 +- `wait_until` 有 deadline；未知 disabled 不空轉。 +- system prompt：Gmail/Outlook 有 connector 走 API。 + +--- + +## PR-07：form / takeover scopes + +狀態：IMPLEMENTED（form 工具已註冊；lease/pause 見 PR-02） + +- `run_form_macro` 失敗即停、逐步 evidence。 +- Display writer lease 原本就有；pause scope 見 PR-02。 +- Agent 工具 `form_fill`：用最新 snapshot locator／element id；`run_form_macro` 先定位，第一個失敗就停；可選 `submit` click。 + +--- + +## PR-08A Gmail fixture + +狀態：IMPLEMENTED（純函式 fixture，不碰真信箱） + +- batchModify 同 delta 分組、1000 cap。 +- undo 只撤本 plan 的 delta。 +- 真實帳號：`BLOCKED_EXTERNAL`。 + +--- + +## PR-08B Outlook + +狀態：IMPLEMENTED（Graph 語意 fixture，非官方 npm） + +- `$batch` 最多 20；子項 429/401 分開。 +- categories merge，不覆蓋無關分類。 +- `Mail.ReadWrite` 不自動允許寄信。 +- 真實 Graph OAuth：`BLOCKED_EXTERNAL`。 + +--- + +## PR-09 TigerVNC 候選 + +狀態：IMPLEMENTED（opt-in，預設仍 Xvfb+x11vnc） + +- `DisplayBackend`；slot 0 = `:1` / 5900 / 6080（T57）。 +- `lazyboy-screen` 讀 `LAZYBOY_DISPLAY_BACKEND=tigervnc_xvnc`。 +- Dockerfile 加入 `tigervnc-standalone-server`，失敗則 script fallback Xvfb。 +- Cua/中文/a11y GUI 對照：`BLOCKED_EXTERNAL`，**未改 default**。 + +--- + +## PR-10 設定 / 文件 + +狀態：IMPLEMENTED + +`.env.example`：`LAZYBOY_EXECUTION_PROFILE`、`LAZYBOY_DISPLAY_BACKEND`、`LAZYBOY_BROWSER_BACKEND`、concurrency、tool install flags。 +`migrations/022_agent_computer.sql`。 + +--- + +## O01–O48 + +| ID | 狀態 | +|---|---| +| O01 | IMPLEMENTED | +| O02 | IMPLEMENTED（短 exec）；長 job 見 O04 | +| O03 | IMPLEMENTED + CAS | +| O04 | PARTIAL（Computer 內 background exec + cancel；PTY `terminal.*` DEFERRED） | +| O05 | IMPLEMENTED（form macro + `form_fill` 工具） | +| O06 | IMPLEMENTED（stdio 不在 API spawn；HTTP/SSE 遠端；echo 在 Computer） | +| O07 | DEFERRED（未量測 docker exec 開銷） | +| O08 | IMPLEMENTED（`boot_for(need_gui=false)` 不 ensure_screen；`LAZYBOY_RUNNER_ONLY` 不 boot XFCE） | +| O09 | IMPLEMENTED（router） | +| O10 | IMPLEMENTED（revoke 從下次 schema 拿掉 computer_mcp；stdio 不再進 defs） | +| O11 | 保留現況 | +| O12 | IMPLEMENTED | +| O13 | DEFERRED | +| O14 | IMPLEMENTED（PR-01） | +| O15 | 保留現況（既有 profile bind） | +| O16 | PARTIAL（verifier 層 + wait 不算進度） | +| O17 | 實驗／opt-in | +| O18 | DEFERRED | +| O19 | 保留現況 | +| O20 | 保留現況 | +| O21 | 保留現況 | +| O22–O24 | DEFERRED | +| O25 | IMPLEMENTED／保留 | +| O26 | PARTIAL（既有 screen/profile lease） | +| O27 | IMPLEMENTED（epoch 函式） | +| O28 | PARTIAL（idle 看 runs/skills/jobs；2026-09-10 修正守門原本永遠回 false，並在判斷前 reconcile job 狀態） | +| O29 | IMPLEMENTED（package/binding 分離；共用 bytes 不共享帳號） | +| O30 | IMPLEMENTED（`LAZYBOY_NATIVE_JOB_CONCURRENCY`，預設 2；配額滿時先向 Computer reconcile 未 poll 的 job，不會永久卡死） | +| O31 | DEFERRED | +| O32 | DEFERRED（T59：保留 `-ac` 直到 Xauthority 接到 Cua/AT-SPI；VNC 仍走認證 proxy） | +| O33 | IMPLEMENTED（sample MCP 安裝 REST + UI） | +| O34 | IMPLEMENTED（digest／zip-slip／unpinned npx 拒絕） | +| O35 | IMPLEMENTED（installed ≠ ready；AuthRequired 不可用） | +| O36 | PARTIAL（熱啟用 bindings；無獨立 schema cache 服務） | +| O37 | IMPLEMENTED（0.0.2 旁置、job pin 舊版、rollback／remove） | +| O38 | BLOCKED_EXTERNAL | +| O39–O40 | IMPLEMENTED fixture；真實帳號 BLOCKED_EXTERNAL | +| O41 | IMPLEMENTED（dispatch 寫 operations + outbox；flush 補送） | +| O42 | IMPLEMENTED（errorCode/jobId/operationId） | +| O43 | IMPLEMENTED（列表 + 下載；磁碟 hash 不符 CONFLICT） | +| O44 | PARTIAL（verifier 層） | +| O45 | IMPLEMENTED（CAS + operation id） | +| O46 | IMPLEMENTED（`GET /api/computer/{id}/health`：db／sandbox／display／jobs／operations） | +| O47 | DEFERRED | +| O48 | DEFERRED（無固定環境雙模式 P95；不寫未測倍數） | + +--- + +## T01–T64 + +| ID | 狀態 | 證據／理由 | +|---|---|---| +| T01 | IMPLEMENTED | `should_deliver_observation_image` | +| T02 | IMPLEMENTED | 換模型／takeover 強制交付（同真值表） | +| T03 | IMPLEMENTED | `pixel_actions_need_vision`；text-only 無圖 | +| T04 | IMPLEMENTED | `typed_browser_errors` | +| T05 | IMPLEMENTED | Team/Dedicated path + echo 路徑測試 | +| T06 | PARTIAL | generation 拒絕舊 job；未覆蓋所有偽造 grant 變體 | +| T07 | 保留現況 | task 無 Docker socket；未加新 sentinel fixture | +| T08 | IMPLEMENTED | zip-slip／`..` path 拒絕 | +| T09 | IMPLEMENTED | invalid UTF-8 binary payload | +| T10 | PARTIAL | 工具錯誤非空成功；磁碟滿未測 | +| T11 | IMPLEMENTED | `stale_hash` CAS CONFLICT | +| T12 | IMPLEMENTED | Supervisor `/exec` 回完整 `CommandResult`；reap 讀 `exit` 檔，`false`/缺檔為 Failed/1 | +| T13 | IMPLEMENTED | Computer-side job_op cancel 走 `computer_cancel_script`（TERM→KILL 整個 process group，驗證後才標 cancelled；2026-09-10 修正 pid 錯誤）；status 缺欄位當 TRANSPORT，不發明 running | +| T14 | DEFERRED | API 重啟標 interrupted，不 resume 容器 PID | +| T15 | IMPLEMENTED | generation interrupt | +| T16 | IMPLEMENTED | operation_id hash／mismatch；2026-09-10 第二輪：key 改 per-bot、in-flight 同 id 回 `UNKNOWN_EFFECT`、傳輸失敗不存成 succeeded（sqlx 測試 `ledger_is_per_bot_and_does_not_rerun_in_flight_work`） | +| T17 | IMPLEMENTED | outbox flush 補送且不重複（sqlx 測試 `outbox_flush_does_not_duplicate_recorded_operations`；2026-09-10 修正 `SELECT 1` 解碼 bug） | +| T18 | IMPLEMENTED | `journal_failure` 拒 mutation | +| T19 | IMPLEMENTED | actor-scoped MCP；stdio 不在 API 持有 client | +| T20 | PARTIAL | NOT_BOUND／revoke；無真實 token revoke | +| T21 | IMPLEMENTED | `canary_is_stripped`；2026-09-10 第二輪：ledger／snippet 實際套用 `redact_secret_patterns`（之前傳空 canary 等於沒脫敏） | +| T22 | 保留現況 | 既有 prompt 邊界；無新 injection corpus | +| T23 | PARTIAL | 同 display lease 既有；Cua+worker 未雙開 fixture | +| T24 | PARTIAL | pause/takeover；HTTP in-flight barrier 未單獨測 | +| T25 | IMPLEMENTED | wait 不算 milestone | +| T26 | IMPLEMENTED | Gmail batchModify fixture | +| T27 | IMPLEMENTED | fixture 429/partial；真實帳號 BLOCKED_EXTERNAL | +| T28 | IMPLEMENTED | undo 只撤本 plan | +| T29 | IMPLEMENTED | 無強制拆 Team；migration 不 UNIQUE computer_id | +| T30 | IMPLEMENTED | `boot_for(need_gui=false)` 不 `ensure_screen`；`maybe_ensure_screen` + FakeSandbox native write/read/exec；`start.sh` runner-only 不 boot XFCE | +| T31 | PARTIAL | job 配額；無壓力 bench | +| T32 | DEFERRED | 無郵件分類品質集 | +| T33 | PARTIAL | broker 不進 generic env；無任意 exec 讀 socket fixture | +| T34 | IMPLEMENTED | artifact 從 Computer 下載 | +| T35 | PARTIAL | CAS／operation replay；resume 不重播未全測 | +| T36 | PARTIAL | mutating tools 走 ledger；MCP stdio 不再是旁路 spawn | +| T37 | IMPLEMENTED | `team_computers_share`；migration 註解+測試 | +| T38 | IMPLEMENTED | Team `bots//` 路徑 | +| T39 | IMPLEMENTED | shared/ + CAS | +| T40 | PARTIAL | 多 slot 契約；無雙 writer GUI fixture | +| T41 | PARTIAL | display writer lease 既有 | +| T42 | IMPLEMENTED | pause scope 停 A 不 bump B | +| T43 | PARTIAL | idle 看 jobs；整機 pause 未單測 recreate 全集 | +| T44 | 保留現況 | 既有 ensure-screen single-flight | +| T45 | IMPLEMENTED | Dedicated 刪 bot；Team Computer 存活 | +| T46 | IMPLEMENTED | echo 安裝／呼叫 `executionLocation=assigned_computer` | +| T47 | IMPLEMENTED | binding per bot；套件 per computer | +| T48 | IMPLEMENTED | digest／zip-slip／unpinned npx | +| T49 | IMPLEMENTED | `plan_version_switch` job pin | +| T50 | IMPLEMENTED | revoke／remove；revoke 掉 schema | +| T51 | IMPLEMENTED | Outlook fixture；非官方 npm | +| T52 | IMPLEMENTED | Graph 子項 429/401 | +| T53 | PARTIAL | fixture 分頁語意；真實 delta BLOCKED_EXTERNAL | +| T54 | IMPLEMENTED | categories merge 不盲覆蓋 | +| T55 | BLOCKED_EXTERNAL | 無 Cua/中文/a11y GUI | +| T56 | PARTIAL | generation bump；backend 熱切未做 | +| T57 | IMPLEMENTED | slot 0 = :1/5900/6080 | +| T58 | 保留現況 | 既有 noVNC；無新 DPI fixture | +| T59 | DEFERRED | 保留 `-ac`；VNC 走認證 proxy；Xauthority 未接到 Cua | +| T60 | IMPLEMENTED | 同 T30：native 路徑不 attach viewer；GUI 之後才 `lazyboy-screen ensure` | +| T61 | IMPLEMENTED | 未審查 env token／unpinned npx 拒絕 | +| T62 | 保留現況 | 同 UID 不宣稱強隔離 | +| T63 | PARTIAL | job 配額；renderer OOM 未測 | +| T64 | PARTIAL | 套件路徑可重建；舊 job 不裝活 | + +## 2026-09-10 複查：Bug 修正 + +本輪先跑計劃第 16 節的完整 gate，再修實際發現的 bug；未新增功能。 + +驗證命令與結果（本機，拋棄式 Postgres `pgvector/pgvector:pg16` 於 `127.0.0.1:5434`）： + +- `cargo fmt --all -- --check`：原本有 diff（`artifacts.rs`、`computer.rs` 等新檔未格式化）→ 已 `cargo fmt --all`，現在 exit 0。 +- `cargo clippy --workspace --all-targets -- -D warnings`：exit 0。 +- `DATABASE_URL=… cargo test --workspace`：**全部通過**（api 157 passed / 3 ignored；control 150；harness 37；contracts 7；sandbox 7；supervisor 2）。前次紀錄的「sqlx 未連」現在已實跑；14 個 sqlx 測試（db／memory／monitor／routing::fan_out）全綠。 +- `make test-agent-computer`：exit 0。 +- `apps/web`：`npm run build`（`tsc --noEmit` + vite）通過。 + +修正： + +1. **`lazyboy-api` 測試 binary 結束時 SIGABRT**（`ort` `Mutex poisoned` → `release_env_on_exit` 二次 panic）。根因：`memory.rs` 用 `catch_unwind` 包住 `TextEmbedding::try_new`，但 `ort` 找不到 `libonnxruntime.so` 時是在持有全域鎖的情況下 `expect` panic，鎖被 poison 後程序退出時 `.fini_array` hook 再 panic → abort。正式環境沒設 `ORT_DYLIB_PATH` 時，API 正常關機也會變 exit 134。修法：`memory.rs` 新增 `preload_onnx_runtime()`，先走 `ort::init_from(path)`（回 `Result`）預載 dylib，失敗即標 `Unavailable` 不再進 fastembed；`crates/api/Cargo.toml` 直接依賴同版 `ort ="=2.0.0-rc.13"`（lock 無新增 crate）。測試 `database_enforces_agent_scope_and_queries_do_not_leak` 改成顯式 `ModelState::Unavailable`，不再靠環境缺 ONNX 來觸發；新增 `a_missing_onnx_runtime_is_an_error_not_a_panic`。反向驗證：`ORT_DYLIB_PATH=/nonexistent cargo test -- --ignored memory_model_recovers_after_cache_failure` 只因無模型 fail，**不再 abort**。 +2. **Computer 內 background job 的 pid 是錯的（T13 process-tree cancel 實際上沒生效）**。`computer_background_launch` 原本是 `mkdir -p … && setsid nohup sh … & echo $!`，`&` 作用在整個 `a && b` list，`$!` 是 bash 子 shell 的 pid，不是 `setsid` 後的 session leader。後果：`status` 只是碰巧能用（子 shell 會 wait）；`cancel` 只殺子 shell，真正的 `sh`→子程序變孤兒繼續跑，之後 `status` 讀不到 exit 檔還會誤報 `failed`。修法：`mkdir` 獨立一行；wrapper 先 `printf %s "$$" > …/pid`，launcher 等 pid 檔再輸出（`setsid` 若 fork 也正確）。新增 `computer_cancel_script(pid)`：`kill -TERM -- -pgid` → 最多 1s 等待 → `kill -KILL -- -pgid` → 再驗證；exit 0 才算整棵樹已消失。`supervisor/docker.rs` cancel 改用它，且只有樹確實消失才把 job 標 `cancelled`（否則維持 `running`，不假稱取消）。測試：`computer_cancel_script_kills_the_whole_process_tree`（本機真起 `sh -c 'sleep 300; sleep 300'`，用 `pgrep -g` 驗證子 `sleep` 一起死）；另在真實 Team 容器 `lb-team-space-c7bace…` 內手動跑 launcher＋cancel：pid=pgid=sid，`tree gone`。 +3. **supervisor `job_follow_up` 沒檢查 job 屬於哪個容器**：拿到別的 Computer 的 jobId 時會在自己容器對同號 pid 送 `kill`。現在 `job.computer_id != container` 一律回 `unknown job`。 +4. **`Makefile` `help` 目標壞掉**：新增的 `test-agent-computer` 說明被塞進同一個 `@echo "…` 造成跨行未閉合字串，`make help` 回 `Unterminated quoted string` Error 2。已拆成兩行 `@echo`。 +5. **idle reaper 的「有工作就不暫停」守門一直是死的（HEAD 既有 bug）**：`computer_has_active_work` 把 `SELECT 1`（INT4）解成 `(i64,)`，sqlx 回 `ColumnDecode mismatched types`，被 `matches!(Ok(Some(_)))` 吞成 false → 只要 `computers.updated_at` 十分鐘沒動，正在跑 run 的 Computer 也會被 suspend。改成 `sqlx::query(...).fetch_optional` 只看有沒有列。sqlx 測試 `idle_reaper_reconciles_unpolled_jobs_with_the_computer` 同時覆蓋 run 與 job 兩種活性。 +6. **`flush_outbox` 去重失效（T17 宣稱的「不重做」實際沒生效）**：同樣的 `SELECT 1`→`i64` 解碼錯誤讓 `exists` 永遠 None，每筆 outbox 都再寫一次 `run_activity`，Activity 出現重複 tool 事件。已改為列存在檢查；查詢出錯時保留 outbox 列到下一輪，不猜。新增 sqlx 測試 `outbox_flush_does_not_duplicate_recorded_operations`。 +7. **背景 job 配額會永久卡死**：`computer_jobs.status` 只在 agent 呼叫 `exec status` 時更新；兩個跑完但沒人 poll 的 job 會讓該 Computer 之後永遠 `QUOTA_EXCEEDED`，idle reaper 也永遠以為有工作。修法：`tools.rs` 配額看起來滿時先 `refresh_running_jobs`（向 Computer 問最舊 16 筆 running 的真實狀態；supervisor 回 `unknown job` 標 `interrupted`）再重算；`computer.rs` 的 `pause_idle_computers` 在判斷活性前做同樣的 reconcile（suspended 容器無法問，`stop_parked_computers` 不做）。 + +未修、已知： + +- host 端 `JobSupervisor::start`（`crates/control/src/jobs.rs`）用 piped stdout 但只在子程序結束後 `read_to_end`；輸出超過 pipe buffer（~64 KiB）的子程序會永久阻塞。目前正式路徑不用它（背景 job 全在 Computer 內，`exec_background_does_not_spawn_on_the_api` 有守門），僅測試使用；若之後要在 API 行程內跑 job 必須先改成邊跑邊 drain。 +- supervisor 的 `ComputerJobTable` 在記憶體；supervisor 重啟後舊 jobId 回 `unknown job`（API 側已把 `computer_jobs` 標 interrupted，屬同一限制，T14 仍 DEFERRED）。 + +回滾：還原 `crates/api/src/memory.rs`、`crates/api/Cargo.toml`（ort 直接依賴）、`crates/api/src/tools.rs`（refresh_running_jobs）、`crates/api/src/computer.rs`、`crates/api/src/operations.rs`、`crates/control/src/jobs.rs`、`crates/supervisor/src/docker.rs`、`Makefile` 本節改動即可；無 migration 變動。 + +## 2026-09-10 複查（第二輪）：三路平行 code review 後的修正 + +三個獨立 review（ledger/outbox/artifacts；jobs/tool-manager/install；雙模式/pause/前端）共回報 40 餘項；重複、已在上一節修掉的略過。以下每項都有測試或反向驗證。gate：`cargo fmt --check`、`clippy -D warnings`、`cargo test --workspace`（api 162／control 154／harness 37）、`tsc --noEmit` 全綠。 + +**Ledger（T15–T17，`operations.rs` 重寫）** + +1. **operationId 全域命名空間 → 跨 bot replay／拒絕**：ledger key 改為 `{bot_id}:{operation_id}`，查詢加 `bot_id` 條件。Bot B 選到 Bot A 用過的 id，不會拿到 A 的結果，也不會被 `PAYLOAD_MISMATCH` 擋。 +2. **記憶體 `LEDGER` 拆掉**：與 DB 兩份狀態不一致（一邊 accepted 一邊沒列）、無上限增長（連非 mutating tool 都建列）。Postgres 是唯一真相；`OperationLedger` 只留 control crate 單元測試用。 +3. **`bot_computer_id` 查詢失敗被當「沒有 Computer → Proceed」**：現在 DB 錯誤 = `JOURNAL_UNAVAILABLE`；不需要 Computer 的 mutation（remember／schedule）以 `computer_id = NULL` 入帳（migration 023 放寬 NOT NULL）。 +4. **同 id 同 payload 的 `accepted` 列直接 Proceed → 崩潰後重試會做第二次**：新增 `Begin::InProgress` → 工具回 `UNKNOWN_EFFECT`，要求先讀回狀態、換新 id 重試。 +5. **`finish` 對非 mutating tool 也寫 DB／outbox**：提早 return。`exec status` 改為非 mutating（`exec run/cancel` 仍是）。 +6. **`"xxx failed: …"` 的傳輸失敗被存成 `succeeded` 並永遠 replay**：`outcome_status` 辨識這類文字 → 刪除 accepted 列讓同 id 可重試；不寫入結果。 +7. **replay 丟掉 `pause`**：`Begin::Replay` 帶回 `pause`。 +8. **outbox 寫 `run_activity` 失敗仍標 delivered**：新增 `monitor::try_record_pool`，只有成功才標；且只處理 `created_at` 早於 2 秒的列，避免與 `runs.rs` 的直接寫入賽跑而重複。 +9. **`redact_text(&text, &[])` 是 no-op**：新增 `redact_secret_patterns`（Bearer、`password=`／`token:`、`sk-`／`ghp_`／`AKIA`／`xox`／`AIza`、PEM 私鑰、JSON secret key），ledger 結果與 activity snippet 都經過它。control crate 新增 `regex` 直接依賴（lock 已有）。 +10. retention 新增 `operation_outbox`（delivered，7 天）與 `computer_operations`（非 accepted，90 天）。 + +**Jobs（T13／T14）** + +11. **runner-only 容器 PID 1 是 `sleep infinity`，不收屍**：setsid 後的 wrapper 結束會變 zombie，`kill -0` 仍成功 → job 永遠 `running`。修：`HostConfig.init = true`（docker-init 收屍）；`status` 改先讀 exit 檔、再用 `computer_alive_script`（看 `/proc/*/status` State，Z/X 視為死）。測試 `computer_alive_script_treats_a_zombie_as_dead` 本機造真 zombie 驗證。 +12. **launcher 沒吐出 pid 時仍登記 job（pid None）→ 永遠 running、不能 cancel**：現在直接回錯誤，不登記幽靈 job。 +13. **`exec status/cancel` 不看歸屬與世代**：新增 `job_ownership`：別的 bot 的 job → `UNKNOWN_JOB`；`generation` 小於目前 Computer 世代或已 `interrupted` → 直接回 interrupted，不去問新容器；`persist_job` 的 upsert 不再把 `interrupted` 蓋回 `running`。 +14. **`pin_running_jobs` 把同 Computer 上其他 bot 的 job 也 pin 了**：加 `bot_id` 條件。 + +**Tool Manager（T18–T21）** + +15. **revoke 後重裝撞 `UNIQUE(package_row_id, bot_id)` → 500**：`ON CONFLICT … DO UPDATE SET status='ready' RETURNING id`；切版本走 `switch_binding`（同 tx 內先清掉同 (package_row, bot) 的非 ready 殘留）。 +16. **rollback 會把 `revoked` binding 復活**：非 `ready` → 409。 +17. **執行前不驗 digest**：`verified_run_argv` 在同一條命令內 `sha256sum -c` 通過才 `exec python3`；不符 exit 97 → `PACKAGE_TAMPERED`。測試 `tampered_package_is_refused_before_it_runs` 真改檔驗證。 +18. **`validate_manifest` 只檢查非空**：現要求 64 hex；`compute-at-install` 佔位會被拒。 +19. `InstallBody` 只吃 `package_id`、前端送 `packageId`：serde `rename` + `alias` 兩者皆收。 +20. `refuse_unpinned_npx` 把 `@scope/pkg` 的 `@` 當版本 pin：修為去掉 scope 前綴再判斷。 + +**其他** + +21. **`form_fill` 靜默截斷**：>20 欄或欄位缺 name／locator → `INVALID_ARGUMENT`，不再填半張表回報成功。 +22. **`looks_like_css` 誤判 `v1.2 Release`／`C++ Guide` 這種標題為 CSS**：click／type 先做精確唯一標題比對；找不到才依 locator 形狀決定 `SELECTOR_UNSUPPORTED`／`TARGET_NOT_FOUND`／`TARGET_STALE`，並各給對應說明。 +23. **`computers.display_backend` 從未寫入**：boot 轉 running 時寫入 `DisplayBackend::from_env()`；health 不再永遠回欄位預設值。 +24. **`delete_bot` 不清 `tool_bindings`／`computer_jobs`／`computer_operations`／`computer_artifacts`**：同 tx 一起刪（Team Computer 的配額不再被已刪 bot 佔用）。 +25. **`wait` 在 `PROGRESS_WHEN_REPEATED` 內，與本文件「wait 不刷新無進展時鐘」相反**：改成 heartbeat 語意 — `wait` 只更新 `last_heartbeat_turn`（不會被當 loop halt），但 stale 教練訊息照常觸發；`polling_a_build_or_a_queue_is_not_treated_as_spinning` 同時鎖住兩半契約。 +26. **Gmail fixture 沒有真的 undo**：新增 `undo_plan(before, batches)`，只還原本次計畫實際改動的 label（原本就有的不拔、原本沒有的不補、人工新加的不動），測試做 plan→undo 往返驗證。 +27. 前端：`RunActivityEntry.operationId` 補型別；`runs.rs` 從工具回傳 JSON 抽 `jobId` 寫進 activity，monitor 原有的 `entry.jobId` 顯示終於有值；並顯示 `op <短 id>`。 + +**Migration**：新增 `migrations/023_agent_computer_fixes.sql`（`computer_operations.computer_id` DROP NOT NULL、移除多餘 `computer_operations_id_hash`、outbox pending 部分索引、`computer_operations(bot_id, created_at)` 索引）。不改 022（已可能套用過）。 + +**review 提到但本輪不動、記錄在此**： +- Pause scopes／`resume_after_pause`／`route_operation` 有純函式與測試但沒接進 runs 主迴圈；takeover 實際只有 GUI 全暫停，沒有 scope 級細分（T22–T24 應視為 PARTIAL，不是 DONE）。 +- `write_file` 走 `bash -lc` 字串插值路徑（HEAD 既有）；`normalize_workspace_path` 有限制字元，本輪未改成 argv。 +- supervisor `ComputerJobTable` 仍在記憶體（T14 DEFERRED 不變）。 + +回滾：還原本節列出的檔案，並 `DELETE FROM _sqlx_migrations WHERE version=23` 後 `ALTER TABLE computer_operations ALTER COLUMN computer_id SET NOT NULL`（若已無 NULL 列）。 + +## 誠實未完成 + +P0/P1 可在本環境落地的已接上。下面不是「做完」： + +1. 真實 Gmail/Outlook OAuth／主信箱：`BLOCKED_EXTERNAL`。 +2. TigerVNC 當預設 + Cua/中文/a11y GUI：`BLOCKED_EXTERNAL`；預設仍 Xvfb+x11vnc。 +3. Postgres sqlx 整合測試此環境未連。 +4. **O04 PTY `terminal.*`：DEFERRED** — 非互動走 `exec`+jobs+process-tree cancel；TUI 協作仍用 GUI `shell`。不假裝有 PTY。 +5. 跨重啟回收容器內仍在跑的 PID：標 interrupted，不是 resume。 +6. 任意第三方套件商店、完整 OAuth broker：沒有憑證。 +7. O48 無未測速度倍數。 +8. 外掛 UI 未在瀏覽器點過（無瀏覽器工具）；已 `tsc --noEmit`。 + +雙模式：Team/Dedicated 契約不變；**沒有** UNIQUE `bots.computer_id`。stdio MCP **不是** API 子行程。 + +回滾：還原本輪 crate / migration 022 / lazyboy-screen `-ac` 註解 / sample-mcp / 前端 plugins+i18n。不要 force push。 diff --git a/docs/plan/LAZYBOY_AGENT_COMPUTER_IMPLEMENTATION_PLAN_V3.md b/docs/plan/LAZYBOY_AGENT_COMPUTER_IMPLEMENTATION_PLAN_V3.md new file mode 100644 index 0000000..79d60e1 --- /dev/null +++ b/docs/plan/LAZYBOY_AGENT_COMPUTER_IMPLEMENTATION_PLAN_V3.md @@ -0,0 +1,1227 @@ +# LazyBoy：共用／私人 Computer、混合工具與可安裝連接器實作計劃 + +- 文件版本：3.0（2026-09-10；保留共用／私人主機，新增逐項優化、TigerVNC 與外部工具安裝） +- 對象：接手 LazyBoy 的 coding agent 與 reviewer。 +- 審查基準：`igs170911/LazyBoy`，`e6afa324530e19922909d4692c28fb005cc05a7a`；沿用前次已核對的基準；本次另讀此 commit 的 display 啟動與 MCP 程式，未聲稱本機或最新遠端 HEAD 已改好。 +- 文件性質：實作規格與驗收計劃；不是已套用的 patch、不是已通過的測試報告，也沒有實測提速倍數。 +- 建議在 repository 內保存於 `docs/agent-computer-implementation-plan.md`。 + +> 最終目標：**共用主機（Team Computer）與私人主機（Dedicated Computer）都是正式、長期支援的產品模式。** 每個 agent 綁定自己獲准使用的持久 Computer 與工具；可以多個 agent 共用一台 Computer，也可以一個 agent 使用私人 Computer。實際任務與檔案操作在綁定的 Computer 內完成，不必每步在桌面演出，但必須有操作紀錄與結果驗證。工具可安裝、授權、更新與移除。 + +> **取代舊版：不要執行 2.0 中「移除 Team 模式、所有 agent 強制獨立容器、全面遷移 shared → dedicated」的要求。** 本版已直接修正架構、資料約束、PR、測試與啟動指令，不只是附加例外。不要刪除既有共用 Computer 或使用者資料。 + +本版閱讀順序：第 0–3 節為產品與執行邊界；第 15 節為實作依賴；第 18 節為 48 項改善清單；第 19 節是 TigerVNC；第 20–21 節是外掛安裝與 Outlook；第 16 節含 64 個測試案例。 + +## 0. 給 coding agent 的執行指令 + +直接依本文件修改既有程式，不要只重新產生另一份計劃。先確認本地 HEAD、工作目錄與既有測試，再依第 15 節的 PR 順序逐段實作。不得覆蓋使用者未提交的修改。每個階段均須附上實際 diff、測試結果、已知限制及回滾方式。 + +本文件優先於先前健檢文件中以下已變更的假設： + +1. **不再要求所有動作在 VNC 或可見終端機演出。** 正確的結構化紀錄即可；桌面是需要時可觀看、可接管的執行介面。 +2. **保留 shared 與 dedicated 兩種模式，禁止強制轉換或取消 Team Computer。** shared 允許多 agent 對一個 Computer，沿用 per-agent workspace／display／profile 與 shared/；dedicated 才要求獨立 Computer。共享資源與私人授權分開，不能把不同資料夾／DISPLAY 宣稱為惡意程式隔離。 +3. 不把「用 API／CLI 加速」解讀為在 LazyBoy API 主機上執行 agent 工作。連接器、MCP 子程序、檔案工具、命令與瀏覽器控制的實際執行端都在指定 Computer。 +4. 保留現有 Rust harness、Cua、Supervisor、SandboxProvider、React/noVNC、Vault、記憶、排程及 checkpoint；不做全面改寫。 +5. 自動操作的範圍是使用者授權、該 Computer 能提供的能力。不得將「任何事情」翻譯成繞過 CAPTCHA／2FA、任意取用其他 agent／主機資料，或免除高風險操作核准。 + +遇到外部 OAuth、GPU、GUI 或付費模型環境缺失時，完成能完成的程式、fixture、mock 與單元測試；對受影響整合項目記錄 `BLOCKED_EXTERNAL` 和精確缺項。不得把 mock 成功寫成真實服務已驗證；也不得為完成測試而操作真實主信箱。 + +--- + +## 1. 必須成立的產品契約 + +### 1.1 不可違反的條件 + +| ID | 條件 | 必要驗收證據 | +|---|---|---| +| I01 | agent 的 shared／dedicated 綁定正確；私人 Computer 不與其他 agent 隱式共用，共用 Computer 明確列 membership 與 per-agent 執行 scope | 兩模式建立／恢復測試；membership、slot、profile 與文件 scope 驗證 | +| I02 | 所有 agent 命令、任務檔案操作、連接器與本地 MCP 在該 Computer 執行 | API 與 Computer 放入不同 sentinel；程序／namespace 與 server-bound computer identity | +| I03 | agent 任意程式不能在 API／Supervisor 主機執行 | 偽造 target、host fallback、MCP spawn 測試；只保留可信任管理操作 | +| I04 | 工具是否可用取決於 agent grant、Computer 健康、account、scope 與 resource policy | 未授權、已撤銷、錯帳號與跨 agent 測試 | +| I05 | 每次工具呼叫可追蹤，包含失敗、拒絕、取消與結果未知 | operation ledger、UI timeline、outbox 重播／斷線測試 | +| I06 | 原生檔案與命令不需要 screenshot 或 vision 模型 | text-only 模型與 screenshot counter 測試 | +| I07 | 同一資源的寫入不競爭；不同 agent 不被全域鎖拖住 | 兩 agent 併行、慢 MCP、同頁雙寫測試 | +| I08 | 不確定的副作用不盲目重播 | 逾時但伺服器已執行、重送相同 operation、取消競爭測試 | +| I09 | 最終成功必須有任務層驗證證據 | file hash、表單確認值、讀回 label；工具 200 不足以完成 | +| I10 | 暫停、接管、取消與恢復涵蓋背景工具，不只涵蓋滑鼠 | barrier/fencing、background job、in-flight API 測試 | +| I11 | 密碼、token、broker key 不進一般 prompt、tool args、trace 或記憶 | secret canary、log／checkpoint／錯誤回應掃描 | +| I12 | 失敗、重啟或升級不得默默換電腦、換帳號或遺失產物 | container recreation、binding generation、artifact checksum 測試 | +| I13 | shared 與 dedicated 都支援原生工具加速；共用模式不得被降級成 GUI-only 或待淘汰模式 | shared 原生 file／exec／connector 與雙 display 並行測試 | +| I14 | 安裝套件、授權工具、綁定帳號是三個不同動作 | 共用安裝一次，未授權 agent 仍不能透過 broker 使用別人的帳號 | +| I15 | 任務執行、agent 截圖、人類桌面串流各自可量測，不混為一種延遲 | noVNC 關閉時 native 任務仍成功；VNC 與模型 critical path 分開 | +| I16 | 對「應用層 scope」與「OS 強隔離」誠實標示，不能用 prompt 承諾未知程式的安全 | 同 UID／任意 shell／第三方 MCP 的 threat-model 測試與限制說明 | + +### 1.2 「檔案都在 Computer 內」的精確定義 + +**任務檔案**包含原始附件、下載檔、repo checkout、暫存檔、CSV、報表、程式輸出、完整工具輸出與任務執行產物。其讀寫、解析、壓縮、轉檔與掃描程序都必須在對應 Computer 執行。 + +允許持久 volume 的實體 bytes 由 Docker 管理並儲存在宿主機磁碟；這不等於 agent 可以直接讀寫宿主機路徑。不得把 API 主機的 `/`、任意 home 或 Docker socket 掛入 Computer。 + +**控制面資料**仍可保留在既有 PostgreSQL：agent 設定、授權 metadata、run/checkpoint、經脫敏的工具摘要、artifact reference、hash、耗時與事件狀態。API 可串流附件進 Computer、串流工具結果／下載給使用者、轉送模型需要的內容，但不得在主機暫存並處理任務檔案。大型原始輸出與 artifact 正本留在 Computer，API 只保存有權限的 reference。 + +使用外部模型或 SaaS API 仍會傳送必要資料；「在 Computer 執行」不是離線或零資料外送的承諾。模型推論及編排可在控制面；對 Gmail 等目標服務的工具請求由 Computer 內的連接器發出。 + +### 1.3 能力與安全邊界 + +「能做各種電腦工作」以能力可擴充為目標，不限定 Gmail 或某個網站。但容器中的硬體、GPU、音訊、USB、kernel、特定 GUI 或企業 SSO 是否可用，必須由 capability probe 回報，不能偽造已支援。 + +一般命令以非 root 使用者執行。額外套件優先裝在使用者環境；需系統權限的安裝由受控、可稽核的管理流程核准。不可為了可操作性讓 LLM 自行取得 Docker daemon、host root 或修改 Runner 的權限設定。 + +--- + +## 2. 現況與重用範圍 + +下列 F01、F03 於本次再次讀取原始碼確認；其餘為同一 commit 的前次健檢發現，coding agent 開始時仍須核對現行實作。來源見附錄 A。 + +| ID | 基準版本發現 | 修改方向 | +|---|---|---| +| F01 | `tools.rs::pack_observation()` 在 vision + 非 Identical 時回傳 None，條件反轉 | 先補紅燈測試，修正截圖交付；區分捕獲與交付狀態 [R1] | +| F02 | agent-facing `shell/read_file/write_file/list_files` 透過可見終端機與截圖 | 改接真正原生工具；GUI 終端機保留給 TUI／協作 [R1] | +| F03 | `SandboxProvider` **已存在** `execute/list_files/read_file/write_file`；Supervisor 已有容器內 exec 與檔案路徑 | 優先重用與強化，不必另造平行 sandbox 架構 [R2–R4] | +| F04 | `runs.rs` 將 email 歸 browser-first，MCP 另列，優先級重疊 | capability-aware routing；prompt 只是配套 [R5] | +| F05 | DOM browser 與 saved login 被 `vision_guard()` 一併限制 | 分 semantic、visual 與 credential capability [R1] | +| F06 | browser 宣稱 CSS selector／waitMs，但 adapter 能力與使用方式不一致 | 修契約、型別化錯誤、fixture；不假設 driver 自動實作 [R6] | +| F07 | `policy.rs` 重複 wait/shell 可刷新進度，災難上限不能當短任務預算 | 驗證 milestone、時間預算、有限恢復 [R7] | +| F08 | API 的 `mcp.rs` 啟動 stdio 子程序；hub 的全域鎖跨遠端 await | MCP runtime 搬進 Computer，授權隔離、局部鎖 [R8] | +| F09 | `crates/sandbox/src/docker.rs` 部分檔案通道以文字 JSON／lossy conversion 處理 bytes | 補 byte-safe 傳輸、大小限制、錯誤狀態與雜湊 [R3] | +| F10 | `lazyboy-screen` 已有 slot→display／VNC／websockify 對應、per-display Cua／DBus、獨立 profile；x11vnc 使用 `-noxdamage` | 保留既有多 display 語意；以可切換 Xvnc backend 測試，不把 Team 視為單一桌面 [R9] | +| F11 | `lazyboy-screen` 已用 `xfwm4 --compositor=off`；ensure_slot 會串行啟動桌面／VNC／Cua／終端機等 | 關 compositor 是現況，不是尚未做的新提速；按需拆 readiness 與修啟動 dependencies [R9] | + +特別注意：F03 代表原生執行「有底層基礎」，不代表既有實作已完整提供取消、持續 session、原子寫入、byte-safe 傳輸、授權隔離或任務稽核。不可只把 `shell()` 改呼叫 `execute()` 就宣布整個計劃完成。 + +### 2.1 檔案落點 + +| 現有位置 | 主要修改 | +|---|---| +| `crates/contracts/src/` | tool capability、operation、job、artifact、typed error 等可序列化契約 | +| `crates/control/src/sandbox.rs` | 重用／擴充原生方法、job lifecycle、串流及限制 | +| `crates/controld/src/` | Computer 內 Tool Runner 的路由、job 管理、檔案、browser adapter、local MCP／connectors | +| `crates/sandbox/src/docker.rs` | Rust API 到指定 Computer 的 transport；不得對錯誤回傳空成功 | +| `crates/supervisor/src/` | provisioning、Computer 身分綁定、受控 transport、資源／lifecycle；不執行 agent 任意主機程式 | +| `crates/api/src/tools.rs` | agent-facing schema、具 grant 的 dispatch、structured outcome；拆出子模組避免繼續膨脹 | +| `crates/api/src/runs.rs` | tool discovery、執行路由、checkpoint、結果與驗證、operation ID | +| `crates/harness/src/policy.rs` | milestone／預算／loop guard 與 typed recovery | +| `crates/api/src/mcp.rs` | control-plane 設定與 runtime directory；不再在 API spawn agent MCP | +| `crates/api/src/monitor.rs`、`apps/web/src/run-monitor.tsx` | activity timeline、耗時分解、job／artifact 狀態 | +| `crates/api/src/computer.rs`、`vault.rs`、`workspace.rs`、`attachments.rs` | 雙模式 Computer assignment／membership、scope、safe streaming、credential broker 整合 | +| `image/computer/`、Compose、`migrations/`、`tests/` | runtime 啟動、版本固定、漸進遷移、測試與部署 | + +新 module 名稱可依現有慣例調整；不得在 Rust 核心外再加一個完整 Python agent orchestrator。必要的 Node browser worker 可以是受控執行器，不是第二套總指揮。 + +--- + +## 3. 目標架構與責任分離 + +```text +LazyBoy API + Rust Harness [控制面] + ├─ 任務／模型／工具路由／grants／驗證 + ├─ Computer assignments + memberships + └─ PostgreSQL：操作紀錄、經脫敏 metadata、設定 + ↓ 受控 transport +Supervisor / SandboxProvider [管理面：生命週期、資源限制、身份驗證] + ┌──────────┴────────────────┐ + ▼ ▼ +Team Computer T [共用] Dedicated Computer P [私人] + ├─ agent A context └─ agent C context + │ ├─ bots/A/ ├─ 私人持久 home/workspace + │ └─ display/profile A └─ 私人 display/profile C + ├─ agent B context + │ ├─ bots/B/ + │ └─ display/profile B + ├─ shared/ [成員依 grant 存取] + └─ 版本化工具套件 [可共用程式，不隱含共用帳號] + │ │ + └─ 各 Computer 內的 Runner／jobs／fs／MCP／connectors／Cua + 工具資料、下載、暫存、完整輸出與執行產物留在該 Computer +``` + +### 3.1 執行位置 + +擴充既有 `controld`／SandboxProvider，不重建另一套總指揮。API 只管編排、授權與紀錄；Supervisor 可透過 Docker exec 啟動 Computer 內可信任 helper，不能把 LLM 指令放到自己的 shell。附件與產物允許受控串流，不在 API 主機暫存解析。remote SaaS 是獲准的目標服務，不是遠端檔案處理或遠端 code-interpreter fallback。 + +MCP／連接器的本地 client、下載、解壓、SDK 與檔案程序皆在目標 Computer。遠端 MCP 的伺服器程式仍在遠端，不能宣稱都在本機；只有使用者准許的遠端 SaaS 存取可啟用，會代處理本機 task files／代跑 shell 的遠端工具預設拒絕。 + +### 3.2 兩模式與資料約束 + +- `ComputerMode` 的既有 enum／儲存值先核對並保持相容；本文件用 shared/dedicated 表示產品語意，不要求隨意改資料庫字串。 +- `computer_id` 為邏輯電腦；provider_ref 可變。container 重建增加 `computer_generation`。display／browser 重啟另有 session generation，不需連帶使所有原生 job 失效。 +- 每個 agent 同時有一個 active assignment。**shared Computer 的 computer_id 可被多個 agent 參照**；不能替此欄位無條件加 UNIQUE。 +- dedicated 的排他性用對應 owner／部分約束／transaction enforce；shared 用明確 membership（相同核准 space/team）、display slot、workspace scope 與 grants。 +- 保留使用者既有選擇和建立 agent 時的共用／私人選項；新的 hybrid executor 不應改變 ComputerMode 的預設或現有 assignment。 +- provisioning／screen 啟動做 per-computer／per-slot single-flight；刪除 agent 只移除自己的 assignment 和資源。Team 仍有其他成員、job 或 viewer 時不得 stop/destroy 整台 Computer。 +- Computer idle 由全部成員的 jobs、MCP、screens、writers、outbox 共同判斷；不能因 A 閒置就凍結 B。 +- `runner_ready`、`browser_ready(profile)`、`desktop_ready(display)`、`viewer_ready(display)` 拆開。原生工作不等 XFCE／VNC；查看桌面也不能變成業務工作存活的必要條件。 + +### 3.3 共用的範圍，不等於什麼都共享 + +| 資源 | shared 模式預設 | dedicated 模式預設 | +|---|---|---| +| 容器與總 CPU／RAM | 同 Team Computer 共用、配額與公平排程 | 私人 Computer 的配額 | +| agent 工作目錄與 job 歸屬 | bots// 和 own jobs，tool API scope 分開 | 私人 workspace 和 own jobs | +| shared/ | 經 membership／read/write grant 分享 | 可保留本機 shared/ 名稱，但不自動跨 Computer | +| 瀏覽器、DISPLAY、DBus／AT-SPI session | 沿用各 agent 的 slot／profile，不能任意拿別人的 ref | 私人 slot／profile | +| 套件 bytes | 同版本可共用受保護唯讀 package store | 此 Computer 的 package store | +| 工具 instance／設定／OAuth | 依 agent／grant／account 分開；明確允許才共用 instance | 私人 agent scope | +| 記憶／run／操作紀錄 | agent 身份不合併；shared 檔案事件可依 membership 顯示 | agent scope | + +`shared/` 路徑需依既有 `resolve_bot_workspace_path`／scope 處理，不要直接重寫一套忽略舊路徑的 resolver。shared 寫入使用 canonical resource key 與版本衝突檢查，兩個 agent 看到的是同一份真正的檔案，不是各自副本卻叫 shared。 + +### 3.4 誠實的信任邊界 + +shared 是受信任團隊的協作 Computer。**同容器、同 UID、可執行任意 shell 時，資料夾 scope 與不同 DISPLAY 不構成對惡意程式的強隔離。** 不可宣稱 tool grant 能阻止所有自行寫腳本讀其他可讀檔案／X11 session 的行為。 + +保留 shared 的原生能力，但敏感 broker／Runner metadata 必須與 task user 分離保護；僅向批准的 connector instance 提供 scoped credential。對需要跨 agent 機密性的工作，使用 private Computer 或新增經測試的 per-agent OS user／工作沙盒；此選項不能變成刪除 shared 的理由。 + +同一共用主機安裝未受信任套件等於引入可執行程式，必須顯示受影響 Team／檔案／session 範圍；沒有可驗證的 sandbox 就不能宣稱只影響某一 agent。強化模式僅在對應 OS 行為真的測過後才能標為安全隔離。容器對宿主機的邊界仍依 Docker 權限／mount／network 實作 [E4]。 + +### 3.5 持久化與升級 + +兩模式都保留資料與 browser profile。user-space 工具使用版本化目錄與 lock manifest；OS 套件採 image／受控 recipe 重建，不承諾任意 apt 安裝跨 recreate 永久保留。 + +shared 和 private 都原地升級 schema／Runner，不做強制模式遷移。使用者日後明確要求換模式時，才啟動 dry-run、停止點、授權 copy、checksum、衝突處理與可回滾的 assignment 切換。 + +--- + +## 4. 契約：身份、工具結果與 operation ledger + +以下為**建議新增契約**，不是現有 API。欄位命名可調整，但語意不能消失。 + +### 4.1 可信任執行 context + +```json +{ + "space_id": "space-A", + "user_id": "user-A", + "bot_id": "bot-A", + "computer_id": "computer-A", + "computer_generation": 3, + "assignment_id": "assignment-A", + "computer_mode": "shared", + "display_session_id": "screen-A", + "workspace_scope_id": "scope-A-and-shared", + "run_id": "run-42", + "step_id": "step-3", + "operation_id": "op-17", + "attempt_id": "attempt-1", + "grant_id": "grant-9", + "control_epoch": 8, + "deadline_at": "2026-09-10T08:00:00Z" +} +``` + +這些身份由伺服器按登入 actor 與 run 綁定，**LLM 不得填寫或更換 computer_id、grant、account mapping 或 filesystem 根目錄**。工具參數只能是任務需要的業務參數。到 Runner 時再次驗證 token audience、agent、電腦 generation、grant、到期與 control epoch。 + +Supervisor shared token 不得進入 agent 的 env、home 或 prompt。每個 Computer 使用短效、限範圍認證；不要認為服務藏在 Docker 內網就等於有授權。 + +### 4.2 Capability descriptor + +```yaml +name: gmail.labels.apply_plan +version: "1" +execution_plane: computer +requires_vision: false +required_grants: [gmail.labels.write] +required_scopes: ["https://www.googleapis.com/auth/gmail.modify"] +side_effect: reversible_write +supports_batch: true +retry_mode: verify_before_retry +resource_key_template: "gmail:{account_ref}:mailbox" +result_schema: ToolResultV1 +``` + +capability 必須區分：installed、runtime_connected、authenticated、authorized、healthy。MCP annotations、tool description、模型自評或網頁內容都不是可信任授權來源。registry 僅向模型暴露此 run 真正能用的少量工具。 + +### 4.3 Structured ToolResult + +```json +{ + "schema_version": 1, + "operation_id": "op-17", + "status": "succeeded", + "effect": "confirmed", + "execution": { + "computer_id": "computer-A", + "computer_generation": 3, + "executor": "native_process", + "job_id": "job-7" + }, + "data": { + "stdout": "12 files processed\n", + "stderr": "", + "exit_code": 0, + "signal": null, + "truncated": false, + "output_artifact_id": null + }, + "artifacts": [], + "evidence": [{"kind": "process_exit", "exit_code": 0}], + "timing": {"queue_ms": 4, "execution_ms": 31}, + "error": null +} +``` + +- `status`：accepted、running、succeeded、failed、cancelled、timed_out、needs_auth、needs_approval、needs_human、policy_denied、unknown。 +- `effect`：none、confirmed、partial、unknown。`succeeded` 只表示該工具契約完成，**不自動代表整個 task 完成**。 +- `exit_code` 在未結束時為 null；因 signal 結束另記 signal。不得把 timeout 或空輸出視為 exit 0。 +- 所有結果共用 envelope；不能再讓任意錯誤文字看起來像正常 `text_outcome`。 +- wrapper 外層 HTTP 非 2xx、解析失敗、schema mismatch 必須保留錯誤；不能回傳空 file list／空 file contents 冒充成功。 + +### 4.4 Operation identity、重送與恢復 + +`operation_id` 代表一次具體意圖，由 harness 產生並在網路重送時保持相同。`attempt_id` 區分傳輸嘗試。不可只 hash 相同 args 作 dedupe：使用者可能真的要求合法重做同一動作。 + +Runner 在副作用前，先將 accepted intent 寫入受保護的 durable journal；其後記錄 started、結果與驗證。控制面先記錄 dispatch intent，再傳送；UI 訂閱中央事件。中央斷線時由本地 outbox 補送，透過唯一 event key 去重。journal 無法可靠寫入時，不開始新的有副作用工作。 + +同一 operation 重送：running 回原 job，finished 回既有結果；never blindly rerun。Runner／容器崩潰或外部 API 回應丟失時，狀態可為 unknown，交給 verifier 做 read-back。沒有第三方 idempotency 或可觀察後置條件時，不可宣稱 exactly-once；需人工判斷或停止。 + +同一 operation ID 搭配不同 payload hash、帳號或 grant 必須拒絕。舊 computer_generation／control_epoch 的動作不得在新容器或相應接管 scope 後繼續執行。shared 模式使用 computer、agent 與 display 的分層 epoch，不能 A 接管就無條件使 B 的獨立工作失效。 + +--- + +## 5. 原生命令、檔案與背景工作 + +### 5.1 最小工具集合 + +| 工具群（建議名稱） | 行為 | 截圖需求 | +|---|---|---| +| `exec.run` | argv 或明確 shell mode，短作業直接回結果，長作業回 job_id | 無 | +| `exec.status/output/cancel` | 查工作狀態、依 cursor 取輸出、確認取消 | 無 | +| `terminal.start/interact` | 真正需要 PTY／TUI／持續互動的程序 | 文字優先，必要時 GUI | +| `fs.list/stat/read/write/patch/move` | 原生 file API，byte-safe、有限讀取、原子變更 | 無 | +| `fs.search` | 在 Computer 搜尋；binary／巨大資料有界處理 | 無 | +| `artifact.list/export` | 回傳 Computer 內 artifact reference；下載串流 | 無 | +| `browser.*` | 同一 Chromium 的 DOM／語意操作 | 按需 | +| `computer.observe/act` | AT-SPI／視覺桌面操作 | 視覺動作必需 | +| `account.*`、`gmail.*`、`mcp.*` | 受控 account handle 與 per-agent grant | 通常無 | + +既有工具名稱可保留為 versioned alias，以免破壞 skills。alias 亦須經同一 policy、audit 與 Computer dispatch；不能留下可繞過的新舊兩套入口。 + +### 5.2 Exec 與 session 的正確語意 + +- 一般命令優先 argv array，避免重新插入 host shell；需要 pipeline 時顯式 `mode=shell`，只在 Computer 內解譯。 +- `cwd`、HOME、PATH、locale、環境變數與 session scope 由 Computer runtime 管理。不自動繼承 API 的雲端金鑰、DB URL、Supervisor token。 +- `exec.run` 的 stdout、stderr、exit status 來自實際程序，不從 screenshot 或文字 prompt 猜測。 +- 預設 subprocess 非互動且不維持前次 `export/cd`。需要持續環境時顯式使用 scoped session；不得讓 UI 宣稱持續，實際卻每次新 shell。 +- PTY 可能合併 stdout/stderr，需在 schema 註明。任意互動 shell 的 command completion 不可靠時回 `running/unknown`，不得用尾端 `$` 字元當完成證據。 +- 背景 job 由 Runner 擁有 lifecycle。HTTP request 結束不是 job 結束；回傳 job_id，保留 stdout cursor、deadline、PID/starttime／process group identity。 +- 使用 bounded streaming、背壓與磁碟 quota。模型預設只收到摘要／頭尾片段，完整輸出放 Computer artifact。 +- 取消必須處理 process group／subprocess tree：TERM → grace → KILL，確認狀態後才回 cancelled。不要誤殺整個 desktop、其他 job 或 PID 回收後的新程序。 +- request deadline 與 job runtime timeout 分離；達 runtime timeout 後依明確 policy 停止工作。transient network timeout 不等於工作已停止。 +- API restart 可找回同一 Runner job；container recreate 後原程序不再存在，要標記 interrupted／unknown 並驗證產物，不得假裝還在 running。 +- 對未知 arbitrary shell，`changes_state=unknown`，不能因命令開頭是 `cat` 就假設無副作用；shell、子程序、網路都可能改變狀態。 + +### 5.3 原生檔案與產物 + +檔案工具在 Computer 的 filesystem namespace 內執行，不能由 API 直接打開 volume 對應的主機路徑。 + +預設任務可寫 root 為 assignment 對應 workspace 與專屬 tmp；shared 模式再加入明確授權的 shared/ 子樹。其他 Computer 內路徑依 OS 權限與 grant 開放，並遵守第 3.4 節的 shared 信任邊界。此限制不要求 agent 只能處理某個固定副檔名，也不禁止在自己的電腦使用已安裝的系統工具。 + +必要實作： + +- Unicode、空白、引號、`$()`、反引號、換行檔名都須以資料而非 shell 程式碼處理。 +- byte-safe 讀寫；UTF-8 text API 與 binary API 分離。禁止 `from_utf8_lossy` 無聲毀損 binary。小 binary 可用受限 base64，大型檔案用串流；不要將整份內容放 argv。 +- `fs.read` 支援 line／byte range、最大回傳量與 continuation；二進位回 metadata/reference，不交模型亂解碼。 +- `fs.write/patch` 支援 expected version/hash、目標資源鎖、temporary file + same-filesystem atomic rename；需要 durability 的操作同步資料與目錄，讀回核對 hash。 +- 不存在檔案、權限不足、編碼錯誤、磁碟滿、大小超限必須各自回 typed error，不得變空字串成功。 +- 路徑驗證須處理 `..`、symlink、TOCTOU、mount boundary。不要只做字串 prefix 或先 canonicalize 再無鎖 open；使用 descriptor-relative／capability-based 安全打開方式，依平台選受支援方案並測試。 +- 對同一檔案的 read-modify-write，要用該次讀取的 version/hash 做 optimistic check；不同 run 並行修改會 conflict，不可覆蓋人類更新。 +- `artifact_id` server 產生並綁 creator_bot/computer、visibility_scope、相對路徑、media type、size、hash、來源 operation、retention；記錄生成時 generation 作 provenance。持久檔案在重建後經重新核對 hash／scope 可重新解析，不能永久因 generation 變動而失聯。下載 API 不接受任意 host path。 +- 附件入站、UI 下載、connector attachment、browser download 都必須走 Computer 內串流路徑，帶大小／quota／雜湊驗證。不得把 task bytes 放 API `/tmp` 再處理。 +- destructive file operation 額外依風險核准；一般使用者已明確要求且可逆的本機整理可在 run grant 內批次執行，不必每個檔案再問一次。 + +### 5.4 保留 GUI 能力但不再把 GUI 當萬用資料通道 + +移除「所有 shell/file output 都一定是 screenshot」的 system prompt。原生工具結果直接進模型；不為了視覺展示自動開 terminal、不重演指令、不額外截圖。 + +需要真正 TUI、GUI 終端機協作或使用者指定觀看時，才使用 `terminal.*`／Cua。相同 job output 可另做 UI viewer；不要求第一版將每個背景命令鏡像到桌面。 + +--- + +## 6. Capability-aware routing 與模型迴圈 + +### 6.1 工具選擇規則 + +在權限、資料位置與帳號已匹配的前提下，優先順序是： + +```text +服務提供的結構化 API（適用於該任務） + → Computer 內原生程式／檔案／已安裝 CLI + → 同一 browser 的 DOM／accessibility + → Cua 原生 accessibility + → 新鮮截圖 + 視覺座標 + → 需要人類協助 +``` + +這是依任務的偏好，不是要求每件事逐層嘗試。例如讀本機 CSV 直接 fs/exec，無需先找 SaaS API；登入後操作 canvas 可直接進視覺路徑。路由針對每個子任務重算，不能整個 run 一次選 browser 後鎖死。 + +**授權拒絕、帳號不符、缺少 capability scope 不是換另一工具繞過政策的理由。** 同一服務操作的 policy 必須套用到 connector、browser、shell HTTP 與 MCP 等路徑。無 API 但使用者確實授權該網站的 GUI 操作，可在明確 GUI grant 下執行；不可把 `POLICY_DENIED` 當成這種情況。 + +### 6.2 小任務不要多套一圈 planner + +- 簡單明確的 file／exec 任務，用既有模型回合或 deterministic intent-to-tool route，不再額外呼叫大模型做一份長計劃。 +- 多步任務才持久化 `TaskPlan{goal,steps,dependencies,success_checks,budget}`。 +- 每一步保存 input references、採用 route、已驗證 milestone、產物與 blockers。checkpoint 不只是聊天文字摘要。 +- 工具 discovery 依任務載入；不要把所有 agent 的所有 MCP schema 塞進每次 prompt。 +- schema cache 依 version／grant scope 快取；授權撤銷必須立即失效，不得因快取繼續使用。 +- 分類、格式轉換、批次檔案處理在一次受控工具工作中完成；不是每一列、每個字、每個 click 都呼叫模型。 +- 已知確定步驟可 executor-side macro，但需有限步數、每步 evidence／trace 與失敗即停；不是任意舊座標重播。 + +### 6.3 Typed errors 與處理規則 + +| Error | 行為 | +|---|---| +| `AUTH_REQUIRED`／`TOKEN_REVOKED` | 進 NEEDS_AUTH，保留 task；停止有副作用請求 | +| `POLICY_DENIED`／`SCOPE_DENIED`／`ACCOUNT_MISMATCH` | 明確拒絕此路徑；不改用 UI／shell 繞過 | +| `TARGET_STALE` | 重新 snapshot／定位一次；只對尚未執行的操作重建動作 | +| `TARGET_NOT_FOUND` | 有界重新觀察／定位；視覺 fallback 需新鮮截圖 | +| `SELECTOR_UNSUPPORTED` | 換成受支援語意 locator；不再重送同一不支援 selector | +| `TARGET_DISABLED` | 檢查可觀察的前置條件；僅有已知等待條件才 wait | +| `RATE_LIMITED`／暫時服務錯誤 | 本地 bounded backoff + jitter／Retry-After；不由 LLM 空轉 | +| `TIMEOUT_EFFECT_UNKNOWN` | 先 read-back；不能 blindly retry mutation | +| `COMPUTER_UNAVAILABLE` | 恢復原 Computer，驗證 generation；不換 host 或另一 bot | +| `CONFLICT` | 重新讀最新狀態，重規劃／重新核准差異 | +| `CAPABILITY_UNSUPPORTED` | 回報缺少套件／裝置／模型能力；不宣稱已操作 | +| `CANCELLED`／`STALE_CONTROL_EPOCH` | 不再派送；保留真實已發生效果 | + +### 6.4 Model 使用策略 + +初版保留現有模型供應商與選模 UI，不綁定特定商業模型。原生／DOM 工具允許通過契約測試的 text-only 模型；視覺座標需要具有經驗證 vision 能力的模型。 + +優先減少模型回合與多餘圖片，再測模型速度。可選的高難度模型升級最多有界觸發；不能讓不同 agent 彼此委派形成無限 nested harness。 + +若模型不支援所需能力，誠實回 `MODEL_CAPABILITY_MISMATCH`；不得把 `vision=true` 假裝打開。prompt、schema 與 provider request 三者須有整合測試。 + +--- + +## 7. Browser 與 Cua：保留、修正、再測另一 backend + +### 7.1 預設決策 + +先保留 Cua driver 與既有 Cua semantic browser，修好 screenshot、schema 與 stale refs。把 browser 操作抽成受控 `BrowserExecutor` 介面，供未來替換，不先承諾換 driver 一定更快。 + +可選對照方案固定為 **Computer 內的 Playwright worker，連接該 Computer 現有 Chromium**，避免第一版同時引入 agent-browser、Browser Use、Hermes 三套。需要比較其他方案時另外開 ADR／實驗，不能耽誤核心原生工具交付。 + +Playwright 官方支援 CDP attach 到既有 Chromium，但明確指出相較原生 Playwright protocol 功能完整度較低，必須對實際 browser launch flags、frames、downloads 與 input 做相容性測試 [E3]。若對照 backend 未通過 fixture，繼續用修好的 Cua，不擴大替換。 + +### 7.2 同一 browser/session 的規範 + +- 不要求使用者保持 VNC 開啟；但 agent 使用的 browser 必須屬於其 Computer，之後能在該桌面檢查及接管。 +- 優先 attach 現有 browser instance；不得另啟程序同時占用同一份 profile。保存 computer-local session identity 與明確 tab/frame ref。 +- Cua 與 Playwright **不得同時寫同一分頁／桌面**，必須共享 lease。driver 換手後清掉舊 references 並重新觀察。 +- CDP 與 Runner 不對主機公網發布，不交 LLM 任意 URL；由可信任 runtime 對指定 browser 建立連線。 +- 不把 headless／remote browser 當作無提示替代登入狀態的方法。 +- 登入、導航、frame 切換、人工接管與 browser restart 皆是 refs invalidation boundary。 + +### 7.3 修正截圖條件與交付狀態 + +基準版本 `pack_observation()` 修正為 vision 且畫面非 Identical 時才產生 image；純文字模型永不接收 image [R1]。 + +進一步拆開 `last_captured_frame`、`last_delivered_frame`、model/history identity。首次需要視覺、人工接管恢復、換 vision model、圖片歷史壓縮後，即使畫面 bytes 未變，也必須允許 force delivery。 + +只有圖片真正加入模型 request 後才更新 delivered 狀態。不可因背景監控抓過圖，就以為模型看過。沒有最新視覺依據時，不准猜座標。 + +### 7.4 語意與座標能力分開 + +DOM／accessibility 操作檢查有效語意 target、權限、origin、lease，不要求 vision。pixel-only 操作檢查 vision、最新 observation identity、viewport 尺寸與 control epoch。 + +browser locator 使用 tagged type，例如 `snapshot_ref`、`role_name`、`css`；只有 backend 真實支援的類型才暴露。不要在名稱相近時偷偷做模糊匹配。 + +`wait_until` 由 executor 觀察明確條件，設 deadline；避免所有工具固定 sleep 一秒。元素 disabled 的原因可能是缺欄位，不是時間到了就好。 + +可新增 bounded `browser.fill_form`：每欄重新定位並驗證，按鈕提交前驗證前置條件，提交後讀回確認。不可把五個會失效的數字 id 一次盲送。 + +### 7.5 底層可擴充，不預設全面操作保證 + +支援 normal click、type、scroll、drag、key、window focus、file upload/download、dialogs，以及 Cua 已提供的桌面能力。每一項提供 capability probe 與 fixture；不支援的 widget 明確回 unsupported。GPU、硬體或系統管理另外加受控 capability，不以一個 `run_anything_as_root` 代替設計。 + +--- + +## 8. Concurrency、Pause、Takeover 與 Cancel + +### 8.1 按真實資源上鎖，不按整台共用主機一律上鎖 + +| 資源 | 建議 lock key／規則 | +|---|---| +| GUI／native input | `(computer_id, display_session_id)` 的 writer lease；同 display 排他、不同 display 可並行 | +| Browser | profile/session 對應 writer lease；涉及 OS focus／clipboard／upload dialog 時一併取 display lease | +| 檔案 | canonical resource identity／重疊 subtree；shared/ 同檔必須命中同一 lock，配合 expected hash | +| 信箱與連接器副作用 | provider + canonical account/tenant/mailbox；跨 Computer／agent 的同帳號亦要協調 | +| MCP client | instance + grant + account 的有限 semaphore，不跨 network await 持有全域 hub lock | +| 套件安裝／升級 | Computer/package/version；同包原子切換，active instance pin 原版本 | +| container stop／recreate | Computer lifecycle barrier，等待所有成員 writer／job 安全停下 | + +同 display 的 Cua 與 DOM 不同時 mutation；不同 display 的 DOM 可以並行，不能因同 Computer 就全部串行。browser context 不可共享 mutable singleton。多資源依固定順序取鎖、設 deadline、避免 deadlock，長等待不占用無關 lease。 + +受控 file 工具與已驗證 scoped jobs 可按不重疊資源並行。**任意 shell 不能靠 LLM 宣稱的路徑或 read_only 取得並行保證**：有 OS sandbox 證據才降到 scope 鎖；無界且可能影響全 Computer 的腳本／系統安裝保守取得 Computer-wide lease。這是高權限工作例外，不是讓所有 shared 工作都上全域鎖。 + +初始併發為可調試驗：每 agent native jobs 2、service reads 4、每 display GUI writer 1，同時設定 per-Computer／provider 上限與公平排程。不可各 agent 自行放大到壓垮共用 Computer。 + +### 8.2 觀看、接管與暫停的不同 scope + +- Watching 不暫停，也不授予寫入；noVNC 客戶端 viewOnly 只是 UI，伺服器仍須依 lease 拒絕未授權鍵鼠／resize／clipboard。 +- **接管某 agent**：保守暫停該 agent 的新 mutation 與 jobs，取得它的 display；其他 agent 的獨立 display／無衝突資源可繼續。 +- **只接管某 display**：暫停所有正在寫該 display 的工作，不偷偷取得其他 display；涉及相同帳號／共享檔案時透過資源衝突顯示等待。 +- **暫停整台 Computer**：對所有成員、jobs、MCP 與 display 建立 barrier；UI 顯示受影響 agents。只有具 Computer 管理權限的人能做。 +- 人類若要在 shared Computer 任意改系統／全域檔案，應先選整台接管；不能保證其他 agent 不受無界人類 shell 影響。 + +### 8.3 Scope-aware pause barrier 與恢復 + +使用 `(computer_epoch, agent_epoch, display_epoch)` 的 scope-aware fencing 或等價機制；API 和 Runner 各驗證。對該 scope 停派新 mutation、取消未執行項，對 job 採已宣告且驗證過的 cancel／suspend policy。 + +已送出的 SaaS mutation 無法因斷線就撤銷；持續顯示 in-flight／unknown，read-back reconciliation 後才宣稱已完成接管。取消不是 rollback。不能 SIGSTOP 整台共用容器來模擬「暫停 A」。 + +恢復沿用原 run/step，重新檢查 assignment、membership、grants、generation、session 與未知效果；只做未完成步驟。共享檔案與 browser state 可能已變，重新讀回，禁止沿用舊 refs 或覆蓋 B／人類修改。 + +觀察、核准與安全 reconciliation 可走受限恢復路徑，不讓原模型繼續無界派工。 + +--- + +### 8.4 arbitrary code 與可觀察性的實際邊界 + +本計劃保證的是**經過 LazyBoy 工具閘道的每次呼叫、受控子動作及 job lifecycle 有紀錄**；不宣稱等同完整 kernel syscall 或每個應用內部操作稽核。任意 exec 內部腳本仍可能連網、啟動程序與寫檔。 + +因此 arbitrary exec 的 scope、網路與 OS 權限必須保守；需要逐 syscall／所有子程序網路行為的強制稽核時，另做 OS 級監控與測試。不能只改 prompt 就承諾「所有低階行為零漏記」。 + +--- + +## 9. 紀錄、Activity UI 與資料保護 + +### 9.1 必須記錄的事件 + +`run.started`、`step.routed`、`tool.accepted`、`tool.started`、`tool.progress`、`tool.completed`、`tool.failed`、`tool.cancel_requested`、`tool.cancelled`、`tool.effect_unknown`、`verification.completed`、`artifact.created`、`authorization.required`、`takeover.requested/acquired/released`、`run.completed/partial/failed`。 + +事件至少帶 server-bound identity、operation／attempt ID、tool/version、route、開始結束時間、resource scope、經脫敏 args summary、outcome、typed error、output reference、驗證證據與重試理由。 + +不保存模型隱藏推理；只記可稽核的 route rationale，例如「已有 Gmail labels scope，採批次 API」，不是整段思考鏈。 + +### 9.2 UI 顯示 + +在現有 run monitor 擴充，不重做整個前端： + +```text +14:20:10 原生檔案 讀取 orders.csv 成功 18 ms +14:20:10 原生命令 依規則整理 500 列 執行中 job-7 +14:20:11 結果驗證 輸出列數與檔案 hash 符合 通過 +14:20:12 Gmail 標籤修改預覽 80 封 等待核准 +``` + +以上僅為 UI 示例，不是效能實測。 + +每個項目可展開看脫敏輸入、輸出摘要、差異、執行電腦、job、產物與驗證；不能只顯示「我正在努力」。背景 job 可取消；artifact 可從該 Computer 串流下載。觀看紀錄不要求開桌面。 + +### 9.3 稽核安全與 durable outbox + +- 中央 ledger 存 metadata；原始 output／artifact 在 Computer。flush 有背壓、quota 和 retention；避免無上限 screenshot、base64 或郵件內容進 DB。 +- secret redaction 在寫入 trace、logger、exception、MCP debug、checkpoint、UI 之前執行。secret type 使用 opaque handle；不靠事後 regex 當唯一防線。 +- Runner 的 journal/outbox、broker keys 和管理憑證，必須置於 task shell 不可任意改寫的權限區域；中央 accepted intent 可協助查出本地結果缺口。 +- 限制任務使用者對 broker／Runner proc env、memory、socket 與管理檔案存取；不能只因非 root 就假設同 UID 程序完全隔離。 +- Browser 已登入狀態代表具有該帳號的存取權。若一般程式可以讀相同使用者的 browser profile／session 或控制 X11，就不能聲稱能抵抗惡意程式對所有 session 資料的窺探。部署要明確記錄此 threat model；更強隔離需 OS user／sandbox 與實測，不得僅用文案保證。 +- 不記錄原始密碼、OAuth token、Authorization header、secret query、剪貼簿 secret。密碼相關操作停用或遮蔽自動 screenshot／recording；不可先拍下再只遮 log 字串。 +- 檔案與郵件正文可能是敏感資料，依最小必要送模型，並提供 retention；secret canary 要掃所有正常與錯誤路徑。 + +--- + +## 10. MCP、帳號與憑證 + +### 10.1 MCP 在 Computer 執行與隔離 + +API 的 MCP 模組僅保存設定／授權 metadata 和 UI 狀態；stdio MCP 子程序與 remote MCP client 均由 Computer Runner 管理。兩模式都以 agent／grant 管理 MCP instance、env、working dir、account binding 與 capability set。shared 可重用唯讀套件 bytes；有狀態且含私人憑證的 instance 預設分開，明確授權且支援 request-level scope 的服務才共用 instance。安裝流程見第 20 節。 + +runtime key 至少包含 `(space,user,bot,computer,generation,server_instance)`，tool 名稱只是顯示名稱，不可只靠 slug 找工具而跨 agent 命中同名 server。 + +- 只讓授權工具進 registry，dispatch 再檢查一次；`definitions()` 不合併全域所有人的工具。 +- 在短臨界區取得 client handle，釋放 hub lock 再 await。必要 semaphore 只限制該 server，不堵住別的 agent。 +- 持續連線與 schema cache；restart／reconnect 清掉舊 generation 的 handles。 +- stdio 套件及版本固定，不在每次 tool call 執行不固定版本的 `npx -y latest`。首次安裝是獨立、有紀錄的準備工作。 +- raw MCP 結果要映射成共用 envelope，保留 `isError`、typed output、截斷與 resource references；不可只當 pretty JSON 文字認為成功。 +- 處理跨 agent server alias 碰撞、404、401、timeout、schema 變動。拒絕 connector 自己要求開放更多權限或修改控制面設定。 +- unknown／未受稽核的 MCP 可執行任意程式時，視同 unknown exec scope，不相信它宣稱 read-only。 + +### 10.2 Credential broker + +保留 Vault 的加密儲存與管理介面，但新增 per-agent／account grant。模型只取得 opaque handle。需要使用時，由可信任 broker 透過受保護 channel 交給 Computer 的受控 connector／欄位填入器；token 不暴露給 generic exec 或其他 MCP 的 env。經審查的第三方 MCP 若只支援環境變數認證，需明確批准其讀取該 token，以該 instance 的受保護 runtime 注入，記錄安全降級；無法阻止共用 task user 讀取時拒絕敏感 token 注入並回報原因，不假裝有 broker handle 即相容。 + +broker／trusted connector 應使用與 task shell 分離的 OS 權限（共享／私人皆適用），不放在任意工作程式可讀的 home、environment、debug output 或 task 可寫 journal。OAuth callback 可以由控制面接收與交換／安全保存授權憑證；之後真正的 Gmail 工具操作由 Computer connector 發出。不得為「全部在電腦內」而把長期 secret 塞給任意 shell。 + +原始密碼填入前檢查 HTTPS、精確核准 origin、frame、form target 與當前 control epoch；導航之後重新驗證。不要自動放寬成任意 Google 子網域。欄位辨識可用 input type／autocomplete／label／表單關聯，不只中文字串搜尋。 + +登入狀態要區分 `ORIGIN_MISMATCH`、`FIELD_NOT_FOUND`、`AMBIGUOUS_FIELD`、`AUTH_REJECTED`、`NEEDS_HUMAN`。密碼被拒不連續重送；2FA／Passkey／CAPTCHA 進人工接管。此專案不實作驗證繞過服務。 + +### 10.3 授權與便利性 + +使用者明確要求的低風險工作可以取得 run-scope grant，不要每個讀檔、click 都重新核准。敏感外送、寄信、付款、刪除、改帳號設定或大批外部寫入要有清楚 scope 與 policy；可設定長期授權，但必須有範圍、到期、撤銷與事件紀錄。 + +核准綁 plan hash、account、目標資源、允許的變更與期限；修改內容變了必須重新核准。generic shell、browser 和 MCP 不得成為其他工具 policy 的繞過通道。 + +**重要實際限制：**對具任意本機程式、已登入 browser、直接網路權限的工作者，只做工具層 allowlist 無法證明所有出站行為都受限。需以 OS 使用者、broker 隔離、網路 egress／proxy policy 實際限制敏感服務憑證與目標；對無法強制的 generic browser／exec 高權限能力明確標示並要求額外授權，不能虛稱與 narrow Gmail tool 同等安全。 + +--- + +## 11. Gmail：第一個端到端業務驗收案例 + +### 11.1 範圍 + +Gmail 只是驗證新架構的第一個 connector，不將 router 寫成 Gmail 特判，其他服務應能使用相同 grants、operation、audit 與 verifier。 + +第一版只做使用者自訂 labels 的預覽、核准、套用與讀回；不自動寄信、不刪除、不封存、不改已讀、不改系統分類分頁／企業 Classification Labels。Gmail labels 與 Inbox categories、Google Workspace 的 Classification Labels 是不同範圍。 + +使用使用者 OAuth，不假設 browser cookie 等於 API token。優先實作必要範圍的授權，使用 `gmail.modify` 時在產品層限制具體操作；該 scope 不只是標籤修改，`gmail.labels` 也不能替代套用郵件標籤所需權限 [E1–E2]。不要要求個人 Gmail 使用 Workspace domain-wide delegation。 + +公開分發／多人部署前檢查 Google 對 restricted scope 與資料處理的要求；開發 fixture 不代表真實 OAuth 發布審核完成 [E2]。 + +### 11.2 正確流程 + +```text +確認 account 與 capability + → search 指定 query／時間範圍，完整處理 pagination + → 受限並行取得 headers；必要時才取得 snippet／body + → 確定規則先分類，模糊案例送模型分批輸出 label enum + → schema validation + 固定測試集校準 + 待確認類別 + → 保存不可變 label plan，顯示預覽 + → run grant 或使用者明確核准 plan hash + → 依相同 add/remove label 組合分組 + → connector 在 Computer 內 batchModify + → 逐筆 read-back 驗證，記錄 partial／unknown + → 回報真實完成數、未處理數、例外與可用撤銷範圍 +``` + +Google 官方 `messages.batchModify` 單次最多 1,000 個 message IDs，同一批使用同一組 add/remove label；成功回應為空，不帶每封郵件的完整結果，因此仍須另行讀回驗證 [E1]。 + +範例驗收採 100 封 fixture mail，不代表使用者授權處理整個真實信箱。生產查詢必須有明確範圍、結果上限、排除條件、pagination 與 preview。 + +### 11.3 權限、模型與準確度 + +郵件正文是**不可信任資料**。分類模型只回傳固定 schema 與允許 labels，不可直接 dispatch tools；正文要求轉寄／改系統規則／下載執行程式一律不構成授權。 + +不要只依 LLM 自報 confidence。建立含繁體中文、英文、帳單、廣告、工作與模糊內容的人工標註測試集；報告 precision、recall、coverage、abstention，避免把所有信放「待確認」就宣稱 100% 準確。 + +初始 target：自動套用類別 precision ≥ 95%，同時公布 coverage／recall；模型不達標時採 preview-only，而不是偷偷降低正確性門檻。此為驗收目標，不是目前已測結果。 + +### 11.4 Retry 與 Undo 的限制 + +每個 plan 包含 message IDs、account reference、原先 labels、requested delta、rule version、hash、到期與 grant。分批有 child operation ID，保存實際套用／驗證的狀態。 + +429／暫時錯誤採有界 backoff；回應丟失先 read-back，只執行仍缺少的核准 delta。token 撤銷停在 NEEDS_AUTH，不轉 GUI 硬闖。 + +撤銷只考慮本次實際新增／移除的 labels，不覆寫整個 label set。若使用者並行修改同一 label 或狀態／history 不清楚，標記 conflict，顯示預覽再核准；**沒有原子 compare-and-swap／明確來源資訊時，不能保證任意並行情況下完全無損的一鍵 undo**。 + +### 11.5 外部環境缺失 + +coding agent 使用本地 fake Gmail HTTP server 完成 pagination、batch、token revoked、rate limit、response lost、partial verification、concurrent change 與 prompt-injection tests。真實帳號測試單獨列為 opt-in integration，不要求密碼／token 貼到聊天，也不自行建立或刪除真實郵件。 + +--- + +## 12. Verifier、進度、有限恢復與預算 + +### 12.1 三層完成狀態 + +1. **Transport**：呼叫是否收到合法回應。 +2. **Operation effect**：程序退出、欄位值改變、檔案 hash 符合、label 實際存在。 +3. **Task success**：使用者要求的所有必要結果與限制是否達成。 + +只有第 3 層完成才回 run.completed。部分成果回 partial，明確列已完成／未完成／阻礙，不用最後一句自然語言假裝通過。 + +### 12.2 Evidence-based progress + +- file：輸入／輸出 hash、筆數、schema、測試結果。 +- browser：特定欄位、URL／目標記錄、確認頁中的對應值；只有頁面換網址不足以驗證交易成功。 +- Gmail／Outlook：已讀回 verified 的 message 數、仍待處理／待確認數；不能混用兩者批次／標籤模型。 +- build：實際 job status、exit code、test summary；仍在輸出 log 不等於任務持續前進。 +- wait／poll：heartbeat 可表示 worker 活著，但不刷新 milestone progress。 + +像畫面時鐘、游標閃爍或工具 args 換了，也不算任務進展。對沒有 deterministic verifier 的模糊視覺任務，可使用受限視覺驗證或人類確認，但標示證據較弱，不能當確定完成。 + +### 12.3 起始預算（新設定，可調整） + +```yaml +interactive: + max_llm_turns: 20 + max_no_progress_seconds: 60 + max_semantic_recovery_attempts: 2 + max_route_switches_per_step: 2 + wall_budget_seconds: 180 +batch: + bounded_chunk_size: 100 + max_semantic_recovery_attempts: 2 + progress_source: verified_items +long_job: + lifecycle_owner: runner + completion_source: job_event + runtime_deadline: required +``` + +這些值是合理起始實驗，不是已測最佳設定。正式 task budget 與保險上限分離。長工作依 task 明確 deadline，不用 180 秒把正常編譯截斷；human_wait 另計。interactive 沒進度時重新路由一次，仍卡住就清楚暫停，不能把上限調回四小時當修復。 + +watchdog 必須能在慢模型／工具 await 期間運作，不只在下一輪才檢查。等待已知條件採本地事件／bounded polling，不反覆付費請 LLM 決定再等一秒。 + +--- + +## 13. 效能與正確性量測 + +### 13.1 必要指標 + +- `run/step/operation/model/tool_version/backend/computer_generation/mode/assignment/display_session`。 +- cold／warm，queue、model TTFT／total、tool execute、observation、recovery、human wait。 +- model input/output tokens（provider 沒回則標 unavailable）、schema 大小、LLM turns、tool calls、retry／route switches。 +- screenshot count／bytes、artifact bytes、verified items／milestones、success／partial／failure rate。 +- native CPU／memory／disk、每 agent／Computer 並行數、MCP server latency、display capture 與 VNC viewer latency。 + +並行存在時，不可把所有 span duration 相加當 run wall time；同時報 wall time、exclusive spans 與 critical path，避免重複計算 tool 與其內部 observe 的時間。 + +### 13.2 Benchmark gates + +以下為**待實測驗收 target**，不是完成時間承諾。先固定 reference host、container quotas、CPU architecture、模型版本、網路條件及冷暖狀態，提交 baseline，再進行比較。 + +| Case | 正確性 gate | 效能／迴圈 gate | +|---|---|---| +| 1 MiB 本地讀寫／hash | byte-for-byte 正確；private 不跨 Computer，shared 按授權共享而非混用 | warm tool-only P95 target ≤ 500 ms；0 screenshot | +| 短命令 `printf`／非零退出 | 真實 stdout/stderr/status | warm tool-only P95 target ≤ 1,000 ms；0 screenshot | +| 500 列 CSV 整理 | deterministic expected output／schema／hash 通過 | 不逐列 LLM；同條件已驗證完成 median target ≤ baseline 50% | +| 本地五欄表單 | 每欄正確＋提交確認 | bounded macro；模型決策回合 target ≤ 3，不含登入 | +| Gmail 100 封 fixture | requested label delta 逐筆驗證、不做額外修改 | 0 GUI；label write call 按組合／batch，而非每封一次 | +| 兩 agent 同時工作 | 結果分離、沒有 credential 或 artifact 混用 | 慢 A MCP 不阻塞 B client／registry；非全域串行 | +| 長命令與 reconnect | 狀態可找回，取消不留未知子程序 | 無需 LLM 忙碌 polling；模型次數不隨等待秒數增長 | + +每一候選設定先固定種子／fixture 重跑至少 30 次；正式以 P95 當門檻前建議至少 100 次並附樣本數。少量樣本只作探索；有延遲或 rate limit 不隱藏。舊路徑若無法完成並驗證該任務，其相對提速列為 N/A，改報新路徑絕對耗時與成功率，不捏造 baseline 倍數。 + +只比較成功且驗證過的同一工作，**同時列所有任務成功率與失敗耗時**，避免只挑成功樣本或「更快失敗」造成假提速。未達 target 就提交 profiling、差距與修正項，不改 fixture 偷渡。 + +--- + +## 14. 持久化、設定與雙模式相容升級 + +### 14.1 資料模型 + +沿用既有表、ID 與 enum，migration 採 repository 下一號。不重命名既有 shared/team mode 就讓舊資料無法載入。 + +- Computers：mode／owner_team／generation／provider_ref／backend／readiness。 +- Assignments/memberships：每 bot 一個 active assignment；shared computer 可多 bot，dedicated 有條件排他約束。保存 workspace scope、display slot、profile id。 +- Grants：bot／team membership／Computer／account／operation／expiry；同 Computer 不等於同 grant。 +- Package installations：Computer/package/version/digest/source/trust/status。Binding：安裝包 → agent(s)／server instance／account／capabilities；套件與認證分表。 +- Jobs／operations／events：保留第 4 節與原有 schema，加入 assignment、display/resource scope 和版本 provenance。 +- Artifacts：creator bot、visibility、relative path、hash、source op；重建後核對，不讓永續檔案被 generation 永久孤立。 +- Display sessions：每 slot 的 display/backend/epoch、profile 和 lifecycle。MCP、package 與 desktop 各可單獨健康檢查和恢復。 + +### 14.2 設定 + +以下是新增設定的建議命名；必須實作 parser／validation／`.env.example`，不是當成目前已可用的環境變數： + +```text +LAZYBOY_EXECUTION_PROFILE=legacy|hybrid +LAZYBOY_DISPLAY_BACKEND=xvfb_x11vnc|tigervnc_xvnc +LAZYBOY_BROWSER_BACKEND=cua|playwright +LAZYBOY_TOOL_AUDIT_REQUIRED=true +LAZYBOY_NATIVE_JOB_CONCURRENCY=2 +LAZYBOY_SERVICE_READ_CONCURRENCY=4 +LAZYBOY_TOOL_INSTALL_ENABLED=true +LAZYBOY_TOOL_INSTALL_REQUIRE_APPROVAL=true +``` + +Computer mode 保留既有 UI/API 選擇與預設值；不加強制 dedicated 的 rollout。legacy/hybrid 是 executor 路徑版本，**不是 shared/private 的替代名稱**。 + +雙模式皆先 opt-in 測試，再 canary。不能用 feature flag 恢復截圖錯誤、secret 洩露或 host task fallback。 + +### 14.3 原地升級與可選模式切換 + +預設原地升級 shared 與 dedicated，保留 bot assignment、shared/、private workspace、登入、skills、排程和群聊。既有 Team 不標為待淘汰、migration_required，也不自動拆成多台私人電腦。 + +只對需要調整的 metadata／工作檔路徑做有測試的相容 migration，保留回滾資料。使用者明確要求換模式時才執行：dry-run → manifest → 停止相關 writers → 在來源 Computer 匯出／目標 Computer 匯入 → hash/permissions check → 原子切換 assignment → 恢復。共享 profile／credential 歸屬不明時列衝突，不複製給所有人。 + +共用 package 更新或整台 image recreate 必須先列出所有受影響 members/jobs；不要為單 agent 升級重啟別人的工作。移除一個 agent 不刪共享資料或共享安裝包；最後一個 reference 消失後仍需 retention／管理政策決定 GC。 + +--- + +## 15. 實作順序：可分段提交的 PR + +不做「一次替換全部底層」的大型 PR。核心正確性先落地；TigerVNC 和第二 browser adapter 都有獨立開關，不把它們的相容性問題混入原生工具改動。 + +| PR | 實作與主要落點 | 依賴 | Gate／產出 | +|---|---|---|---| +| PR-00 | 核對 HEAD／git status／AGENTS.md／工具鏈；建立 shared/private baseline、紅燈 fixtures | 無 | 記錄原有失敗、可重現截圖／工具契約問題；不覆蓋未提交修改 | +| PR-01 | 修 `tools.rs` 截圖與 semantic/vision guard、browser contracts、typed result | 00 | 新圖交付、text-only DOM、unsupported/stale/disabled 正確區分 | +| PR-02 | 保留兩模式；assignment/membership／Runner identity／per-display context／protected broker boundary | 01 | shared A/B 仍同 Computer、private C 獨立；不能偽造 scope；沒強制模式轉換 | +| PR-03 | operation ledger／Computer outbox／Activity UI／事件與秘密脫敏 | 02 | 斷線補送／去重、rejected/partial/unknown 可見；失去 audit 不開始 mutation | +| PR-04 | 原生 exec/files/jobs／range & binary／artifact streaming／scope-aware cancel | 03 | 兩模式都能 text-only CSV 工作、0 screenshot、真 exit code、取消子程序；第一個核心可交付階段 | +| PR-05 | 可安裝 Tool Manager、Computer 內 MCP／連接器、版本 pin／grants／OAuth broker／hot reload | 04 | 本機 sample MCP 實際安裝可用；共用套件不共享帳號；更新回滾不重啟整台 Computer | +| PR-06 | per-step capability router、少量 schemas、milestone verifier／watchdog／有界恢復 | 05 | wait 不算進度、無 planner 套娃、policy denied 不繞路、未知副作用先讀回 | +| PR-07 | Browser session／profile／display leases、form macro、人類接管与恢復 | 06 | 同 display 排他、不同 display 可並行；接管 A 不凍結無衝突 B；resume 不重做 | +| PR-08A | Gmail labels adapter 作業務基準；沿用第 11 節 | 07 | fixture 預覽／核准／apply／read-back／衝突測試；不操作主信箱 | +| PR-08B | **Outlook 外掛作擴充性驗收**；走第 20 節安裝契約與 Graph adapter | 05；業務 E2E 需 07 | 不改核心 dispatch 也能載入外掛；read/categories/分頁/batch/401/429/共用模式授權測試 | +| PR-09 | DisplayBackend 介面＋TigerVNC Xvnc 候選＋舊 backend rollback；第 19 節 | 02/03 可先開始；整合 gate 需 07 | Cua/AT-SPI/X11/中文/多 display/noVNC 相容；benchmark 不退步才考慮 default | +| PR-10 | 雙模式原地升級、完整測試／效能與部署文件／rollout | 08A/08B/09 | 第 1 節契約與第 16 節測試有證據；Outlook 真實帳號或 GUI 缺環境須明確 blocked | +| PR-11（可選） | Playwright worker 對照；Selkies／Xpra／Wayland 分別另開 ADR | 核心不需等待 | 一次只換一層、無證據不採用；不能為完成清單全部安裝 | + +每個 PR 可以拆小，但不可依賴尚不存在的安全邊界就先打開高權限工具。PR-04 的必要 job cancel／audit 不拖到 PR-07。PR-05 即使外部 OAuth 不可用，也必須能安裝並執行本機測試 MCP，證明「外部工具能加入」不是只有 UI。 + +### 15.1 Coding agent 執行方式 + +先建立進度檔，以 `NOT_STARTED / IMPLEMENTED / PARTIAL / BLOCKED_EXTERNAL / VERIFIED / DEFERRED` 表示真實狀態。先完成 PR-00/01 的程式與測試，再依 DAG 推進；不是再交一份更長的計劃。 + +逐項更新第 18 節的 O01–O48：採用／保留現況／實驗／延期／阻礙、實際 diff、測試命令與結果。標「已實作」不等於「已驗收」。有互斥候選時，只實作滿足目標的最小方案；不能同時把所有替代框架塞進主線。 + +--- + +## 16. 最小測試矩陣與 Definition of Done + +| Test ID | 場景 | 必須證明 | +|---|---|---| +| T01 | vision × Changed/Similar/Identical | image 交付真值表正確 | +| T02 | 首次畫面／換模型／壓縮歷史／takeover resume | 未變但模型沒看過的圖仍交付 | +| T03 | 純文字模型 DOM／native tool | 可用語意工具、不收到圖、不猜座標 | +| T04 | selector unsupported／stale／disabled | 正確錯誤、有限恢復、無盲點 | +| T05 | private A/B 同路徑與 shared A/B 相同相對路徑 | 私人互隔離；共用 own workspace 不誤取，shared/ 依授權真共享 | +| T06 | 偽造 bot/computer/grant/generation | API 及 Runner 都拒絕 | +| T07 | host sentinel／Docker socket／管理 API | task 不可讀取或操作 | +| T08 | `..`／symlink race／奇異 Unicode 檔名 | 不逃逸／不 command injection | +| T09 | binary 含無效 UTF-8 與 1 MiB 檔案 | byte-for-byte 正確 | +| T10 | 權限不足／磁碟滿／HTTP 403／解析失敗 | 不回空成功 | +| T11 | 同檔並寫、expected hash 不符 | conflict 而非覆蓋 | +| T12 | stdout/stderr／非零退出／signal | 真實程序狀態 | +| T13 | timeout／子程序／cancel 競爭 | 停止程序樹，誠實回 pending/unknown | +| T14 | API restart／Runner reconnect | 原 job 不重做、output cursor 可續 | +| T15 | container recreation | generation 失效，舊 job 不假裝存活 | +| T16 | 同 operation 重送／相同 ID 不同 args | 去重；不同 payload 被拒絕 | +| T17 | central DB/網路中斷 | durable outbox 最終補送，不重做副作用 | +| T18 | journal 不可寫／quota 用盡 | 不開始新的 untracked mutation | +| T19 | 同名 MCP／慢 server | namespacing 正確，沒有全域 await lock | +| T20 | token revoked／wrong account／grant expiry | 停止並回 NEEDS_AUTH／denied | +| T21 | secret canary 在成功及 exception 路徑 | prompt/log/checkpoint/UI 無 secret | +| T22 | website/email/MCP prompt injection | 不把外部內容當授權與工具指令 | +| T23 | 同一 page 的 Cua＋browser worker | mutation 排他，refs 不混用 | +| T24 | takeover 時 background job＋HTTP in flight | barrier 真實，顯示未確定效果 | +| T25 | screenshot clock／重複 wait | 不刷新 milestone、不無限 loop | +| T26 | fake Gmail pagination／batch limits | 沒漏信、label 組合分批正確 | +| T27 | Gmail response lost／429／partial verify | bounded retry、read-back、如實 partial | +| T28 | Gmail concurrent same-label change／undo | conflict-aware，不宣稱無損無條件撤銷 | +| T29 | 兩模式原地升級；使用者要求模式切換的 dry-run | shared 不被強制拆開，原資料／profile／排程保留 | +| T30 | noVNC 關閉／desktop 未啟動 | native 工作仍完成且有紀錄 | +| T31 | 兩 agent／多 run 壓力 | 可量測並行、資源受控、結果不交叉 | +| T32 | 分類測試集含模糊郵件 | precision/recall/coverage/abstention 全部報告 | +| T33 | arbitrary exec 讀 broker env/socket/journal | 保護區不可見，不靠 prompt 擋 | +| T34 | UI download/附件串流 | 不使用 API host 暫存任務檔案 | +| T35 | 取消後 resume | 已驗證操作不重播；grant/ref 重新檢查 | +| T36 | 所有 aliases／native/MCP/browser 入口 | 都經統一 policy、operation、audit | +| T37 | 建立／恢復 shared 與 private agent | 兩模式維持原選擇；shared 允許多 assignment，不誤套 UNIQUE | +| T38 | shared A/B own workspace 相同檔名 | 透過受控 fs/exec context 正確命中，不因 cwd singleton 混用 | +| T39 | shared/ 同一檔案協作與未授權會員 | 成員獲准則可見；同檔 CAS/conflict，未授權工具請求被拒 | +| T40 | Team 不同 DISPLAY 的兩個 browser writer | 可並行；profile、DBus、AT-SPI、Cua socket 與截圖不串台 | +| T41 | 同一 DISPLAY／profile 的 browser/Cua/人類 | 排他 writer 與 server-side control epoch，有界排隊 | +| T42 | 暫停／接管 shared agent A | 無衝突 B 的 native/job/display 不被凍結；A 不再派送 | +| T43 | 整台 Team pause／recreate | 所有 members/jobs 被追蹤；無假暫停／孤兒 writer | +| T44 | 兩 agent 同時首次 provision／ensure screen | single-flight、slot 配置唯一、只有一個合法 instance | +| T45 | 刪除 Team 的一位成員 | 其他人的 Computer／shared files／package／jobs 不被刪 | +| T46 | 本機測試 MCP 匯入＋實際安裝＋執行 | 安裝程序和 bytes 在 Computer；有版本／hash／audit | +| T47 | 共用安裝包＋不同 agent 帳號 grant | 只重用唯讀程式；未授權 agent 無 broker token／tool capability | +| T48 | 錯誤 digest／路徑逃逸／擴權 manifest | 啟用前拒絕，existing tool 不受影響，套件 hooks 無無界權限 | +| T49 | 安裝新版本時有 active jobs | active pin 舊版、新 run 用驗證新版；可 rollback、不全機 restart | +| T50 | 移除／撤銷一個 connector binding | 只停止相應授權；共用套件／其他 agent bindings 仍運作 | +| T51 | Outlook fake messages/categories adapter | 安裝後能 read/preview/apply/verify；未授權 send/delete 不可用 | +| T52 | Graph $batch HTTP 200 但子項 429/401/5xx | 逐子項結果／Retry-After／未知 read-back；不全批重做 | +| T53 | Outlook nextLink/deltaLink／token revoke | 完整分頁；delta 依 folder 綁定，失效需安全重同步／授權 | +| T54 | Outlook categories 並行修改／未支援 If-Match | 不假稱 CAS；偵測可見衝突、回報限制，不盲覆蓋 | +| T55 | TigerVNC Xvnc × Cua × XFCE fixture | 截圖／XTEST／AT-SPI／中文／鍵鼠／clipboard／視窗辨識通過 | +| T56 | X server restart／backend 切換 | 舊 refs、display epoch 失效；既有 GUI sessions 不宣稱無縫存活 | +| T57 | slot 0/1 的 DISPLAY 與 VNC port | 明確保留 :1→5900／:2→5901 對應，不依 Xvnc 預設推算 | +| T58 | noVNC CSS scaling／resize／高 DPI | 座標 transform／實際尺寸正確、文字可讀，不拿 viewer 有損畫面做驗證 | +| T59 | 未授權 VNC keyboard/resize/clipboard | 伺服器拒絕；不只靠客戶端 viewOnly；Xauthority 配置驗證 | +| T60 | 無人觀看、VNC proxy 未啟動 | native 與必要 browser 工作仍可做；不被 viewer readiness 阻塞 | +| T61 | 第三方 MCP 僅能接受 env token | 明確 trust review／保護 process env；不能保護就拒絕敏感注入 | +| T62 | shared 同 UID 無界 shell 的 threat model | 不把目錄 scope／tool annotations 宣稱強隔離；core secrets 不可讀 | +| T63 | 慢 agent／資源不足／renderer crash | 公平隊列／總配額；不因 A 閒置或 OOM 自動破壞 B 狀態 | +| T64 | Computer recreation 後套件與 artifact | manifest 可重建；persistent artifact 校驗後可取；不重用舊 jobs | + +測試層次：unit → fake Runner／fake model／fake SaaS → disposable real Computer integration → explicit opt-in external account/model E2E。fixture 真實 GUI 測試無 display 時標 blocked，不改成假 assertions。 + +第一輪先從 repository 查實際工具鏈與 scripts。可用時執行： + +```bash +cargo fmt --all -- --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace +``` + +前端依 `apps/web/package.json` 已存在 scripts 跑 typecheck/build/tests；不要宣稱不存在的 npm／Make target 已通過。需要的 `test-agent-computer`、`bench-agent-computer` 等新入口必須在此實作中真的加入。 + +### 16.1 全案完成的最低定義 + +- shared Computer 上 A/B 協作、private Computer 上 C 獨立，兩模式關閉桌面仍可做工具工作。 +- 檔案、exec、MCP、connector 在正確 Computer；不存在 host task fallback。 +- API／工具／UI 的成功、失敗、partial、unknown、cancel、needs_auth 語意一致。 +- Activity 看得到操作摘要、結果、耗時、電腦、job／artifact 與驗證。 +- 原生檔案／命令能由 text-only 模型完成且不產生截圖。 +- Cua 仍可處理必要桌面操作；語意和視覺工具的 boundary 正確。 +- Gmail 與 Outlook fixture 預覽→核准→套用→驗證完成；Outlook 走可安裝外掛契約；外部帳號測試誠實標示。 +- 核准、secret、timeout、並行、接管、重啟與撤銷都有測試，不以功能展示代替。 +- TigerVNC 以候選 backend 通過兩模式相容性／readiness／noVNC 測試；是否切為預設依結果，不宣稱保證提速。 +- 外部 MCP 安裝、授權、更新、撤銷、移除可在 UI/API 操作，active job 不被靜默換版本。 +- O01–O48 逐項有 disposition／理由／證據；P2/P3 可有合理延期，不要求全部換掉。 +- 實際 benchmark 附環境與成功率；沒有未測的速度倍數或「全網站皆可操作」宣稱。 + +--- + +## 17. Coding agent 的交付格式與停止點 + +每個 PR 更新 `docs/agent-computer-progress.md`： + +```markdown +## PR-XX：名稱 +狀態：IMPLEMENTED / PARTIAL / BLOCKED_EXTERNAL +基準 commit：... +修改檔案與設計差異：... +新增／修改測試：... +實際執行命令與結果：... +未執行測試及原因：... +效能結果與樣本數：... +安全／相容性／雙模式／migration 影響：... +對應 Oxx／Txx 與 disposition：... +回滾方式：... +下一個可執行步驟：... +``` + +不要把「已寫測試」與「已執行且通過」混為一談。不要透過刪測試、跳過 auth、放大 retry budget、強制 vision 或把未知效果標成功來完成 gate。 + +沒有外部憑證時完成內部必要範圍，精確列出 integration blocker；不向聊天索取密碼。沒有使用者授權，不對真實信箱做寫入、不刪除 legacy 資料、不正式部署或 force push。 + +需要決策但沒有資料時，選擇符合本文件不變條件的最小改動；只有不可逆資料遷移／真實帳號核准等真正需要使用者的步驟才暫停。不要因架構工作較大就只交另一份計劃。 + +--- + +## 18. 逐項改善清單：48 項，不等於 48 項全部替換 + +P0：正確性／產品邊界；P1：核心交付；P2：量測後採用；P3：可選／延後。以下是本案的待辦與候選，沒有實測提速倍數。每項在進度檔有 disposition、證據和回滾；已存在的優化要標「保留」，不要再次算成新成果。 + +### A. 原生執行與工作流程 + +| ID | 項目 | 優先級 | 具體改動 | 要改善的問題 | 驗收／限制 | +|---|---|---|---|---|---| +| O01 | 修正新圖交付 | P0／必做 | 修反轉條件，分 captured/delivered，接管／換模型強制交付 | 無圖硬做造成的錯誤與重試 | 真值表＋模型 request fixture；[R1] | +| O02 | 原生 exec | P1／必做 | 命令在綁定 Computer 執行，直接取 stdout/stderr/exit | 消除終端機打字與讀圖 | 兩模式 text-only 命令 0 screenshot | +| O03 | 原生 files | P1／必做 | 重用並強化 SandboxProvider，不透過 sed 畫面讀檔 | 減少回合／避免 binary 毀損 | range、Unicode、binary、shared 路徑；[R2–R4] | +| O04 | 持續 jobs 與 PTY | P1／必做 | 短命令直接返回，長作業由 Runner 管理、事件通知／真取消 | 不請 LLM 反覆看是否做完 | API 重啟可追 job；process-tree cancel | +| O05 | 批次程序／form macro | P1／必做 | 已知步驟在受控 executor 內完成，有 pre/post checks | 不逐列／逐 click 呼叫模型 | 每個子動作有 trace；失敗即停 | +| O06 | 套件／連線常駐 | P1／必做 | 暖 MCP instance、HTTP pooling、版本化 cache | 降低啟動與重複連線成本 | warm/cold 分測；不共享私人 env | +| O07 | 穩定 Runner transport | P2／量測 | 優先現有安全 transport；高 overhead 才改持續 UDS/HTTP 通道 | 減少每次 docker exec 的啟動成本 | 實測 RPC 分解；不能公開 root socket | +| O08 | readiness 分層 | P1／必做 | Runner、browser、desktop、viewer 各自 ready，按需啟動 | 讀 CSV 不先等整個桌面 | no viewer/no desktop 的 native fixture | + +### B. 模型與瀏覽器 + +| ID | 項目 | 優先級 | 具體改動 | 要改善的問題 | 驗收／限制 | +|---|---|---|---|---|---| +| O09 | 按能力路由 | P1／必做 | 原生／已授權 API 優先，DOM 次之，必要時視覺 | 避免 Gmail／Outlook 走慢 UI | route rationale＋權限拒絕不繞過 | +| O10 | 工具 schema 按需載入 | P1／必做 | 僅帶此 run 可用能力，版本／grant cache | 縮短 context、避免選錯工具 | revoke 後立即失效；記 schema tokens | +| O11 | 簡單任務不加 planner | P1／必做 | 用既有一輪決策，複雜任務才 TaskPlan | 避免多 agent／planner 套娃 | 記模型回合、TTFT、正確完成 | +| O12 | 條件等待 | P1／必做 | wait_until／job events 有 deadline，替代固定 sleep | 消除空等與毫無進度輪詢 | 未知 disabled 原因不無限等 | +| O13 | DOM snapshot 限量與增量 | P2／量測 | 聚焦相關區域，revision 與失效 refs 完整處理 | 減少重傳巨大 page text | 錯 revision 回 full；不能漏關鍵元素 | +| O14 | agent 圖片和 viewer 串流分離 | P1／必做 | 高可讀截圖按需取，不把 VNC frame 全塞模型 | 減少模型圖片成本且維持辨識 | 縮放映射／force image／小字 fixture | +| O15 | 既有 browser/session 穩定 attach | P1／必做 | profile/tab/frame 明確綁定，禁止亂開替代瀏覽器 | 減少重登入與 stale reference | 同 profile 不雙開；重啟正確失效 | +| O16 | 驗證、有限恢復與模型升級 | P1／必做 | 進度看 milestone；兩次同義錯誤換策略，難題才升級模型 | 停止二十分鐘空轉 | 未知副作用先讀回；無固定提速保證 | + +### C. 顯示與遠端桌面 + +| ID | 項目 | 優先級 | 具體改動 | 要改善的問題 | 驗收／限制 | +|---|---|---|---|---|---| +| O17 | Xvfb+x11vnc → TigerVNC Xvnc | P2／必做候選測試 | 一個 backend 同時提供 X server 與 VNC，保留 rollback | 可能減少顯示匯出層與維護負擔 | Cua／a11y／中文字／多 display／CPU/latency；[E5] | +| O18 | Xvfb+x0vncserver 備選 | P3／保留替代 | 只換 VNC 匯出層，不替換 X server | Xvnc 相容性卡住時較小改動 | 不能說已移除 Xvfb；[E6] | +| O19 | 檢查 -noxdamage | P2／對照 | 現有 x11vnc 關閉 XDamage；測啟用是否有重繪瑕疵 | 判斷掃描負擔能否降低 | 靜態／捲動／遮擋／影片測試；不可直接刪旗標；[R9] | +| O20 | noVNC/websockify 與畫質檔 | P2／調校 | 先保留；viewer FPS/壓縮可調，無 viewer 減少服務負載 | 改善觀看／頻寬，不假稱模型更快 | 文字清晰、輸入延遲、resize、proxy auth；[E7] | +| O21 | 精簡桌面而非先換 OS | P2／調校 | 保留 XFCE/a11y；已 compositor=off，評估不必要 autostart | 降低多 slot CPU/RAM | 不重做既有優化；不破壞 DBus/a11y；[R9] | +| O22 | Selkies 替代 viewer | P3／可選 | 有高幀率／音訊需求才測；目前有 WebSocket 與選用 WebRTC | 影音觀看可能較適合 | 硬體 encoder 實測、網路與前端成本；[E8] | +| O23 | Xpra 替代 viewer | P3／可選 | 需要單視窗發布／session forwarding 時測 | 另一路 remote app 體驗 | 非 noVNC drop-in；額外 client/輸入整合；[E9] | +| O24 | Wayland／全面換桌面 | P3／延後 | 保留 X11 driver 路徑，未有需求不全面搬遷 | 避免同時引入 capture/input 相容性變動 | 獨立 ADR/fixtures；不是目前速度主解法 | + +### D. 共用／私人與資源協調 + +| ID | 項目 | 優先級 | 具體改動 | 要改善的問題 | 驗收／限制 | +|---|---|---|---|---|---| +| O25 | 雙模式正式保留 | P0／必做 | shared 多 bot→同 Computer；private 排他；不強制遷移 | 符合產品且避免破壞資料 | assignment/member/slot/重建測試 | +| O26 | 按 display／檔案／account 鎖 | P1／必做 | 同資源排他、不同 display 與受控 job 並行 | 避免共用主機整機串行 | Cua/DOM/human 同鎖；canonical shared key | +| O27 | scope-aware pause | P1／必做 | 接管 A、接管 display、暫停整機分開 | B 不被 A 的無關操作凍結 | 真 barrier／fencing；HTTP unknown 誠實顯示 | +| O28 | 共享 lifecycle／single-flight | P1／必做 | 啟動、刪除、idle reaper 看全部 member/jobs | 避免重複建立與誤停他人工作 | member refcount、job/viewer 活性聚合 | +| O29 | 程式共享與帳號授權分離 | P0／必做 | 共享唯讀 package，per-agent runtime/bindings/grants | 省重複安裝、不共享私人帳號 | 未知程式不能靠 annotations 保證安全 | +| O30 | 公平排程與總配額 | P1／必做 | 每 agent／Computer 的 CPU/RAM/jobs/requests 有界 | 抑制單 agent 拖慢所有人 | 壓力/OOM/慢 MCP；不無界並行 | +| O31 | Chromium /dev/shm 與程序回收 | P2／量測 | 調 per-Computer shm、init/reaper、renderer 健康 | 減少可避免的崩潰與孤兒程序 | 不能盲開 ipc=host/SYS_ADMIN/no-sandbox；[E10] | +| O32 | X11/DBus/profile 權限與 session | P1／必做 | per-slot identity，評估 Xauthority，系統安裝有管理鎖 | 避免串台與共用 session 風險 | 同 UID 非強隔離；Xvnc :1 port 對應測試 | + +### E. 可安裝外部工具／Outlook + +| ID | 項目 | 優先級 | 具體改動 | 要改善的問題 | 驗收／限制 | +|---|---|---|---|---|---| +| O33 | Tool Manager 安裝入口 | P1／必做 | catalog、manifest／受控 package URL、本地上傳三種入口 | 新增工具不改核心 dispatch | 實際 sample MCP 安裝，不只是設定畫面 | +| O34 | 來源版本與完整性 | P1／必做 | pin digest/version，審核 hooks，安全解壓與 quota | 更新可重現、防無界安裝 | 拒絕未審核 URL、路徑逃逸、錯 digest | +| O35 | 安裝／啟動／認證／授權分開 | P1／必做 | 每階段顯示狀態、failure reason 和 scope | 不再「裝好了卻不能用」 | installed ≠ connected ≠ authenticated ≠ authorized | +| O36 | 熱啟用與 registry 更新 | P1／必做 | health/schema 驗證，原子登錄能力；grant 撤銷失效 | 不為加工具重啟整個 LazyBoy | 正在執行 run 的能力版本與安全點更新 | +| O37 | 版本更新／回滾／移除 | P1／必做 | 新版本旁置，active job pin 舊版，bindings/reference GC | 共用主機其他 agent 不受破壞 | 新增權限重新核准；共享套件引用仍存活 | +| O38 | OAuth／token broker | P1／必做 | Microsoft/Google 正規授權，scoped token、refresh single-flight | 減少重登入與憑證混用 | MFA/admin policy 正常接管；secret canary；[E11] | +| O39 | Outlook Graph 外掛 | P1／必做驗收 | 讀信→categories preview/apply/verify；寄信另授權 | 驗證架構不只 Gmail 特判 | 個人/工作帳號對應 scopes；[E12–E14] | +| O40 | Outlook 批次／增量同步 | P1/P2 | Graph 20 子請求／逐項狀態；需要時 folder delta | 減少網路回合及每次全信箱掃描 | 429/Retry-After/分頁/ImmutableId；[E15–E18] | + +### F. 正確性、紀錄與可維護性 + +| ID | 項目 | 優先級 | 具體改動 | 要改善的問題 | 驗收／限制 | +|---|---|---|---|---|---| +| O41 | operation journal/outbox | P0/P1 | 副作用前記 intent；斷線 durable 補送與去重 | 避免掉紀錄與重做工作 | journal 故障不開始 mutation | +| O42 | 活動紀錄串流而非桌面演出 | P1／必做 | 工具與 job 的結構化事件、實際耗時／結果 | 人可追查、不逼每步打字 | noVNC 關閉仍可看進度 | +| O43 | artifact 存 Computer＋脫敏 | P1／必做 | 全文输出在 Computer，API 存摘要/reference，限量保留 | 避免 DB/context 被圖片/日誌塞滿 | secret/大檔/retention/串流授權測試 | +| O44 | Task 驗證與郵件分類品質 | P1／必做 | API 200 不等於任務完成；測 precision/coverage/abstention | 速度與正確性一起看 | read-back／真實 fixture、失敗也計入 | +| O45 | 衝突與冪等／未知效果 | P1／必做 | 同 operation 不重做；same-file CAS；未知 API 先回讀 | 降低重送／覆蓋／重複外部副作用 | 不假稱通用 exactly-once／無損 undo | +| O46 | 部件健康與版本相容矩陣 | P1/P2 | Runner/display/Cua/browser/plugin 個別 probe/recover | 失敗不用重啟整台 Computer | 不影響其他成員；pin 上游版本 | +| O47 | 已驗證 skill／流程重用 | P2／後續 | 將常用流程保存語意步驟與 checks，環境變更可失效 | 降低每次從零探索 | 不保存密碼／舊座標；profile/版本變更重驗 | +| O48 | 端到端＋viewer 雙 benchmark | P1／必做 | 分模型、工具、snapshot、queue、viewer，固定環境兩模式 | 找真瓶頸，不憑工具名字換底層 | 成功率/median/P95/CPU/RAM/可讀性；一項一改 | + + +--- + +## 19. TigerVNC 與桌面後端：實作規格 + +### 19.1 明確的兩條路 + +**候選 A：Xvnc 取代 Xvfb+x11vnc。** TigerVNC 的 Xvnc 同時提供 X server 與 VNC server [E5]。此案希望減少虛擬顯示再由另一程序匯出的層次；收益是待測假設，不保證 agent 推理更快。 + +**候選 B：保留 Xvfb，只用 x0vncserver 取代 x11vnc。** x0vncserver 是匯出既有 X display，不建立新的 display [E6]。它不是「Xvfb 一起換掉」。只有候選 A 遇相容性阻礙時再測 B，避免一次三套都當主線。 + +noVNC 是 viewer；websockify 是 WebSocket 到 VNC 的橋接。換 Xvnc 不代表可以直接刪掉這兩層；第一版保留既有 authenticated screen proxy 與 noVNC UI [E7]。 + +### 19.2 原始碼落點 + +- `image/computer/Dockerfile`：加入發行版可用且 pin 的 TigerVNC 套件，保留 baseline fallback。安裝後記錄實際 binary 名稱／版本（不同套件可能叫 Xvnc 或 Xtigervnc），不是假設名字一定相同。 +- `image/computer/lazyboy-screen`：將 start_xvfb/start_vnc 抽成 DisplayBackend，沿用 ensure_slot 和 per-slot identity／locking。保留顏色、中文、DBus／AT-SPI、Cua socket、profile、日誌。 +- `image/computer/start.sh`／`crates/controld`：分 runner/display/viewer readiness；按需啟動而非工具讀檔前全部 boot。 +- `crates/control/src/screen.rs`／supervisor：保留 slot→display/view port 契約，增加 backend/session epoch 與 capabilities。实际檔案／函式存在與否先核對，不盲插模板。 +- `screen_proxy.rs`／noVNC UI：保留 server-side auth／membership/control lease、斷線恢復與鍵鼠權限；不要只有 viewOnly 前端旗標。 + +### 19.3 最容易踩錯的地方 + +1. 基準 slot 0 是 DISPLAY `:1`、RFB `5900`、web `6080`；slot 1 是 `:2`、`5901`、`6081`。Xvnc 常用預設是 5900+display-number，**不得拿預設值默默改掉既有對應**；明確設定 rfbport，寫契約測試 [R9][E5]。 +2. 每 slot 的 X11、Cua、DBus、AT-SPI、XDG_RUNTIME_DIR 與 browser profile 一起綁定，不用全域可變 DISPLAY。不能以「共用主機」為由退回所有 agent 同一滑鼠。 +3. Cua 的 X11 截圖／輸入、XTEST、RANDR、clipboard、視窗 enumerations 需要在選定套件實際 probe；AT-SPI 是桌面／應用和 bus 的能力，不是 Xvnc 自動提供。 +4. 先固定現有 1280×800／24-bit 並保持一致 DPI／scale，避免把 backend 差異和解析度差異混在 benchmark。viewer 可 CSS fit，但座標需轉成實際 display pixels；resize 成功後 invalidate refs。 +5. human viewer 的有損壓縮／FPS 調整不能降低 agent 驗證圖的可讀性。模型 observation 仍由明確的高可讀 capture 路徑取得。 +6. X11/RFB 僅在受控邊界可達；不裸露公網的 5900/6080/CDP。評估移除 `-ac` 需要同步實作 Xauthority/client credentials，不能只删旗標導致 Cua 壞掉。 +7. 現有 `-noxdamage` 可能是舊相容性 workaround，只有在重繪 fixture 過關後才能改。`xfwm4 --compositor=off` 已經存在，不能重複列提速成果 [R9]。 +8. Xvnc 是 X server，殺掉它就會中斷使用它的 X clients；無 viewer 時可以停／降頻 viewer bridge，但不能因此殺死仍在工作的 display。stop/recreate/backend 切換需受控停止點，**不是無縫熱切 X server**。 +9. 清理 stale pidfile／X lock／Chromium SingletonLock 前先確認對應程序與 profile ownership；不以 pgrep 字串近似或一律 rm lock 把活躍 session 搞壞。 +10. 大版本更換仍可能影響 Unicode clipboard、快捷鍵、drag、輸入焦點；測手機／桌面 reconnect、human takeover、雙 viewer read-only 與 writer。 + +### 19.4 對照測試與採用門檻 + +固定模型、任務、容器資源、Cua/Chromium/noVNC 版本、screen size 與 LAN/WAN 條件。至少含空桌面、靜態網頁、捲動、輸入表單、對話框、影片、兩個 Team displays 同時操作。 + +分開量測： + +- **Agent path**：model time、工具時間、Cua snapshot latency、action→verified effect、完成率、重試數。 +- **Viewer path**：input-to-present latency、frame update interval、重繪完整性、小字可讀性、reconnect、網路 bytes。 +- **資源**：每 display／整台 Computer CPU、RSS、程序數；有 viewer 與無 viewer 各測。 + +硬 gate 是既有 GUI／輸入／a11y 正確性不退步，不能用更模糊畫面換好看的 FPS。採用預設前提出量測理由；只有 viewer 變順時，報告 viewer 改善，不宣稱整個 agent 任務提速。無 GUI 環境則標 BLOCKED_EXTERNAL 並保留 baseline default。 + +### 19.5 不建議同時改的東西 + +Selkies、Xpra、Wayland、換整個 Linux 發行版是不同級別的變更。Selkies 目前官方文件列 WebSocket 為預設、WebRTC 為選用，不能只照舊 README 說一定需要 WebRTC/TURN；硬體編碼可用與否也要 probe [E8]。Xpra 有自己的 HTML5/session forwarding 架構，不是改一個 VNC executable 就完成 [E9]。沒有影音／單應用發布需求時，先保留 noVNC。 + +--- + +## 20. 可安裝外部工具：Tool Manager 與擴充契約 + +### 20.1 三種形式與真正的執行位置 + +| 形式 | 實作方式 | 限制 | +|---|---|---| +| stdio MCP | Computer 內啟動經審查的 Node/Python/Rust executable，generic MCP adapter 載入 schemas | MCP package 可執行程式；固定版本與 runtime grants；API 不 spawn | +| CLI／自製本機工具 | 套件裝在 Computer，manifest 把 argv/stdin/stdout/schema 接到受控 executor | 不能直接把任意 shell 字串當受信任 metadata；binary 必須在 scope 中 | +| HTTP MCP／SaaS connector | client 在 Computer；呼叫批准的外部服務，artifact/download 落在 Computer | remote MCP server 仍在遠端；不能外包本機任務檔案處理／code execution | + +普通 REST SDK 不會自動變成 agent tool；需要一層 adapter／MCP server／manifest wrapper。完成通用載入後，新 MCP 不應每次修改 Rust 核心 dispatch；新 provider 的特殊 API 語意仍可能需要寫 adapter。 + +先支援手動來源與小型內建 catalog，不以「先蓋完整 marketplace」阻擋交付。來源可為受控 manifest URL、上傳套件、已知 catalog；下載與解包在指定 Computer。公開 registry 只協助 discovery，不等於套件已審核安全。 + +### 20.2 必須分開的四個層次 + +```text +Package：程式、來源、版本、hash、相依 +Installation：安裝在哪個 Computer、狀態、可重建資料 +Binding：哪個 agent 可以看見／呼叫哪些 capabilities +Connection：哪個真實帳號、OAuth／secret grant、允許操作 +``` + +shared Computer 可以一份唯讀 package 供 A/B 用；A 的 Outlook grant 不會因此授給 B。相同套件可有兩個私人 server instances／設定。shared/ 是否可讀寫另外核准，不因已安裝就自動全開。 + +### 20.3 安裝與啟用流程 + +1. **選來源與目標**：顯示 shared/private Computer、受影響 agent、package publisher、版本、雜湊、平台／CPU 架構需求。 +2. **驗 manifest**：schema version、entrypoint、dependency lock、network/service domains、filesystem scope、工具名衝突、requested permissions。 +3. **核准安裝**：套件程式執行與資料存取風險分開。不能由模型、網頁、郵件或外掛自己的提示自動授權。 +4. **Computer 內 staging install**：versioned 目錄、受限網路／CPU／磁碟、safe archive extraction，防 traversal/symlink/hardlink 逃逸和解壓炸彈；package install hooks 同樣受控。 +5. **健康／契約測試**：啟動、tools/list、schema validation、read-only mock call、version/compatibility。不能在 healthcheck 時默默寄信或修改真實帳號。 +6. **登錄 registry**：安裝及 schema 成功後原子啟用 binding。顯示 INSTALLED／CONNECTED／AUTH_REQUIRED／READY／DEGRADED，不用單一「成功」。 +7. **使用者 OAuth**：核對 account/tenant/scopes；同意後 token 保存在 broker 保護區，模型只見 opaque handle；新增寫入權限再核准。 +8. **實際測試與紀錄**：每次 tool call 綁 agent/computer/package version/account，透過統一 policy/audit/result；外掛失敗不讓整個 harness 崩潰。 + +工具入站權限、網路 egress 與 OS sandbox 由 host/runtime 強制；不能信 MCP annotations 的 `readOnlyHint` 就斷定沒有副作用 [E19–E20]。支援標準 MCP 不等於支援任意 auth 形態：HTTP OAuth discovery/PKCE 與 stdio 的本機憑證配置是不同流程；不把標示 SSE 的 server 當成一定可用 Streamable HTTP，按 negotiated transport 實测 [E20]。 + +### 20.4 狀態、更新與安全點 + +安裝狀態與連線狀態分開：`PENDING_APPROVAL → INSTALLING → INSTALLED`；instance 為 `STARTING → CONNECTED → AUTH_REQUIRED/AUTHENTICATED → READY/DEGRADED/DISABLED`。可用性是 installed + compatible + healthy + grant + account，不只 process alive。 + +更新採 staging → contract tests → permission diff → atomic activation。當前 operation 和長 job pin `package_digest + schema_version + connection`；新 run 才用新版，或在明確安全點重取 registry。外掛的 scope 變大或 tool schema 有 breaking change 要重新核准，不能默默延用舊 grant。 + +移除先撤銷 binding、阻止新呼叫、drain/cancel 對應 instance；其他 agent 使用的 package 不刪。若外掛持有 token，僅停止 process 不代表遠端 token 已撤銷；提供 provider logout/revoke 流程和殘留風險說明。 + +一般 user-space plugin 安裝／更新不重建 API image、不重啟整台 Computer。系統 package／顯示 driver／需 root 的改動另外走管理核准和 maintenance barrier，shared 時顯示全部受影響成員。 + +### 20.5 範例 manifest（設計範例，不是已發佈 npm 套件） + +以下示範的是 **要由 coding agent 實作／審查的本機 Outlook adapter**。空的 artifact/digest 必須拒絕安裝，不能填想像中的套件名稱或把此範例當已可直接安裝： + +```yaml +manifest_version: 1 +id: lazyboy.example.outlook +version: 0.0.0-example +source: + kind: reviewed_local_artifact + artifact_ref: null # 實作完成後填實際來源 + sha256: null # 安裝器不可接受空 hash +runtime: + kind: stdio_mcp + execution_location: assigned_computer + entrypoint: [./outlook-adapter] + state_scope: agent_binding +installation: + supported_modes: [shared, dedicated] + share_immutable_package: true +permissions: + filesystem: [own_workspace] + shared_paths: [] + network_services: [microsoft_graph, microsoft_identity] + needs_host_access: false + needs_root: false +connection: + provider: microsoft_graph + credential_delivery: broker + grant_scope: agent +capabilities: + - outlook.messages.list + - outlook.messages.read + - outlook.categories.preview + - outlook.categories.apply_plan + - outlook.categories.verify_plan +``` + +名稱、路徑與 manifest API 都是新設計；Graph 的官方 API／OAuth 不等於存在官方同名 MCP package。第三方工具若僅接受 env token，按第 10.2 節顯示權限與相容限制，不把未實作 broker 相容性藏起來。 + +### 20.6 MVP UI/API + +UI 在 Computer／Agent 的 Tools 頁：新增來源、選安裝目標、分配 agent、連接帳號、權限摘要、測試、啟用／停用、版本／更新／回滾／移除、活動紀錄。流程可以跨頁引導，不要求先設計大型商店。 + +建議的內部 API 職責：validate manifest、install job、read installation status、create/revoke binding、start OAuth、list effective capabilities、update/rollback、remove。這些路由需新的 typed contracts／auth／idempotency，不讓 LLM 呼叫安裝管理 API 自行批准。 + +Registry 改變以 event 通知 harness/UI，授權撤銷立即阻止新 dispatch；活躍 run 在下一安全點讀取版本化 registry。不要每回合重連全部 MCP，也不要讓一次安裝把所有工具 schema 全塞入 prompt。 + +--- + +## 21. Outlook connector：第一個外部擴充驗收案例 + +### 21.1 產品選擇與帳號範圍 + +建議以 **Microsoft Graph v1.0 + 使用者 delegated OAuth** 建立 Computer-local adapter；將它包成通用 loader 能安裝的 MCP／本機工具，證明外掛模型不是只支援 built-in Gmail。個人 Outlook.com 與 Microsoft 365／Exchange Online 工作／學校帳號各測；實際 tenant policy、admin consent、MFA 或授權限制正常顯示，不承諾公司帳號一定免審。 + +「Outlook connector」是泛稱。Power Automate／Logic Apps 的 Outlook connector 是其平台整合，不是通用 Linux binary 可直接下載進 LazyBoy [E21]。本計劃不採外部 Power Automate 作本機檔案處理或 workflow 執行的隱性替代。Outlook 桌面程式裡加入的任意 IMAP／本機 Exchange 帳號，也不能由名稱就推定支援 Graph；以 mailbox provider probe 為準。 + +### 21.2 權限逐步增加 + +| 階段 | 權限／範圍 | 可用行為與限制 | +|---|---|---| +| 基本郵件列表 | 評估 Mail.ReadBasic 是否足夠 | headers/基本資料；不能假定包含分類所需正文／preview [E14] | +| 需讀內容分類 | delegated Mail.Read 或較高已核准 scope | 只取必要欄位，必要正文才取，資料是 untrusted content | +| 修改 message categories | delegated Mail.ReadWrite | tool 層只開放 categories，不因 broad scope 自動允許刪信／draft 等 [E12] | +| 建立 master categories | MailboxSettings.ReadWrite | 額外權限；第一版可先用既有分類，避免未必要的 scope [E13] | +| 寄信／搬資料夾／日曆 | 另外定義 capability 與核准 | 不列為郵件分類預設權限，不自動打開 | + +OAuth 採正式 authorization code + PKCE／適用的受支援 auth library；按 app 類型正確保護 client credential，state/nonce／redirect URI 核對；是否申請 offline_access 依 refresh 需求 [E11]。不從 browser cookie 抽 token、不讀使用者密碼、不略過 MFA。 + +### 21.3 端到端流程 + +```text +在指定 shared/private Computer 安裝 Outlook adapter + → 對 agent 建 binding + → 使用者授權 Microsoft 帳號 + → 檢查 mailbox/account identity 與 operation grant + → 取指定範圍郵件、完整處理分頁 + → rule-first + model 批次分類、模糊結果留待確認 + → 產生只含 categories 變更的 immutable plan + → 核准 plan hash／account／message 範圍 + → 執行 Graph 更新（有界批次） + → 逐項讀回／驗證／記錄 changed/unchanged/failed/unknown +``` + +Graph 以 `PATCH /me/messages/{id}` 的 categories 屬性更新，這不是 Gmail label id 的 add/remove API [E12]。維持原有非本次變更分類，先讀當前 categories 計算目標；不要直接把固定分類陣列覆蓋其他人的分類。 + +對同 mailbox 使用 canonical account resource lock。若 API 支援可驗證的 conditional update，需實測其 If-Match/etag 語意；**本計劃不預設 Graph message PATCH 支援通用 CAS**。若無原子比較交換，外部 Outlook 客戶端仍有競爭窗口，必須標示 best-effort、遇可見衝突停止／重規劃；本地 lock／事後讀回不等於保證不丟失人類並行更新。 + +### 21.4 批次、分頁與常駐同步 + +- Graph JSON `$batch` **最多 20 個子請求**，每個子項獨立狀態；batch HTTP 200 不代表全部成功。按子項處理 401/429/5xx 與 Retry-After；這與 Gmail 1,000 message IDs 的 batchModify 不同 [E15–E16]。 +- `$select` 僅取必要欄位；完整跟隨 server 的 `@odata.nextLink`，不自行拼接 skip／把第一頁當全部 [E14]。驗證 continuation URL 仍是允許的 provider，防 SSRF／account confusion。 +- 使用 `Prefer: IdType="ImmutableId"` 需在需要的每個 request／batch 子 request 保持一致；它在同信箱移動 folder 時穩定，但跨 archive mailbox／匯出再匯入仍有例外 [E17]。 +- recurring sync 後續用 **每個 folder 的 message delta**，維護相應 nextLink/deltaLink；不是任意 search 的全信箱通用增量游標。token 失效時有安全重同步流程 [E18]。 +- pending/unknown write 先回讀，不能全批盲重送；分類 cache 需包含 mailbox/message/content version/taxonomy version，不能拿另一帳號或舊內容結果套用。 +- 附件下載到 Computer，掃描／解析／資料轉換也在 Computer；長期不需要的信件原文不存中央 DB。 + +### 21.5 必需交付 + +fake Graph HTTP provider、固定測試 mailbox data、範例 package build、可安裝 manifest、read/preview/apply/verify tests、shared A 有權 B 無權的測試、版本更新／token 撤銷與恢復、完整 Activity 記錄。 + +真實 OAuth 不可用不阻擋上述內部實作，但必須將真實帳號 E2E 標 BLOCKED_EXTERNAL。不得把 fixture 的分類數字寫成真的已替使用者整理 Outlook。 + + +--- + +## 附錄 A：依據與查核來源 + +以下是本計劃使用的基準原始碼與官方規格。coding agent 必須以其開始實作時的 HEAD 核對，不盲套行號；新設計與 target 不是上游已具備的功能。 + +### Repository 與前次健檢 + +- 基準 repository：`https://github.com/igs170911/LazyBoy` +- Commit：`e6afa324530e19922909d4692c28fb005cc05a7a` +- [R1] 工具與截圖：`https://github.com/igs170911/LazyBoy/blob/e6afa324530e19922909d4692c28fb005cc05a7a/crates/api/src/tools.rs` +- [R2] SandboxProvider／原生契約：`https://github.com/igs170911/LazyBoy/blob/e6afa324530e19922909d4692c28fb005cc05a7a/crates/control/src/sandbox.rs` +- [R3] Sandbox transport／檔案 bytes：`https://github.com/igs170911/LazyBoy/blob/e6afa324530e19922909d4692c28fb005cc05a7a/crates/sandbox/src/docker.rs` +- [R4] Supervisor 容器內執行：`https://github.com/igs170911/LazyBoy/blob/e6afa324530e19922909d4692c28fb005cc05a7a/crates/supervisor/src/docker.rs`；routes：`https://github.com/igs170911/LazyBoy/blob/e6afa324530e19922909d4692c28fb005cc05a7a/crates/supervisor/src/main.rs` +- [R5] Harness loop／prompt：`https://github.com/igs170911/LazyBoy/blob/e6afa324530e19922909d4692c28fb005cc05a7a/crates/api/src/runs.rs` +- [R6] Browser adapter：`https://github.com/igs170911/LazyBoy/blob/e6afa324530e19922909d4692c28fb005cc05a7a/crates/control/src/cua/browser.rs` +- [R7] Run policy：`https://github.com/igs170911/LazyBoy/blob/e6afa324530e19922909d4692c28fb005cc05a7a/crates/harness/src/policy.rs` +- [R8] MCP：`https://github.com/igs170911/LazyBoy/blob/e6afa324530e19922909d4692c28fb005cc05a7a/crates/api/src/mcp.rs` +- 前次本對話產物：`LazyBoy_Computer_Use_Healthcheck_2026-09-10.md`。本文件已包含必要內容，可獨立交付；若與舊文件的 GUI 可見性或 Team 共用假設不一致，以本文件第 0–1 節為準。 + +### 官方外部文件（2026-09-10 查閱） + +- [E1] Gmail batchModify／最大 1,000 IDs／空成功回應：`https://developers.google.com/workspace/gmail/api/reference/rest/v1/users.messages/batchModify` +- [E2] Gmail OAuth scopes 與受限資料說明：`https://developers.google.com/workspace/gmail/api/auth/scopes` +- [E3] Playwright connectOverCDP／現有 browser context／相容性限制：`https://playwright.dev/docs/api/class-browsertype` +- [E4] Docker isolation、安全邊界與 daemon 權限參考：`https://docs.docker.com/engine/security/` + +Docker 容器隔離不是不需測試的安全保證；應依威脅模型與官方指引配置權限、network、mounts、resource limits，並以本文件的跨 Computer 測試驗證 [E4]。 + + +### 本版新增來源（2026-09-10 查閱；不是把官方支援當成 LazyBoy 已整合） + +- [R9] LazyBoy display 啟動／Team slots／Xvfb/x11vnc/XFCE/Cua：`https://github.com/igs170911/LazyBoy/blob/e6afa324530e19922909d4692c28fb005cc05a7a/image/computer/lazyboy-screen` +- [E5] TigerVNC Xvnc：`https://tigervnc.org/doc/Xvnc.html` +- [E6] TigerVNC x0vncserver：`https://tigervnc.org/doc/x0vncserver.html` +- [E7] noVNC 與 WebSocket proxy：`https://novnc.com/info.html` +- [E8] Selkies 現行設計／transport／encoder：`https://docs.selkies.io/design`；`https://selkies-project.github.io/selkies/component` +- [E9] Xpra 應用與桌面轉送／HTML5：`https://xpra.org/index.html` +- [E10] Playwright Docker 與 Chromium／程序回收注意事項：`https://playwright.dev/docs/docker`。本案不直接採測試容器常見的 host IPC／SYS_ADMIN 放寬，保留產品隔離要求。 +- [E11] Microsoft authorization code／PKCE：`https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-auth-code-flow` +- [E12] Graph message update／categories／Mail.ReadWrite：`https://learn.microsoft.com/en-us/graph/api/message-update?view=graph-rest-1.0` +- [E13] Graph master categories 建立／MailboxSettings.ReadWrite：`https://learn.microsoft.com/en-us/graph/api/outlookuser-post-mastercategories?view=graph-rest-1.0` +- [E14] Graph messages 列表／$select／分頁／permissions：`https://learn.microsoft.com/en-us/graph/api/user-list-messages?view=graph-rest-1.0` +- [E15] Graph JSON batching／20 子請求：`https://learn.microsoft.com/en-us/graph/json-batching` +- [E16] Graph throttling／子請求失敗：`https://learn.microsoft.com/en-us/graph/throttling` +- [E17] Outlook immutable IDs 的範圍與例外：`https://learn.microsoft.com/en-us/graph/outlook-immutable-id` +- [E18] Graph folder message delta：`https://learn.microsoft.com/en-us/graph/api/message-delta?view=graph-rest-1.0` +- [E19] MCP tools／annotations 信任限制：`https://modelcontextprotocol.io/specification/2025-11-25/server/tools` +- [E20] MCP HTTP authorization／transport：`https://modelcontextprotocol.io/specification/2025-11-25/basic/authorization` +- [E21] Microsoft Office 365 Outlook 平台 connector：`https://learn.microsoft.com/en-us/connectors/office365/` + +研究只證明上游文件與基準程式呈現的行為；本機 Docker/GUI/企業帳號、實際效能及完整威脅邊界仍須 coding agent 按測試矩陣驗證。不存在單純換 VNC 就保證所有 agent 任務提速的證據。 diff --git a/docs/plan/LAZYBOY_CODING_AGENT_START_HERE_V3.md b/docs/plan/LAZYBOY_CODING_AGENT_START_HERE_V3.md new file mode 100644 index 0000000..2aa1f1c --- /dev/null +++ b/docs/plan/LAZYBOY_CODING_AGENT_START_HERE_V3.md @@ -0,0 +1,34 @@ +# LazyBoy coding agent 啟動指令 — V3 + +本版取代舊版啟動指令。與 `LAZYBOY_AGENT_COMPUTER_IMPLEMENTATION_PLAN_V3.md` 一起使用。 + +## 可直接貼給 coding agent + +請閱讀隨附 V3 主計劃，直接修改目前 LazyBoy repository，不要只重新產生計劃。 + +**產品必須同時保留共用主機 Team Computer 與私人主機 Dedicated Computer。** 共用模式允許多個 agent 綁定同一 Computer，沿用 per-agent workspace/display/profile 與 shared/；私人模式才是排他 Computer。不得把所有 agent 強制改成獨立容器，不刪 Team 模式、不強制遷移、不把 shared 降級成 GUI-only。 + +所有任務命令、檔案處理、MCP 的本地 runtime 與連接器請求在 agent 當前獲准的 Computer 內執行。工具不必在桌面演出，但有結構化紀錄、真實程序狀態與結果驗證。API/Supervisor 只管理與傳輸，不執行 agent 任意主機任務。 + +先做： +1. 查 HEAD、git status、AGENTS.md、README、Cargo/前端 scripts，保護使用者未提交修改。基準 commit 為 e6afa324530e19922909d4692c28fb005cc05a7a，但以當前程式核對。 +2. 完成 PR-00 baseline/regression，先修 PR-01 截圖與工具契約；重用 SandboxProvider 已有 execute/list/read/write，不從換模型或全新 harness 開始。 +3. 按主文件第 15 節 DAG 實作；PR-04 必須能交付兩模式的 native fast path。PR-05 要能真的安裝一個 sample MCP。Gmail/Outlook 分 PR-08A/B；TigerVNC 做 PR-09 候選測試，不與 core exec 改動綁死。 +4. 逐條處理 O01–O48，給出採用、保留現況、實驗、延期或阻礙及理由；P2/P3 不要求全部替換。T01–T64 的實測與缺項要誠實標記。 + +必要規則： +- shared 多 bot→同 computer_id 合法；每 bot 一個 active assignment。不要對所有 assignments.computer_id 無條件加 UNIQUE。 +- 同一 display/profile 的 GUI mutation 排他；不同 display、受控且無衝突的 jobs 可並行。任意無界 shell 不能靠模型填 read_only 假裝安全。 +- 接管 A、接管某 display、暫停整台 Computer 分清楚。不要為了暫停 A 而 freeze 仍有 B 工作的 Team 容器。 +- 共享程式套件不等於共享 Outlook/Gmail 帳號；package installation、agent binding、credential connection 分離。 +- 同 UID 共用 arbitrary shell 不是強機密隔離；保護 broker/core keys，對第三方工具清楚標示權限與信任邊界，不能只用資料夾和 prompt 擋。 +- 外部 stdio MCP、CLI、審核後 HTTP MCP/Graph adapter 走通用 Tool Manager。安裝/啟動/認證/授權/健康狀態分開；包版本與 hash 固定，安全解壓，支援熱啟用、更新、回滾、撤銷、移除。 +- Outlook 建立本機 Graph adapter 的可安裝範例，不假造官方 npm package。Mail.ReadWrite 與 MailboxSettings.ReadWrite 不混用；Graph batch 最多 20 子請求，逐項判斷 429/401 等結果。寄信、刪信、日曆另行核准。 +- TigerVNC 以 Xvnc 取代 Xvfb+x11vnc 是候選 A；x0vncserver 只替換匯出層。先保留 noVNC/websockify/Cua/XFCE，維持既有 slot/port/profile 對應與中文/a11y，測試後才改 default；保留 rollback。 +- 截圖/模型/工具/觀看串流分開量測；既有 compositor 已 off 不當成新成果。不用弱畫質換假提速。 +- tool 200/exit 0 不等於 task completed；未知副作用先 read-back。job_id、process-tree cancel、byte-safe files、CAS/conflict、idempotency/outbox、secret redaction 都要有測試。 +- 沒有 OAuth、GUI 或真實模型時，完成程式與 disposable/local fixtures，外部項標 BLOCKED_EXTERNAL。不能把 mock 成功當真實整合通過。 + +每段更新 docs/agent-computer-progress.md：狀態、實際 diff、Oxx/Txx、測試命令與結果、未測原因、benchmark 樣本、雙模式影響、回滾、下一個可執行步驟。不刪測試掩蓋問題、不索取明文密碼、不動真實主信箱、不刪舊 shared 資料、不自動正式部署或 force push。 + +現在先完成 PR-00/01 的實際修改與驗證，再依依賴推進。暫停時留下精確 checkpoint，不把未完成全案說成完成。 diff --git a/docs/plan/LAZYBOY_OPTIMIZATION_CHECKLIST_V3.md b/docs/plan/LAZYBOY_OPTIMIZATION_CHECKLIST_V3.md new file mode 100644 index 0000000..7e87c65 --- /dev/null +++ b/docs/plan/LAZYBOY_OPTIMIZATION_CHECKLIST_V3.md @@ -0,0 +1,86 @@ +# LazyBoy V3：48 項優化與採用判斷 + +本清單是 V3 主計劃第 18 節的獨立副本；產品契約、來源與實作細節以主計劃為準。共用／私人都保留；不是要求把所有候選工具都裝上。 + +## 18. 逐項改善清單：48 項，不等於 48 項全部替換 + +P0：正確性／產品邊界；P1：核心交付；P2：量測後採用；P3：可選／延後。以下是本案的待辦與候選，沒有實測提速倍數。每項在進度檔有 disposition、證據和回滾；已存在的優化要標「保留」，不要再次算成新成果。 + +### A. 原生執行與工作流程 + +| ID | 項目 | 優先級 | 具體改動 | 要改善的問題 | 驗收／限制 | +|---|---|---|---|---|---| +| O01 | 修正新圖交付 | P0／必做 | 修反轉條件，分 captured/delivered，接管／換模型強制交付 | 無圖硬做造成的錯誤與重試 | 真值表＋模型 request fixture；[R1] | +| O02 | 原生 exec | P1／必做 | 命令在綁定 Computer 執行，直接取 stdout/stderr/exit | 消除終端機打字與讀圖 | 兩模式 text-only 命令 0 screenshot | +| O03 | 原生 files | P1／必做 | 重用並強化 SandboxProvider，不透過 sed 畫面讀檔 | 減少回合／避免 binary 毀損 | range、Unicode、binary、shared 路徑；[R2–R4] | +| O04 | 持續 jobs 與 PTY | P1／必做 | 短命令直接返回，長作業由 Runner 管理、事件通知／真取消 | 不請 LLM 反覆看是否做完 | API 重啟可追 job；process-tree cancel | +| O05 | 批次程序／form macro | P1／必做 | 已知步驟在受控 executor 內完成，有 pre/post checks | 不逐列／逐 click 呼叫模型 | 每個子動作有 trace；失敗即停 | +| O06 | 套件／連線常駐 | P1／必做 | 暖 MCP instance、HTTP pooling、版本化 cache | 降低啟動與重複連線成本 | warm/cold 分測；不共享私人 env | +| O07 | 穩定 Runner transport | P2／量測 | 優先現有安全 transport；高 overhead 才改持續 UDS/HTTP 通道 | 減少每次 docker exec 的啟動成本 | 實測 RPC 分解；不能公開 root socket | +| O08 | readiness 分層 | P1／必做 | Runner、browser、desktop、viewer 各自 ready，按需啟動 | 讀 CSV 不先等整個桌面 | no viewer/no desktop 的 native fixture | + +### B. 模型與瀏覽器 + +| ID | 項目 | 優先級 | 具體改動 | 要改善的問題 | 驗收／限制 | +|---|---|---|---|---|---| +| O09 | 按能力路由 | P1／必做 | 原生／已授權 API 優先，DOM 次之，必要時視覺 | 避免 Gmail／Outlook 走慢 UI | route rationale＋權限拒絕不繞過 | +| O10 | 工具 schema 按需載入 | P1／必做 | 僅帶此 run 可用能力，版本／grant cache | 縮短 context、避免選錯工具 | revoke 後立即失效；記 schema tokens | +| O11 | 簡單任務不加 planner | P1／必做 | 用既有一輪決策，複雜任務才 TaskPlan | 避免多 agent／planner 套娃 | 記模型回合、TTFT、正確完成 | +| O12 | 條件等待 | P1／必做 | wait_until／job events 有 deadline，替代固定 sleep | 消除空等與毫無進度輪詢 | 未知 disabled 原因不無限等 | +| O13 | DOM snapshot 限量與增量 | P2／量測 | 聚焦相關區域，revision 與失效 refs 完整處理 | 減少重傳巨大 page text | 錯 revision 回 full；不能漏關鍵元素 | +| O14 | agent 圖片和 viewer 串流分離 | P1／必做 | 高可讀截圖按需取，不把 VNC frame 全塞模型 | 減少模型圖片成本且維持辨識 | 縮放映射／force image／小字 fixture | +| O15 | 既有 browser/session 穩定 attach | P1／必做 | profile/tab/frame 明確綁定，禁止亂開替代瀏覽器 | 減少重登入與 stale reference | 同 profile 不雙開；重啟正確失效 | +| O16 | 驗證、有限恢復與模型升級 | P1／必做 | 進度看 milestone；兩次同義錯誤換策略，難題才升級模型 | 停止二十分鐘空转 | 未知副作用先讀回；無固定提速保證 | + +### C. 顯示與遠端桌面 + +| ID | 項目 | 優先級 | 具體改動 | 要改善的問題 | 驗收／限制 | +|---|---|---|---|---|---| +| O17 | Xvfb+x11vnc → TigerVNC Xvnc | P2／必做候選測試 | 一個 backend 同時提供 X server 與 VNC，保留 rollback | 可能減少顯示匯出層與維護負擔 | Cua／a11y／中文字／多 display／CPU/latency；[E5] | +| O18 | Xvfb+x0vncserver 備選 | P3／保留替代 | 只換 VNC 匯出層，不替換 X server | Xvnc 相容性卡住時較小改動 | 不能說已移除 Xvfb；[E6] | +| O19 | 檢查 -noxdamage | P2／對照 | 現有 x11vnc 關閉 XDamage；測啟用是否有重繪瑕疵 | 判斷掃描負擔能否降低 | 靜態／捲動／遮擋／影片測試；不可直接刪旗標；[R9] | +| O20 | noVNC/websockify 與畫質檔 | P2／調校 | 先保留；viewer FPS/壓縮可調，無 viewer 減少服務負載 | 改善觀看／頻寬，不假稱模型更快 | 文字清晰、輸入延遲、resize、proxy auth；[E7] | +| O21 | 精簡桌面而非先換 OS | P2／調校 | 保留 XFCE/a11y；已 compositor=off，評估不必要 autostart | 降低多 slot CPU/RAM | 不重做既有優化；不破壞 DBus/a11y；[R9] | +| O22 | Selkies 替代 viewer | P3／可選 | 有高幀率／音訊需求才測；目前有 WebSocket 與選用 WebRTC | 影音觀看可能較適合 | 硬體 encoder 實測、網路與前端成本；[E8] | +| O23 | Xpra 替代 viewer | P3／可選 | 需要單視窗發布／session forwarding 時測 | 另一路 remote app 體驗 | 非 noVNC drop-in；額外 client/輸入整合；[E9] | +| O24 | Wayland／全面換桌面 | P3／延後 | 保留 X11 driver 路徑，未有需求不全面搬遷 | 避免同時引入 capture/input 相容性變動 | 獨立 ADR/fixtures；不是目前速度主解法 | + +### D. 共用／私人與資源協調 + +| ID | 項目 | 優先級 | 具體改動 | 要改善的問題 | 驗收／限制 | +|---|---|---|---|---|---| +| O25 | 雙模式正式保留 | P0／必做 | shared 多 bot→同 Computer；private 排他；不強制遷移 | 符合產品且避免破壞資料 | assignment/member/slot/重建測試 | +| O26 | 按 display／檔案／account 鎖 | P1／必做 | 同資源排他、不同 display 與受控 job 並行 | 避免共用主機整機串行 | Cua/DOM/human 同鎖；canonical shared key | +| O27 | scope-aware pause | P1／必做 | 接管 A、接管 display、暫停整機分開 | B 不被 A 的無關操作凍結 | 真 barrier／fencing；HTTP unknown 誠實顯示 | +| O28 | 共享 lifecycle／single-flight | P1／必做 | 啟動、刪除、idle reaper 看全部 member/jobs | 避免重複建立與誤停他人工作 | member refcount、job/viewer 活性聚合 | +| O29 | 程式共享與帳號授權分離 | P0／必做 | 共享唯讀 package，per-agent runtime/bindings/grants | 省重複安裝、不共享私人帳號 | 未知程式不能靠 annotations 保證安全 | +| O30 | 公平排程與總配額 | P1／必做 | 每 agent／Computer 的 CPU/RAM/jobs/requests 有界 | 抑制單 agent 拖慢所有人 | 壓力/OOM/慢 MCP；不無界並行 | +| O31 | Chromium /dev/shm 與程序回收 | P2／量測 | 調 per-Computer shm、init/reaper、renderer 健康 | 減少可避免的崩潰與孤兒程序 | 不能盲開 ipc=host/SYS_ADMIN/no-sandbox；[E10] | +| O32 | X11/DBus/profile 權限與 session | P1／必做 | per-slot identity，評估 Xauthority，系統安裝有管理鎖 | 避免串台與共用 session 風險 | 同 UID 非強隔離；Xvnc :1 port 對應測試 | + +### E. 可安裝外部工具／Outlook + +| ID | 項目 | 優先級 | 具體改動 | 要改善的問題 | 驗收／限制 | +|---|---|---|---|---|---| +| O33 | Tool Manager 安裝入口 | P1／必做 | catalog、manifest／受控 package URL、本地上傳三種入口 | 新增工具不改核心 dispatch | 實際 sample MCP 安裝，不只是設定畫面 | +| O34 | 來源版本與完整性 | P1／必做 | pin digest/version，審核 hooks，安全解壓與 quota | 更新可重現、防無界安裝 | 拒絕未審核 URL、路徑逃逸、錯 digest | +| O35 | 安裝／啟動／認證／授權分開 | P1／必做 | 每階段顯示狀態、failure reason 和 scope | 不再「裝好了卻不能用」 | installed ≠ connected ≠ authenticated ≠ authorized | +| O36 | 熱啟用與 registry 更新 | P1／必做 | health/schema 驗證，原子登錄能力；grant 撤銷失效 | 不為加工具重啟整個 LazyBoy | 正在執行 run 的能力版本與安全點更新 | +| O37 | 版本更新／回滾／移除 | P1／必做 | 新版本旁置，active job pin 舊版，bindings/reference GC | 共用主機其他 agent 不受破壞 | 新增權限重新核准；共享套件引用仍存活 | +| O38 | OAuth／token broker | P1／必做 | Microsoft/Google 正規授權，scoped token、refresh single-flight | 減少重登入與憑證混用 | MFA/admin policy 正常接管；secret canary；[E11] | +| O39 | Outlook Graph 外掛 | P1／必做驗收 | 讀信→categories preview/apply/verify；寄信另授權 | 驗證架構不只 Gmail 特判 | 個人/工作帳號對應 scopes；[E12–E14] | +| O40 | Outlook 批次／增量同步 | P1/P2 | Graph 20 子請求／逐項狀態；需要時 folder delta | 減少網路回合及每次全信箱掃描 | 429/Retry-After/分頁/ImmutableId；[E15–E18] | + +### F. 正確性、紀錄與可維護性 + +| ID | 項目 | 優先級 | 具體改動 | 要改善的問題 | 驗收／限制 | +|---|---|---|---|---|---| +| O41 | operation journal/outbox | P0/P1 | 副作用前記 intent；斷線 durable 補送與去重 | 避免掉紀錄與重做工作 | journal 故障不開始 mutation | +| O42 | 活動紀錄串流而非桌面演出 | P1／必做 | 工具與 job 的結構化事件、實際耗時／結果 | 人可追查、不逼每步打字 | noVNC 關閉仍可看進度 | +| O43 | artifact 存 Computer＋脫敏 | P1／必做 | 全文输出在 Computer，API 存摘要/reference，限量保留 | 避免 DB/context 被圖片/日誌塞滿 | secret/大檔/retention/串流授權測試 | +| O44 | Task 驗證與郵件分類品質 | P1／必做 | API 200 不等於任務完成；測 precision/coverage/abstention | 速度與正確性一起看 | read-back／真實 fixture、失敗也計入 | +| O45 | 衝突與冪等／未知效果 | P1／必做 | 同 operation 不重做；same-file CAS；未知 API 先回讀 | 降低重送／覆蓋／重複外部副作用 | 不假稱通用 exactly-once／無損 undo | +| O46 | 部件健康與版本相容矩陣 | P1/P2 | Runner/display/Cua/browser/plugin 個別 probe/recover | 失敗不用重啟整台 Computer | 不影響其他成員；pin 上游版本 | +| O47 | 已驗證 skill／流程重用 | P2／後續 | 將常用流程保存語意步驟與 checks，環境變更可失效 | 降低每次從零探索 | 不保存密碼／舊座標；profile/版本變更重驗 | +| O48 | 端到端＋viewer 雙 benchmark | P1／必做 | 分模型、工具、snapshot、queue、viewer，固定環境兩模式 | 找真瓶頸，不憑工具名字換底層 | 成功率/median/P95/CPU/RAM/可讀性；一項一改 | + diff --git a/image/computer/Dockerfile b/image/computer/Dockerfile index db00eb8..2a6ba24 100644 --- a/image/computer/Dockerfile +++ b/image/computer/Dockerfile @@ -88,6 +88,7 @@ RUN --mount=type=cache,id=lazyboy-apt-cache-$TARGETARCH,target=/var/cache/apt,sh websockify \ x11-utils \ x11vnc \ + tigervnc-standalone-server \ xdg-utils \ xfce4-panel \ xfce4-settings \ diff --git a/image/computer/lazyboy-screen b/image/computer/lazyboy-screen index dec2ac3..713e746 100755 --- a/image/computer/lazyboy-screen +++ b/image/computer/lazyboy-screen @@ -101,6 +101,10 @@ wait_port() { return 1 } +display_backend() { + printf '%s' "${LAZYBOY_DISPLAY_BACKEND:-xvfb_x11vnc}" +} + start_xvfb() { local display="$1" local number="$2" @@ -109,11 +113,42 @@ start_xvfb() { return 0 fi rm -f "/tmp/.X${number}-lock" "/tmp/.X11-unix/X${number}" + # T59: keep -ac until Xauthority is given to Cua/AT-SPI/x11vnc in-image. + # Public VNC is still the authenticated screen proxy, not a naked 5900. Xvfb "$display" -screen 0 1280x800x24 -ac +extension RANDR +render -noreset >>"${log}-xvfb.log" 2>&1 9>&- & echo $! > "${log}-xvfb.pid" wait_display "$display" } +# Candidate A: TigerVNC Xvnc provides X + RFB in one process. rfbport is +# explicit so slot 0 stays 5900 even though display is :1. +start_xvnc() { + local display="$1" + local number="$2" + local rfb_port="$3" + local log="$4" + if xdpyinfo -display "$display" >/dev/null 2>&1; then + return 0 + fi + local xvnc="" + for c in Xvnc Xtigervnc; do + if command -v "$c" >/dev/null 2>&1; then + xvnc="$c" + break + fi + done + if [[ -z "$xvnc" ]]; then + echo "TigerVNC Xvnc is not installed; falling back to Xvfb" >&2 + start_xvfb "$display" "$number" "$log" + return + fi + rm -f "/tmp/.X${number}-lock" "/tmp/.X11-unix/X${number}" + "$xvnc" "$display" -geometry 1280x800 -depth 24 -rfbport "$rfb_port" -localhost \ + -SecurityTypes None >>"${log}-xvnc.log" 2>&1 9>&- & + echo $! > "${log}-xvfb.pid" + wait_display "$display" +} + alive_pidfile() { local file="$1" [[ -f "$file" ]] || return 1 @@ -368,13 +403,31 @@ ensure_slot() { "$slot" "$display" "$view_port" "$vnc_port" exit 0 fi - start_xvfb "$display" "$number" "$log" || { - echo "Xvfb failed on ${display}" >&2 - cat "${log}-xvfb.log" >&2 || true - exit 1 - } + if [[ "$(display_backend)" == "tigervnc_xvnc" ]]; then + start_xvnc "$display" "$number" "$vnc_port" "$log" || { + echo "Xvnc failed on ${display}" >&2 + cat "${log}-xvnc.log" >&2 || true + exit 1 + } + else + start_xvfb "$display" "$number" "$log" || { + echo "Xvfb failed on ${display}" >&2 + cat "${log}-xvfb.log" >&2 || true + exit 1 + } + fi start_desktop "$display" "$xfce_home" "$log" - start_vnc "$display" "$vnc_port" "$view_port" "$log" || exit 1 + if [[ "$(display_backend)" != "tigervnc_xvnc" ]] || ! port_open "$vnc_port"; then + start_vnc "$display" "$vnc_port" "$view_port" "$log" || exit 1 + else + if ! port_open "$view_port"; then + local novnc=/usr/share/novnc + websockify --heartbeat=30 --web="$novnc" "0.0.0.0:${view_port}" "127.0.0.1:${vnc_port}" \ + >>"${log}-novnc.log" 2>&1 9>&- & + fi + wait_port "$vnc_port" + wait_port "$view_port" + fi start_cua_driver "$display" "$log" || exit 1 start_xterm "$display" "$log" if [[ -n "$profile" ]]; then diff --git a/image/computer/start.sh b/image/computer/start.sh index 1d0ba70..3074a99 100755 --- a/image/computer/start.sh +++ b/image/computer/start.sh @@ -29,6 +29,13 @@ if command -v dbus-launch >/dev/null 2>&1; then eval "$(dbus-launch --sh-syntax)" fi +# Runner-only: native exec/files/MCP without XFCE/noVNC. GUI later uses +# `lazyboy-screen ensure` (T30/T60). +if [[ "${LAZYBOY_RUNNER_ONLY:-0}" == "1" ]]; then + touch /tmp/lazyboy/ready + exec sleep infinity +fi + lazyboy-screen boot-primary || exit 1 touch /tmp/lazyboy/ready pid="" diff --git a/migrations/022_agent_computer.sql b/migrations/022_agent_computer.sql new file mode 100644 index 0000000..b40f1e2 --- /dev/null +++ b/migrations/022_agent_computer.sql @@ -0,0 +1,84 @@ +-- Dual-mode identity, operations, jobs, artifacts, tool packages, grants. +-- shared computers may have many bots; do NOT add UNIQUE(bots.computer_id). + +ALTER TABLE computers + ADD COLUMN IF NOT EXISTS generation INTEGER NOT NULL DEFAULT 1, + ADD COLUMN IF NOT EXISTS computer_epoch BIGINT NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS display_backend TEXT NOT NULL DEFAULT 'xvfb_x11vnc'; + +CREATE TABLE IF NOT EXISTS computer_operations ( + id TEXT PRIMARY KEY, + computer_id TEXT NOT NULL REFERENCES computers (id) ON DELETE CASCADE, + bot_id TEXT NOT NULL, + run_id TEXT, + payload_hash TEXT NOT NULL, + status TEXT NOT NULL, + result TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE UNIQUE INDEX IF NOT EXISTS computer_operations_id_hash + ON computer_operations (id, payload_hash); + +CREATE TABLE IF NOT EXISTS computer_jobs ( + id TEXT PRIMARY KEY, + computer_id TEXT NOT NULL REFERENCES computers (id) ON DELETE CASCADE, + generation INTEGER NOT NULL DEFAULT 1, + bot_id TEXT NOT NULL, + status TEXT NOT NULL, + argv JSONB NOT NULL DEFAULT '[]'::jsonb, + pin_version TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +ALTER TABLE computer_jobs ADD COLUMN IF NOT EXISTS pin_version TEXT; + +CREATE TABLE IF NOT EXISTS computer_artifacts ( + id TEXT PRIMARY KEY, + computer_id TEXT NOT NULL REFERENCES computers (id) ON DELETE CASCADE, + bot_id TEXT NOT NULL, + relative_path TEXT NOT NULL, + sha256 TEXT NOT NULL, + size BIGINT NOT NULL DEFAULT 0, + operation_id TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS tool_packages ( + id TEXT PRIMARY KEY, + computer_id TEXT NOT NULL REFERENCES computers (id) ON DELETE CASCADE, + package_id TEXT NOT NULL, + version TEXT NOT NULL, + sha256 TEXT NOT NULL, + status TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (computer_id, package_id, version) +); + +CREATE TABLE IF NOT EXISTS tool_bindings ( + id TEXT PRIMARY KEY, + package_row_id TEXT NOT NULL REFERENCES tool_packages (id) ON DELETE CASCADE, + bot_id TEXT NOT NULL, + account_id TEXT, + status TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (package_row_id, bot_id) +); + +CREATE TABLE IF NOT EXISTS tool_grants ( + id TEXT PRIMARY KEY, + bot_id TEXT NOT NULL REFERENCES bots (id) ON DELETE CASCADE, + capability TEXT NOT NULL, + account_ref TEXT, + expires_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS operation_outbox ( + id TEXT PRIMARY KEY, + event_key TEXT NOT NULL UNIQUE, + payload JSONB NOT NULL, + delivered BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/migrations/023_agent_computer_fixes.sql b/migrations/023_agent_computer_fixes.sql new file mode 100644 index 0000000..11803f2 --- /dev/null +++ b/migrations/023_agent_computer_fixes.sql @@ -0,0 +1,17 @@ +-- Ledger fixes. +-- * Mutating tools that need no Computer (memory, schedules) are journaled +-- too, so computer_id must be nullable. +-- * The outbox is polled every 200 ms; give the undelivered scan an index and +-- let retention drop delivered rows. +-- * `computer_operations.id` is now `{bot_id}:{operation_id}`; the extra +-- (id, payload_hash) unique index never added anything over the PK. + +ALTER TABLE computer_operations ALTER COLUMN computer_id DROP NOT NULL; + +DROP INDEX IF EXISTS computer_operations_id_hash; + +CREATE INDEX IF NOT EXISTS operation_outbox_pending_idx + ON operation_outbox (created_at) WHERE NOT delivered; + +CREATE INDEX IF NOT EXISTS computer_operations_bot_idx + ON computer_operations (bot_id, created_at); diff --git a/scripts/sample-mcp/echo_server.py b/scripts/sample-mcp/echo_server.py new file mode 100644 index 0000000..8c505bb --- /dev/null +++ b/scripts/sample-mcp/echo_server.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Minimal stdio MCP-ish echo server for LazyBoy Tool Manager fixtures. + +Speaks a tiny JSON-RPC subset: initialize, tools/list, tools/call. +Not a substitute for a full MCP SDK; used to prove install → bind → call. +""" +import json +import sys + + +def reply(message_id, result): + sys.stdout.write(json.dumps({"jsonrpc": "2.0", "id": message_id, "result": result}) + "\n") + sys.stdout.flush() + + +def main(): + for line in sys.stdin: + line = line.strip() + if not line: + continue + request = json.loads(line) + method = request.get("method") + message_id = request.get("id") + if method == "initialize": + reply(message_id, {"protocolVersion": "2025-11-25", "serverInfo": {"name": "lazyboy-echo"}}) + elif method == "tools/list": + reply( + message_id, + { + "tools": [ + { + "name": "echo", + "description": "Echo text from the bound Computer", + "inputSchema": { + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], + }, + } + ] + }, + ) + elif method == "tools/call": + args = (request.get("params") or {}).get("arguments") or {} + text = args.get("text", "") + reply(message_id, {"content": [{"type": "text", "text": text}]}) + elif method == "notifications/initialized": + continue + else: + reply(message_id, {"error": f"unknown method {method}"}) + + +if __name__ == "__main__": + main() diff --git a/scripts/sample-mcp/manifest.yaml b/scripts/sample-mcp/manifest.yaml new file mode 100644 index 0000000..16a421f --- /dev/null +++ b/scripts/sample-mcp/manifest.yaml @@ -0,0 +1,23 @@ +manifest_version: 1 +id: lazyboy.example.echo +version: 0.0.1 +source: + kind: reviewed_local_artifact + artifact_ref: ./echo_server.py + sha256: compute-at-install +runtime: + kind: stdio_mcp + execution_location: assigned_computer + entrypoint: [python3, ./echo_server.py] + state_scope: agent_binding +installation: + supported_modes: [shared, dedicated] + share_immutable_package: true +permissions: + filesystem: [own_workspace] + shared_paths: [] + network_services: [] + needs_host_access: false + needs_root: false +capabilities: + - echo.text