diff --git a/.env.example b/.env.example index 25d462c..be2f9ef 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,8 @@ SANDBOX_SUPERVISOR_TOKEN= # Keep this key stable when rotating the app login token. LAZYBOY_VAULT_KEY= POSTGRES_PASSWORD=lazyboy +# 127.0.0.1 = 只有本機；0.0.0.0 = 開放區網（需 LAZYBOY_APP_TOKEN >= 32 字元）； +# 也可填單一網卡的 IP，把監聽限制在那個介面。 LAZYBOY_BIND_IP=127.0.0.1 SANDBOX_PROVIDER=docker DATABASE_URL=postgres://lazyboy:lazyboy@127.0.0.1:5434/lazyboy @@ -19,9 +21,23 @@ LAZYBOY_COMPUTER_CPUS=2 LAZYBOY_COMPUTER_PIDS=2048 # Only affects the Agent desktop container. Disabled by default. LAZYBOY_COMPUTER_SUDO=false +# Computer-control backend inside each desktop container. Rebuild/recreate +# computers after upgrading. Cua is the only supported computer controller. +LAZYBOY_COMPUTER_DRIVER=cua # Linux only (optional): point this at the host's LXCFS root to make htop/free # report the per-Agent cgroup quota. Leave the default empty directory on macOS. LAZYBOY_LXCFS_ROOT=./data/lxcfs +# 任務長度政策：不再用固定輪數掐掉任務。正常任務一路做到驗證完成，只有 +# 真的鬼打牆（同一個動作重複、同一個錯誤一直失敗、很久沒有新的成功）才會被 +# 提示、接著暫停等你決定；最後兩個是防迴圈失控烧 token 的保險絲，不是額度。 +# soft turns：第 60 輪起，之後每 soft every 輪請模型自我交代「已完成/還缺/下一步」 +# cap turns / hard minutes：最後防火牆，正常任務不該碰到 +LAZYBOY_RUN_SOFT_TURNS=60 +LAZYBOY_RUN_SOFT_EVERY=120 +LAZYBOY_RUN_CAP_TURNS=1000 +LAZYBOY_RUN_SOFT_MINUTES=75 +LAZYBOY_RUN_HARD_MINUTES=240 + LAZYBOY_MEMORY_ENABLED=true LAZYBOY_MEMORY_MODEL_CACHE=./data/fastembed LAZYBOY_MEMORY_TOP_K=8 @@ -31,6 +47,12 @@ LAZYBOY_MEMORY_BYTE_BUDGET=6000 LAZYBOY_EVENT_RETENTION_DAYS=30 LAZYBOY_CHECKPOINT_RETENTION_DAYS=7 LAZYBOY_RUN_RETENTION_DAYS=90 +# Per-run trace shown when hovering the thinking avatar (round, tool results, errors). +LAZYBOY_RUN_ACTIVITY_RETENTION_DAYS=7 LAZYBOY_RECORDING_RETENTION_DAYS=30 LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS=90 LAZYBOY_DB_WARN_MB=1024 + +# Docker 網路名稱：API 與 Agent 電腦的 noVNC 透過它相通（容器內用，不對外）。 +# 同時跑多組 LazyBoy 時改這個名字避免相撞。 +LAZYBOY_SCREEN_NETWORK=lazyboy_screen diff --git a/Cargo.lock b/Cargo.lock index 505e86f..ed14eaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1889,7 +1889,6 @@ name = "lazyboy-api" version = "0.1.0" dependencies = [ "aes-gcm", - "async-trait", "axum", "base64 0.22.1", "cap-std", @@ -1900,9 +1899,7 @@ dependencies = [ "fastembed", "futures-util", "hex", - "hmac", "http", - "http-body-util", "lazyboy-contracts", "lazyboy-control", "lazyboy-harness", @@ -1915,7 +1912,6 @@ dependencies = [ "serde_json", "sha2", "sqlx", - "thiserror", "tokio", "tokio-tungstenite 0.26.2", "tower-http", @@ -1940,6 +1936,7 @@ name = "lazyboy-control" version = "0.1.0" dependencies = [ "async-trait", + "base64 0.22.1", "chrono", "hex", "image", @@ -1948,6 +1945,8 @@ dependencies = [ "serde_json", "sha2", "thiserror", + "tokio", + "tracing", ] [[package]] @@ -1955,12 +1954,8 @@ name = "lazyboy-controld" version = "0.1.0" dependencies = [ "axum", - "base64 0.22.1", - "lazyboy-contracts", "lazyboy-control", - "serde", "serde_json", - "thiserror", "tokio", "tracing", "tracing-subscriber", @@ -1979,7 +1974,6 @@ dependencies = [ "rig-core", "rustls", "rustls-native-certs", - "serde", "serde_json", "thiserror", "tokio", @@ -1992,39 +1986,30 @@ version = "0.1.0" dependencies = [ "async-trait", "base64 0.22.1", - "chrono", - "hex", "lazyboy-contracts", "lazyboy-control", "reqwest 0.12.28", - "serde", "serde_json", - "sha2", - "tokio", ] [[package]] name = "lazyboy-supervisor" version = "0.1.0" dependencies = [ - "async-trait", "axum", "base64 0.22.1", "bollard", "futures-util", "hex", "hmac", - "lazyboy-contracts", "lazyboy-control", "reqwest 0.12.28", "serde", "serde_json", "sha2", - "thiserror", "tokio", "tracing", "tracing-subscriber", - "uuid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c60dcd8..1aeef69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,10 +12,28 @@ members = [ [workspace.package] edition = "2024" +rust-version = "1.98" version = "0.1.0" license = "Apache-2.0" publish = false +# --- Lint policy ----------------------------------------------------------- +# A single place decides how the whole workspace is linted; every crate opts in +# with `[lints] workspace = true`. `make lint` runs clippy with -D warnings, so a +# new warning has to be fixed (or relaxed at the call site with a reason) before +# it reaches main. Thresholds live in clippy.toml, which cargo-clippy only reads +# from the directory it was started in - keep running it at the workspace root. +[workspace.lints.rust] +# Safety-relevant code (the vault, the docker socket) has to stay explicit. +unsafe_code = "warn" +unused_must_use = "deny" +unused_crate_dependencies = "warn" + +[workspace.lints.clippy] +# Debug leftovers and placeholder implementations must not reach a branch. +dbg_macro = "warn" +todo = "warn" + [profile.release] lto = true codegen-units = 1 @@ -34,7 +52,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock", "serd image = { version = "0.25", default-features = false, features = ["jpeg", "png"] } rig-core = "0.42" async-trait = "0.1" -tokio = { version = "1", features = ["macros", "rt-multi-thread", "process", "io-util", "fs", "signal", "time"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "process", "io-util", "fs", "signal", "time", "sync"] } axum = { version = "0.8", features = ["ws"] } tower-http = { version = "0.6", features = ["cors", "trace", "fs"] } tracing = "0.1" diff --git a/Makefile b/Makefile index 175dd58..43ec085 100644 --- a/Makefile +++ b/Makefile @@ -8,12 +8,17 @@ COMPOSE ?= docker compose COMPUTER_IMAGE ?= lazyboy/computer:local WEB_DIR ?= apps/web DATA_DIR ?= ./data +BUILDX_BUILDER ?= lazyboy +# PUSH=1 publishes the manifest list instead of writing an OCI archive. +MULTI_FLAGS ?= +$(if $(filter 1,$(PUSH)),$(eval MULTI_FLAGS := --push)) .PHONY: help env env-force \ up logs ps health down purge \ - computer postgres postgres-down \ + computer computer-multi images-multi postgres postgres-down pg-collation \ + cua-smoke \ build build-api build-supervisor build-controld \ - fmt clippy test clean \ + fmt fmt-check clippy lint audit test clean \ web \ dev dev-supervisor dev-api @@ -31,8 +36,12 @@ help: ## Show this help @echo "" @echo " Individual pieces:" @echo " make computer Build the heavy Debian desktop image (lazyboy/computer:local)" + @echo " make computer-multi Cross-build the desktop image for amd64 + arm64 (PUSH=1 to publish)" + @echo " make images-multi Cross-build desktop + api + supervisor for amd64 + arm64" + @echo " make cua-smoke Run Cua Driver smoke test in a disposable desktop container" @echo " make postgres Start only postgres (127.0.0.1:5434) and wait for ready" @echo " make postgres-down Stop postgres" + @echo " make pg-collation Repair a Postgres collation version mismatch (see docs)" @echo "" @echo " Local dev (postgres in Docker, Rust services on the host):" @echo " make dev Prep .env + postgres + computer image, then print run steps" @@ -43,8 +52,11 @@ help: ## Show this help @echo " make build cargo build --release (whole workspace)" @echo " make build-api cargo build --release -p lazyboy-api" @echo " make fmt cargo fmt --all" - @echo " make clippy cargo clippy (deny warnings)" - @echo " make test cargo test --workspace" + @echo " make fmt-check Report files rustfmt would change (legacy drift exists)" + @echo " make clippy cargo clippy (deny warnings, reads clippy.toml)" + @echo " make lint The Rust gate: clippy with -D warnings" + @echo " make audit cargo deny: RustSec advisories, licenses, sources" + @echo " make test cargo test --workspace (DB tests need: make postgres)" @echo " make web Build the frontend in $(WEB_DIR) (needs node/npm)" @echo " make clean cargo clean" @echo "" @@ -87,7 +99,26 @@ purge: ## Stop containers and delete the postgres data volume # --- Individual pieces ----------------------------------------------------- computer: ## Build the Debian desktop image used to spawn bot computers - docker build -f image/computer/Dockerfile -t $(COMPUTER_IMAGE) . + ./scripts/build-image.sh --tag $(COMPUTER_IMAGE) + +# Cross-builds every supported CPU architecture into one manifest list. Needs a +# docker-container builder + QEMU binfmt; both are bootstrapped by the script. +# PUSH=1 publishes to a registry, otherwise an OCI archive is written. +computer-multi: ## Cross-build the desktop image for all CPU architectures + ./scripts/build-image.sh --file image/computer/Dockerfile --multi $(MULTI_FLAGS) + +# The api and supervisor images carry per-architecture binaries as well (ONNX +# Runtime, node, the distroless libc), so they get the same treatment as the +# desktop image instead of only ever existing for the build host. +images-multi: ## Cross-build every shipped image for all CPU architectures + @for dockerfile in image/computer/Dockerfile image/supervisor/Dockerfile \ + image/api/Dockerfile; do \ + echo "== $$dockerfile"; \ + ./scripts/build-image.sh --file "$$dockerfile" --multi $(MULTI_FLAGS) || exit 1; \ + done + +cua-smoke: computer ## Run the Cua Driver smoke test inside a disposable desktop container + ./scripts/cua-smoke-test.sh --docker --image $(COMPUTER_IMAGE) postgres: ## Start only postgres and wait until it is ready $(COMPOSE) -f docker-compose.yml -f docker-compose.dev.yml up -d postgres @@ -97,6 +128,14 @@ postgres: ## Start only postgres and wait until it is ready postgres-down: ## Stop postgres $(COMPOSE) down postgres +# A pgvector image rebuilt on another glibc leaves every database recording the old +# collation version; Postgres then refuses CREATE DATABASE and `cargo test` hangs on +# PoolTimedOut. Stop the api first, then reindex + refresh each database in place. +pg-collation: ## Repair a Postgres collation version mismatch after an image update + $(COMPOSE) exec -T postgres psql -X -v ON_ERROR_STOP=1 -U lazyboy -d template1 -c "REINDEX DATABASE template1;" -c "ALTER DATABASE template1 REFRESH COLLATION VERSION;" + $(COMPOSE) exec -T postgres psql -X -v ON_ERROR_STOP=1 -U lazyboy -d postgres -c "REINDEX DATABASE postgres;" -c "ALTER DATABASE postgres REFRESH COLLATION VERSION;" + $(COMPOSE) exec -T postgres psql -X -v ON_ERROR_STOP=1 -U lazyboy -d lazyboy -c "REINDEX DATABASE lazyboy;" -c "ALTER DATABASE lazyboy REFRESH COLLATION VERSION;" + # --- Rust / web ------------------------------------------------------------ build: ## Release build of the whole workspace @@ -114,8 +153,23 @@ build-controld: ## Release build of the controld binary fmt: ## Format all Rust code cargo fmt --all -clippy: ## Run clippy, denying warnings - cargo clippy --all-targets -- -D warnings +fmt-check: ## Check formatting without touching files + cargo fmt --all --check + +# Lint policy lives in [workspace.lints] in the root Cargo.toml; the thresholds +# (e.g. too-many-arguments-threshold) live in clippy.toml, which cargo-clippy +# only reads from the directory it is started in - keep running this at the root. +clippy: ## Run clippy over the workspace, denying warnings + cargo clippy --workspace --all-targets -- -D warnings + +lint: clippy ## The Rust quality gate used by CI + +# cargo-deny is a separate CLI: cargo install --locked cargo-deny +audit: ## Supply-chain check (RustSec advisories, licenses, dependency sources) + @command -v cargo-deny >/dev/null 2>&1 || { \ + echo "cargo-deny is not installed: cargo install --locked cargo-deny"; exit 1; } + @echo "(advisories need the RustSec advisory DB, cloned on first run)" + cargo deny check test: ## Run the test suite cargo test --workspace diff --git a/README.md b/README.md index a66b0e7..461eaf0 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ The guides below are currently in Traditional Chinese. | [Architecture](./docs/architecture.md) | Task flow, system architecture, computer lifecycle | | [Interactive diagram](./docs/workflow.html) | Zoomable, searchable HTML chart; download and open | | [Operations](./docs/operations.md) | Resources, env vars, security, site checks, sudo | +| [Agent experience](./docs/agent-experience.md) | Turn limits, persistent terminal, live chat | | [Development](./docs/development.md) | Local dev, checks and tests, directory layout | | [Env example](./.env.example) | Environment variables and defaults | diff --git a/README.zh-TW.md b/README.zh-TW.md index 8f1add3..b79e244 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -111,6 +111,7 @@ npm run dev | [架構與流程](./docs/architecture.md) | 任務流程圖、系統架構、電腦生命週期狀態機 | | [互動流程圖](./docs/workflow.html) | 可縮放、搜尋的 HTML 圖表；下載後開啟 | | [部署與操作](./docs/operations.md) | 資源、環境變數、安全設定、網站驗證、sudo | +| [AI 使用體驗](./docs/agent-experience.md) | 輪次政策、持久終端機、聊天即時推送 | | [開發指南](./docs/development.md) | 本機開發、檢查與測試、目錄結構 | | [設定範例](./.env.example) | 環境變數與預設值 | diff --git a/a.md b/a.md new file mode 100644 index 0000000..0e0baa0 --- /dev/null +++ b/a.md @@ -0,0 +1,2210 @@ +# LazyBoy → Cua Driver Migration Plan + +> 2026-09-07 檢查：Phase 1 規格尚未全部勾完（生產預設仍是 legacy；takeover／錄製端到端未另開測）。opt-in Cua 已可在現有桌面容器使用，驗收見 [docs/cua-review.md](docs/cua-review.md)。本文件仍是目標規格，不能視為完成證明。 + +> **Purpose:** This document is an implementation specification for a coding agent. +> +> Repository: `https://code.30cm.net/daniel.w/lazyBoy` +> +> Cua: `https://github.com/trycua/cua` +> +> Cua docs: `https://cua.ai/docs` +> +> **Main principle:** Do **NOT** rewrite LazyBoy into Cua. Keep LazyBoy as the agent/workspace/product layer and replace the low-level computer-control implementation with Cua Driver incrementally. + +--- + +## 0. Agent Instructions + +You are modifying **LazyBoy**, a self-hosted AI agent workspace. + +The goal is to migrate the low-level GUI/browser computer-control implementation from custom CDP / AT-SPI / X11 code to **Cua Driver**, without breaking LazyBoy's existing product architecture. + +### Non-negotiable rules + +1. **Do not perform a Big Bang rewrite.** +2. **Do not remove the legacy computer-control implementation in the first PR.** +3. Keep LazyBoy's current public Agent tool schema stable unless absolutely necessary. +4. Keep: + - `computer_observe` + - `computer_act` + - `browser` + - `shell` + - file tools + - takeover flow + - memory + - schedule + - vault + - skills/playbooks +5. Cua must initially be an **implementation detail behind LazyBoy abstractions**. +6. Do not expose all Cua MCP tools directly to the LLM. +7. Do not replace `DockerSandbox`, `Supervisor`, noVNC, or persistent bot homes during Phase 1. +8. Add a feature flag / backend selector so legacy behavior can be restored immediately. +9. Every migrated action must have observable verification. +10. Do not guess Cua API/tool names from this document. + +### Cua API freshness rule + +Cua changes quickly. + +Before implementing anything: + +```bash +cua-driver --version +cua-driver doctor +cua-driver list-tools +``` + +For every Cua tool you plan to call: + +```bash +cua-driver describe +``` + +Use the schema reported by the installed Cua Driver version as the source of truth. + +Do not hard-code assumptions from old blog posts, old examples, or this document when the installed version differs. + +--- + +# 1. Current LazyBoy Architecture + +The current architecture is approximately: + +```text +User + │ + ▼ +React Web UI + │ + ▼ +crates/api + │ + ├── Agent loop + ├── Sessions / Runs + ├── Memory + ├── Schedule + ├── Vault + ├── Skills + ├── MCP + └── Tool definitions + │ + ▼ +SandboxProvider + │ + ▼ +crates/sandbox + │ + ▼ +DockerSandbox + │ + ▼ +crates/supervisor + │ + ▼ +Linux Desktop Container + │ + ├── XFCE + ├── Chromium + ├── Xvfb + ├── x11vnc + ├── websockify + └── controld + │ + ├── CDP + ├── AT-SPI + ├── X11 + └── screenshots +``` + +Relevant current components: + +```text +apps/web +crates/api +crates/contracts +crates/control +crates/controld +crates/harness +crates/sandbox +crates/supervisor +``` + +The existing `SandboxProvider` abstraction is an important migration seam and should be preserved. + +Current methods include approximately: + +```rust +async fn provision(...) +async fn prepare(...) +async fn capabilities(...) +async fn ensure_screen(...) +async fn reconnect(...) +async fn suspend(...) +async fn resume(...) +async fn execute(...) +async fn observe(...) +async fn act(...) +async fn connect_screen(...) +async fn list_files(...) +async fn read_file(...) +async fn write_file(...) +async fn stop(...) +async fn destroy(...) +``` + +Do not collapse this abstraction. + +--- + +# 2. Target Architecture + +Phase 1 target: + +```text + LazyBoy Web + │ + ▼ + LazyBoy API + │ + LazyBoy Agent Loop + │ + ┌────────────────┼────────────────┐ + │ │ │ + ▼ ▼ ▼ + Memory Skills Schedule + │ + ▼ + LazyBoy Tool Layer + │ + computer / browser / shell + │ + ▼ + ComputerController + abstraction + │ │ + ┌───────────┘ └────────────┐ + ▼ ▼ + LegacyController CuaController + │ │ + CDP / AT-SPI / X11 Cua Driver + │ + ┌──────────────┼──────────────┐ + ▼ ▼ ▼ + X11 AT-SPI Browser + │ + ▼ + Existing Desktop + Container +``` + +### Important + +Cua is initially used as the **computer-control backend**. + +It is **not** the Agent runtime. + +It is **not** the memory system. + +It is **not** the scheduler. + +It is **not** the user-facing product. + +It is **not** the first-phase sandbox lifecycle owner. + +--- + +# 3. What Must Stay in LazyBoy + +The following are LazyBoy product responsibilities and must remain owned by LazyBoy. + +## Keep unchanged unless required + +### Frontend + +```text +apps/web +``` + +Keep: + +- Chat UI +- remote desktop +- noVNC +- mobile controls +- takeover UI +- bot management +- session UI + +### API / Agent layer + +```text +crates/api +``` + +Keep: + +- Agent loop +- model interaction +- run management +- session management +- tool policy +- takeover workflow +- credential flow +- memory +- schedules +- skills +- MCP integrations + +### Harness + +```text +crates/harness +``` + +Keep model-provider support. + +Cua Driver must not decide which model LazyBoy uses. + +### Supervisor + +```text +crates/supervisor +``` + +Phase 1: keep current behavior. + +Keep: + +- Docker lifecycle +- pause/resume +- resource limits +- persisted bot homes +- screen lifecycle + +### Sandbox + +```text +crates/sandbox +``` + +Phase 1: keep `DockerSandbox`. + +Do **not** replace it with Cua Sandbox yet. + +--- + +# 4. What Cua Should Replace + +The long-term goal is to reduce LazyBoy-owned OS automation. + +Current files that are candidates for replacement or simplification: + +```text +crates/control/src/a11y.py +crates/control/src/a11y.rs + +crates/control/src/cdp.py +crates/control/src/cdp.rs + +crates/control/src/x11.rs +crates/control/src/screen.rs +crates/control/src/overlay.rs +``` + +Do not delete them during the initial migration. + +Instead place them behind a legacy backend. + +The responsibilities that should gradually move to Cua: + +- screenshots +- application/window enumeration +- accessibility tree +- native UI element actions +- pointer input +- keyboard input +- scrolling +- browser state +- browser semantic actions +- window-scoped control +- recording / trajectory capture + +--- + +# 5. Do NOT Expose Raw Cua to the LLM + +LazyBoy currently has a relatively compact Agent-facing tool surface. + +Preserve this. + +Preferred model: + +```text +LLM + │ + ▼ +LazyBoy tools + │ + ├── computer_observe + ├── computer_act + ├── browser + ├── shell + ├── file tools + └── request_takeover + │ + ▼ +LazyBoy policy / safety / vault / state + │ + ▼ +Cua adapter + │ + ▼ +Cua Driver +``` + +Avoid this: + +```text +LLM + │ + ▼ +50+ raw Cua tools + │ + ▼ +OS +``` + +Reasons: + +- larger tool schemas consume context +- LazyBoy loses policy control +- LazyBoy loses stable abstraction +- Cua version changes would leak into prompts +- takeover behavior becomes harder to control +- credential handling becomes harder to constrain +- Agent behavior becomes coupled to Cua implementation details + +Cua should initially behave like a device driver. + +--- + +# 6. Introduce a ComputerController Abstraction + +Create a low-level computer-control abstraction separate from sandbox lifecycle. + +Suggested location: + +```text +crates/control/src/controller.rs +``` + +Possible interface: + +```rust +#[async_trait::async_trait] +pub trait ComputerController: Send + Sync { + async fn health(&self, ctx: &ControlContext) + -> Result; + + async fn observe( + &self, + request: ObserveRequest, + ctx: &ControlContext, + ) -> Result; + + async fn act( + &self, + request: ActionRequest, + ctx: &ControlContext, + ) -> Result; + + async fn browser( + &self, + request: BrowserRequest, + ctx: &ControlContext, + ) -> Result; + + async fn start_recording( + &self, + request: RecordingRequest, + ctx: &ControlContext, + ) -> Result; + + async fn stop_recording( + &self, + ctx: &ControlContext, + ) -> Result; +} +``` + +Names may be adjusted to match existing LazyBoy contracts. + +Do not introduce unnecessary abstractions if equivalent types already exist. + +--- + +# 7. Backend Implementations + +Implement: + +```text +LegacyController +CuaController +``` + +Suggested layout: + +```text +crates/control/src/ +├── controller.rs +├── legacy/ +│ ├── mod.rs +│ ├── a11y.rs +│ ├── cdp.rs +│ ├── x11.rs +│ └── screen.rs +└── cua/ + ├── mod.rs + ├── client.rs + ├── translate.rs + ├── observe.rs + ├── actions.rs + ├── browser.rs + └── recording.rs +``` + +Do not spend the first PR moving all old files if it creates noisy diffs. + +It is acceptable to initially keep existing file locations and only add: + +```text +controller.rs +cua.rs +``` + +Refactor structure after behavior is stable. + +--- + +# 8. Backend Configuration + +Add a configuration value: + +```bash +LAZYBOY_COMPUTER_DRIVER=legacy +``` + +or: + +```bash +LAZYBOY_COMPUTER_DRIVER=cua +``` + +Default during initial rollout: + +```bash +LAZYBOY_COMPUTER_DRIVER=legacy +``` + +After Cua passes production-equivalent validation, the default may become: + +```bash +LAZYBOY_COMPUTER_DRIVER=cua +``` + +Do not remove the legacy option until Cua has passed the migration acceptance suite. + +If configuration already has a typed settings system, add this there instead of reading environment variables throughout the code. + +Suggested enum: + +```rust +pub enum ComputerDriver { + Legacy, + Cua, +} +``` + +--- + +# 9. Recommended Cua Integration Mode + +LazyBoy is an application with built-in computer-use. + +Preferred order for experimentation: + +## POC + +Use the Cua Driver CLI / daemon boundary first if it allows fast validation. + +Examples: + +```bash +cua-driver call '' +``` + +or a controlled local process interface. + +This proves Cua works inside the existing Linux desktop container. + +## Production integration + +Prefer a typed/stable integration boundary. + +Evaluate, based on the installed Cua release: + +1. direct in-process SDK if suitable for LazyBoy's process model +2. private worker / daemon +3. CLI JSON calls as fallback + +Do not choose MCP merely because Agent frameworks often use MCP. + +LazyBoy is embedding computer use inside a product. The internal integration does not have to look like the Agent-facing integration. + +### Rust note + +Cua Driver's core runtime is Rust-based, but public application SDK support may differ by release. + +Do not write custom unsafe bindings unless there is a strong reason. + +Prefer an officially supported application integration surface. + +--- + +# 10. Phase 0 — Discovery / Compatibility Check + +Before changing LazyBoy behavior, create a short engineering report. + +The coding agent must verify the actual runtime environment. + +Inside a real LazyBoy desktop container: + +```bash +echo "$DISPLAY" +ps aux +env | sort +``` + +Confirm: + +- X11 display exists +- XFCE session exists +- AT-SPI bus is available +- Chromium exists +- Cua Driver can start +- Cua Driver can capture the current display +- Cua Driver can enumerate apps/windows +- Cua Driver can operate in Xvfb +- Cua Driver can access Chromium in this environment + +Install Cua Driver using the current official installation method. + +Then run: + +```bash +cua-driver --version +cua-driver doctor +cua-driver list-tools +``` + +Save results in: + +```text +docs/cua-compatibility.md +``` + +Include: + +- installed Cua version +- Linux distribution +- display server +- Cua doctor output summary +- supported Cua tools relevant to LazyBoy +- missing dependencies +- known limitations + +### Stop condition + +If Cua cannot reliably control the existing `XFCE + Xvfb` environment, do **not** proceed with architecture replacement. + +Instead document the blocker first. + +--- + +# 11. Phase 1 — Five-Action POC + +Do not begin by migrating the whole control crate. + +First prove these five capabilities inside the existing LazyBoy desktop: + +1. screenshot +2. accessibility/window observation +3. click native GUI element +4. type text +5. browser semantic action + +Optional sixth: + +6. scroll + +Create a standalone POC path. + +Example: + +```text +scripts/cua-smoke-test.sh +``` + +or: + +```text +crates/control/examples/cua_smoke.rs +``` + +### POC flow + +Suggested test: + +```text +1. Open a native/simple XFCE application. +2. Observe window state. +3. Click an accessible element. +4. Type text. +5. Launch/use Chromium. +6. Navigate to example.com. +7. Read browser state. +8. Trigger a semantic browser interaction if available. +9. Capture final screenshot. +``` + +### POC success criteria + +All five core actions succeed 10 consecutive times without: + +- wrong window actions +- stale element actions +- focus corruption +- unexplained timeouts +- leaving Cua processes behind +- breaking noVNC control + +Do not migrate production paths until this passes. + +--- + +# 12. Phase 2 — Migrate `computer_observe` + +First production migration target: + +```text +computer_observe +``` + +Current LazyBoy observation must remain compatible with the Agent. + +Do not change the Agent prompt contract unless required. + +Cua result must be translated into: + +```rust +ComputerObservation +``` + +Preserve existing concepts where possible: + +- screenshot/image +- dimensions +- cursor +- active window +- UI elements + +### Translation layer + +Implement: + +```text +Cua state + ↓ +CuaObservationAdapter + ↓ +LazyBoy ComputerObservation +``` + +Do not expose Cua-native short-lived element identifiers directly as permanent LazyBoy identifiers. + +Element references may only be valid for a specific observation. + +Treat each observation as a snapshot. + +### Observation ID + +Strongly consider adding or preserving an observation/snapshot identifier. + +Example: + +```rust +pub struct ComputerObservation { + pub observation_id: String, + ... +} +``` + +If changing the contract is too invasive, keep this internal first. + +This will later help detect stale actions. + +--- + +# 13. Phase 3 — Migrate `computer_act` + +Preserve the existing LazyBoy `ComputerAction` DSL. + +Current behavior such as: + +- click +- move +- pointer down/up +- hover +- drag +- type +- keyboard +- wait +- semantic references + +should remain Agent-facing LazyBoy concepts. + +Create a translator: + +```text +LazyBoy ComputerAction + │ + ▼ + CuaActionTranslator + │ + ▼ + Cua Driver +``` + +Example conceptual mapping: + +```text +LazyBoy click(element) + → semantic Cua action when possible + +LazyBoy click(x, y) + → pixel Cua action + +LazyBoy type(text) + → Cua text input + +LazyBoy key(...) + → Cua key action + +LazyBoy scroll(...) + → Cua scroll +``` + +Exact tool names MUST come from: + +```bash +cua-driver list-tools +cua-driver describe ... +``` + +### Preserve semantic-first behavior + +Prefer: + +```text +Accessibility / browser semantic action +``` + +over: + +```text +screen coordinate +``` + +Coordinate action should be fallback, not default. + +--- + +# 14. Preserve LazyBoy Action Safety Logic + +Do not delete useful behavior from: + +```text +crates/control/src/actions.rs +``` + +Existing logic includes concepts such as: + +- action batch limits +- coordinate validation +- element lookup +- stale click prevention +- browser semantic-routing preference +- double-click expansion +- drag normalization + +These should become policy/translation logic above Cua. + +Desired layering: + +```text +Agent request + │ + ▼ +LazyBoy validation + │ + ▼ +LazyBoy action policy + │ + ▼ +Cua translation + │ + ▼ +Cua Driver +``` + +Cua is not a replacement for LazyBoy product policy. + +--- + +# 15. Stale Element Handling + +This is critical. + +Do not assume: + +```text +element 12 +``` + +from one Cua observation refers to the same UI element later. + +The migration must treat element references as snapshot-scoped. + +Preferred flow: + +```text +observe + ↓ +snapshot A + ↓ +Agent selects element + ↓ +act against snapshot A + ↓ +UI changes + ↓ +observe again + ↓ +snapshot B +``` + +If the UI changed materially, do not retry an old semantic reference blindly. + +### Retry rule + +When an action fails: + +1. re-observe +2. re-resolve the target +3. retry with bounded count + +Never loop the same stale action indefinitely. + +--- + +# 16. Phase 4 — Migrate Browser Control + +LazyBoy currently separates browser actions from generic computer actions. + +Keep that separation for the Agent. + +Agent-facing: + +```text +browser +``` + +Internal: + +```text +LazyBoy BrowserRequest + │ + ▼ +Cua Browser Adapter + │ + ▼ +Cua Driver browser tools +``` + +Cua browser control should replace custom CDP behavior gradually. + +### Important + +Preserve the current policy: + +```text +When semantic browser state is available, +prefer browser semantic actions over pixel clicking Chromium. +``` + +Do not make browser automation less reliable during migration. + +### Test cases + +At minimum test: + +- navigate URL +- inspect page +- click semantic element +- type into input +- scroll +- multiple tabs if LazyBoy currently depends on them +- file picker transition +- page refresh +- browser restart +- authenticated persistent profile + +--- + +# 17. Browser Profile Compatibility + +LazyBoy persists browser profiles per computer/bot. + +This is product-critical. + +Do not let Cua silently replace LazyBoy's profile with an ephemeral managed browser unless explicitly intended. + +Verify: + +- existing Chromium profile path remains usable +- cookies survive container pause/resume +- login sessions survive LazyBoy restart behavior as expected +- takeover user and Agent see the same session +- Cua attaches to the correct browser/window + +If Cua requires an explicit browser preparation/attach step, integrate that into LazyBoy lifecycle. + +Do not auto-create a separate profile that breaks existing saved logins. + +--- + +# 18. Phase 5 — Recording Integration + +LazyBoy has an important feature: + +```text +Human demonstration + ↓ +record context/actions + ↓ +model generates playbook + ↓ +future run resolves current UI +``` + +Preserve this architecture. + +Do **not** downgrade it into raw coordinate replay. + +Cua recording should be used as richer source data. + +Target: + +```text +Human / Agent demonstration + │ + ▼ + Cua trajectory + │ + ├── before state + ├── action + ├── after state + ├── screenshots + └── optional video + │ + ▼ + LazyBoy Skill Compiler + │ + ▼ + Semantic Playbook +``` + +Cua trajectory is evidence. + +LazyBoy skill/playbook is the reusable automation. + +### Do not do + +```text +record x=312,y=441 +replay x=312,y=441 forever +``` + +### Do + +Store semantic intent when possible: + +```text +Click the "Sign in" button +``` + +Then resolve it on the current screen during replay. + +--- + +# 19. Takeover Must Keep Working + +User takeover is a core LazyBoy feature. + +Migration acceptance requires: + +```text +Agent running + ↓ +request_takeover + ↓ +Agent stops issuing input + ↓ +User controls noVNC desktop + ↓ +User releases takeover + ↓ +Agent re-observes + ↓ +Agent resumes from current state +``` + +### Required rule + +After takeover ends: + +**Always perform a fresh observation before the Agent performs another UI action.** + +Never reuse pre-takeover element references. + +--- + +# 20. Credential / Vault Boundary + +Cua must not get broad access to LazyBoy secrets by default. + +Keep credential policy in LazyBoy. + +Preferred flow: + +```text +Agent wants login + │ + ▼ +LazyBoy checks: +- allowed domain +- saved credential exists +- HTTPS / policy + │ + ▼ +LazyBoy authorizes injection + │ + ▼ +Cua performs allowed typing/action +``` + +Do not make the Cua integration read the entire Vault. + +Secrets should not appear: + +- in command-line arguments +- in logs +- in Cua debug output +- in trajectory metadata +- in screenshots longer than unavoidable +- in error messages + +Audit recording behavior around passwords. + +If recording is active, ensure sensitive typing can be masked or recording paused. + +--- + +# 21. `controld` Migration Strategy + +Do not delete `crates/controld` during Phase 1. + +It is a useful compatibility boundary. + +Current conceptual API: + +```text +POST /observe +POST /act +``` + +Recommended first migration: + +```text +POST /observe + ↓ +selected ComputerController + ↓ +CuaController or LegacyController +``` + +```text +POST /act + ↓ +LazyBoy validation + ↓ +selected ComputerController + ↓ +CuaController or LegacyController +``` + +This keeps: + +```text +API +SandboxProvider +Supervisor +``` + +largely unchanged. + +Later, if Cua integration makes `controld` unnecessary, remove it in a dedicated architectural PR. + +Do not mix that cleanup into the initial migration. + +--- + +# 22. Docker Image Changes + +Add Cua dependencies to the LazyBoy desktop image. + +The coding agent must locate the actual Dockerfile(s) used for desktop computers. + +Do not assume the path. + +Changes may include: + +- Cua Driver install +- required X11 packages +- AT-SPI dependencies +- ffmpeg if recording/video is enabled +- runtime directories +- permissions +- PATH configuration + +Run: + +```bash +cua-driver doctor +``` + +inside the built desktop container as part of the smoke test. + +### Image versioning + +Pin a known-working Cua version for reproducible builds. + +Do not install uncontrolled nightly builds in the default production image. + +Optionally allow: + +```bash +LAZYBOY_CUA_CHANNEL=stable +LAZYBOY_CUA_VERSION= +``` + +or equivalent build args. + +--- + +# 23. Cua Process Lifecycle + +Do not start one uncontrolled global Cua process for every LazyBoy computer unless architecture requires it. + +Determine which process should own: + +- display connection +- accessibility session +- browser connection +- recordings +- Cua lifecycle + +For LazyBoy's existing architecture, the safest initial design is usually: + +```text +one desktop container + │ + ├── XFCE/Xvfb + ├── Chromium + ├── noVNC + ├── LazyBoy controld + └── Cua runtime/driver +``` + +The driver should only see/control that computer's desktop session. + +### Isolation + +Bot A must never control Bot B's display. + +Tests must explicitly verify isolation. + +--- + +# 24. Health Checks + +Add a Cua health check. + +Possible information: + +```rust +pub struct ControllerHealth { + pub backend: String, + pub version: Option, + pub healthy: bool, + pub degraded: bool, + pub details: Vec, +} +``` + +Surface useful failures: + +- driver not installed +- X11 unavailable +- AT-SPI unavailable +- screenshot unavailable +- browser unavailable +- incompatible Cua version + +Do not return generic: + +```text +computer failed +``` + +when a meaningful diagnosis is available. + +--- + +# 25. Fallback Behavior + +During migration: + +```text +LAZYBOY_COMPUTER_DRIVER=legacy +``` + +must work. + +For `cua` mode, avoid silent fallback for individual actions unless explicitly designed. + +Bad: + +```text +Cua click failed +→ silently xdotool click +``` + +This makes failures impossible to debug. + +Preferred: + +```text +Cua action failed +→ return classified error +→ Agent re-observes / retries / requests takeover +``` + +A feature-flag-level fallback is acceptable. + +An invisible per-action fallback is not. + +--- + +# 26. Logging / Observability + +Add structured logs around Cua calls. + +Include: + +- operation ID +- run ID +- bot ID +- screen ID +- backend +- Cua tool +- duration +- success/failure +- action route if Cua reports it +- observation ID if available + +Never log secret text. + +For typing actions: + +```text +text="" +length=12 +``` + +not: + +```text +text="actual-password" +``` + +--- + +# 27. Metrics + +If LazyBoy has metrics infrastructure, add: + +```text +computer_action_total +computer_action_failed_total +computer_observe_duration_ms +computer_action_duration_ms +computer_browser_action_duration_ms +computer_stale_reference_total +computer_takeover_total +cua_driver_restart_total +``` + +Useful labels: + +```text +backend +action_type +route +result +``` + +Avoid high-cardinality IDs such as `run_id` as metric labels. + +--- + +# 28. Testing Strategy + +## Unit tests + +Test translation only. + +Examples: + +```text +LazyBoy click semantic ref → expected Cua request + +LazyBoy coordinate click → expected Cua request + +LazyBoy type → expected Cua request + +Cua observation → LazyBoy ComputerObservation + +unsupported Cua response → classified error +``` + +Use fake/mocked Cua responses. + +--- + +## Contract tests + +The Agent-facing tool results should remain equivalent between: + +```text +LegacyController +CuaController +``` + +for common scenarios. + +Test: + +- observation shape +- action result shape +- error behavior +- browser results +- takeover interaction + +--- + +## Integration tests + +Run against a real desktop container. + +Test native GUI: + +```text +open app +observe +click +type +verify application state +``` + +Do not validate only that Cua returned `"ok"`. + +Validate independent application state. + +--- + +## Browser E2E + +Use a deterministic local test page rather than an external website. + +Create fixtures for: + +- button +- input +- checkbox +- select +- scroll area +- delayed DOM update +- modal +- new tab +- canvas fallback if needed + +Verify actual DOM/application state. + +--- + +# 29. Migration Acceptance Suite + +Cua backend is not considered ready until all of the following pass. + +## Desktop + +- [ ] screenshot works +- [ ] window enumeration works +- [ ] active window works +- [ ] accessibility elements work +- [ ] semantic click works +- [ ] coordinate click works +- [ ] type text works +- [ ] hotkey works +- [ ] scroll works +- [ ] drag works if supported +- [ ] desktop stays usable through noVNC + +## Browser + +- [ ] attach to correct Chromium +- [ ] use persistent LazyBoy profile +- [ ] navigate +- [ ] observe DOM/browser state +- [ ] semantic click +- [ ] type +- [ ] scroll +- [ ] refresh +- [ ] profile survives pause/resume +- [ ] same browser is visible to human takeover + +## Lifecycle + +- [ ] fresh container +- [ ] existing persisted container +- [ ] pause +- [ ] resume +- [ ] stop +- [ ] restart +- [ ] concurrent bots +- [ ] no cross-bot control + +## Agent flow + +- [ ] `computer_observe` +- [ ] `computer_act` +- [ ] `browser` +- [ ] `shell` +- [ ] takeover +- [ ] resume after takeover +- [ ] skill recording +- [ ] scheduled run + +--- + +# 30. Performance Benchmark + +Before defaulting to Cua, compare against legacy. + +Create a benchmark report: + +```text +docs/cua-benchmark.md +``` + +Measure: + +| Scenario | Legacy | Cua | Winner | +| -------------------- | -----: | --: | ------ | +| screenshot | | | | +| observe desktop | | | | +| native click | | | | +| type text | | | | +| browser snapshot | | | | +| browser click | | | | +| 20-step browser task | | | | + +Also measure: + +- tool-call count +- bytes returned to model +- screenshot count +- total model-visible observation size +- end-to-end task completion time +- failure rate + +The goal is not only faster pointer execution. + +The real goal is: + +```text +fewer Agent turns ++ +less context ++ +higher task success rate +``` + +--- + +# 31. Rollout Plan + +## Step A + +Add: + +```text +ComputerController +LegacyController +``` + +No behavior change. + +All tests must pass. + +--- + +## Step B + +Add Cua smoke test. + +No production traffic. + +--- + +## Step C + +Implement: + +```text +CuaController.observe +``` + +Feature flagged. + +--- + +## Step D + +Implement simple actions: + +```text +click +type +keyboard +scroll +``` + +--- + +## Step E + +Migrate browser state/actions. + +--- + +## Step F + +Add recording integration. + +--- + +## Step G + +Run acceptance suite and benchmark. + +--- + +## Step H + +Change default: + +```text +legacy +→ +cua +``` + +but keep legacy rollback. + +--- + +## Step I + +After stable operation, delete obsolete low-level code in separate PRs. + +Possible deletion candidates: + +```text +a11y.py +a11y.rs +cdp.py +cdp.rs +x11.rs +screen implementation portions +``` + +Only delete code proven unused. + +--- + +# 32. Suggested PR Sequence + +Keep PRs small. + +### PR 1 + +```text +refactor(control): introduce pluggable ComputerController +``` + +- add interface +- wrap legacy +- no behavior change +- tests + +### PR 2 + +```text +build(desktop): install and validate Cua Driver +``` + +- image changes +- doctor +- smoke test +- compatibility doc + +### PR 3 + +```text +feat(control): add Cua observation backend +``` + +### PR 4 + +```text +feat(control): route computer actions through Cua +``` + +### PR 5 + +```text +feat(browser): add Cua browser adapter +``` + +### PR 6 + +```text +feat(skills): ingest Cua trajectories for demonstrations +``` + +### PR 7 + +```text +test(control): add Cua acceptance and benchmark suite +``` + +### PR 8 + +```text +chore(control): make Cua the default backend +``` + +### Later + +```text +chore(control): remove obsolete legacy OS automation +``` + +--- + +# 33. Error Model + +Map Cua failures to typed LazyBoy errors. + +Suggested categories: + +```rust +pub enum ControlError { + DriverUnavailable, + DriverUnhealthy, + DisplayUnavailable, + AccessibilityUnavailable, + BrowserUnavailable, + TargetNotFound, + StaleReference, + PermissionDenied, + Timeout, + Unsupported, + Busy, + InvalidAction, + Internal(String), +} +``` + +Do not expose Cua raw errors directly to the Agent when a stable LazyBoy error can represent them. + +Log the raw cause internally. + +--- + +# 34. Retry Policy + +Do not blindly retry UI actions. + +Recommended: + +### Observation failure + +Retry a small bounded number if transport/runtime error appears transient. + +### Semantic target not found + +```text +re-observe +→ re-resolve +→ retry once +``` + +### Stale target + +```text +re-observe +→ never retry original snapshot ref directly +``` + +### Pixel miss + +Do not repeat the same click indefinitely. + +Preserve LazyBoy's stale-click protection concept. + +### Driver crash + +Restart driver/runtime if safe, then require a fresh observation. + +--- + +# 35. Screenshot Policy + +Avoid sending screenshots to the model when structured UI state is sufficient. + +Prefer: + +```text +semantic / accessibility observation +``` + +Use screenshots when: + +- layout matters +- semantic state is incomplete +- canvas/WebGL is involved +- visual verification is needed + +This reduces: + +- latency +- model context +- vision token cost +- accidental secret exposure + +--- + +# 36. Cua Permission Policy + +If the chosen Cua deployment mode supports permission policies, use allow-list behavior. + +LazyBoy only needs a subset of Cua capabilities. + +Allow only required operations. + +Example conceptual set: + +```text +health/doctor-like observation +window/app listing +screenshot +window state +click +type +key +scroll +browser state +browser click +browser type +browser navigation +recording +``` + +Do not allow unrelated capabilities automatically. + +Use actual current Cua tool names from the installed version. + +--- + +# 37. Security Requirements + +- Cua must not access the host Docker socket from Agent desktops. +- Each bot must remain isolated. +- Keep LazyBoy supervisor on internal network. +- Keep `controld` localhost/internal where applicable. +- Do not expose Cua daemon ports publicly. +- Do not log passwords/tokens. +- Verify trajectory storage permissions. +- Verify screenshots are covered by LazyBoy retention policy. +- Do not allow an Agent to change Cua policies. +- Do not let the Agent select another bot's display/session. +- Cua executable/version should be controlled by LazyBoy image/build process. + +--- + +# 38. Multi-Screen Considerations + +LazyBoy already has screen concepts: + +```text +screen_lease_id +screen_id +screen_slot +display +``` + +Do not discard these. + +Cua must be bound to the LazyBoy-selected display/screen. + +Verify multi-screen behavior before enabling Cua for multi-screen bots. + +If Cua does not safely support LazyBoy's current multi-screen semantics: + +- support one screen first +- return explicit capability information +- do not fake support + +--- + +# 39. Cua Capability Mapping + +LazyBoy should expose backend capabilities. + +Suggested internal structure: + +```rust +pub struct ComputerCapabilities { + pub multi_screen: bool, + pub semantic_desktop: bool, + pub semantic_browser: bool, + pub pixel_actions: bool, + pub recording: bool, + pub background_input: bool, +} +``` + +Only add fields if needed and compatible with existing contract evolution. + +Use capability checks instead of OS-name conditionals where possible. + +Bad: + +```rust +if linux { + ... +} +``` + +Better: + +```rust +if capabilities.semantic_browser { + ... +} +``` + +--- + +# 40. Future Phase — Cua Sandbox + +Do **not** implement this during the Driver migration. + +Later, LazyBoy may support: + +```text +SandboxProvider + │ + ├── LazyBoyDocker + └── CuaSandbox +``` + +Possible future computer types: + +```text +Lightweight Linux +Linux VM +Windows VM +macOS VM +Cloud Computer +``` + +This should be a separate project after Cua Driver migration is stable. + +The current `SandboxProvider` abstraction should make this possible. + +--- + +# 41. Future Phase — Windows / macOS + +One strategic reason for Cua is avoiding custom implementations for: + +```text +Linux AT-SPI/X11 +Windows UIA/input/capture +macOS AX/input/capture +``` + +Do not add Windows/macOS support during the initial Linux migration. + +First ensure the LazyBoy control contract is OS-neutral. + +Then add backend capabilities and sandbox providers later. + +--- + +# 42. Definition of Done — Phase 1 + +Phase 1 is complete when: + +1. LazyBoy can run with: + +```bash +LAZYBOY_COMPUTER_DRIVER=legacy +``` + +and: + +```bash +LAZYBOY_COMPUTER_DRIVER=cua +``` + +2. Existing Agent tool schemas remain compatible. + +3. Cua works in the existing LazyBoy Linux desktop container. + +4. `computer_observe` works through Cua. + +5. `computer_act` core actions work through Cua. + +6. Browser semantic operations work through Cua. + +7. noVNC human takeover still works. + +8. Persistent Chromium sessions still work. + +9. Docker pause/resume still works. + +10. Multi-bot isolation is verified. + +11. Legacy backend remains available for rollback. + +12. No secret is exposed in logs or recording metadata. + +13. Acceptance tests pass. + +14. Benchmark results are documented. + +--- + +# 43. First Task for the Coding Agent + +Do this first and nothing larger: + +## Task + +Create a branch: + +```text +feat/cua-driver-poc +``` + +Then: + +1. inspect current LazyBoy computer-control flow +2. identify actual desktop Dockerfile/image entrypoint +3. install a pinned stable Cua Driver in that image +4. make no Agent-facing schema changes +5. create a Cua smoke test that runs inside a LazyBoy desktop +6. verify: + - `cua-driver --version` + - `cua-driver doctor` + - screenshot + - window/accessibility observation + - native click + - typing + - Chromium/browser state +7. write results to: + +```text +docs/cua-compatibility.md +``` + +8. stop after the POC +9. do not delete any legacy control code +10. report blockers before starting the controller refactor + +### Expected output + +The first PR should answer only: + +> "Can Cua Driver reliably control the existing LazyBoy XFCE + Xvfb desktop container?" + +If the answer is yes, proceed to the next PR. + +--- + +# 44. Second Task After POC Passes + +Create: + +```text +ComputerController +``` + +with: + +```text +LegacyController +CuaController +``` + +Initially make all production requests use: + +```text +LegacyController +``` + +Then route only test/flagged traffic to: + +```text +CuaController +``` + +Do not migrate browser and recording in the same PR. + +--- + +# 45. Architecture Decision + +The intended ownership after migration is: + +| Concern | Owner | +| ----------------------------------- | ------- | +| Product UI | LazyBoy | +| Agent loop | LazyBoy | +| Model provider | LazyBoy | +| Memory | LazyBoy | +| Schedules | LazyBoy | +| Vault | LazyBoy | +| Takeover | LazyBoy | +| Skills/playbooks | LazyBoy | +| Sandbox lifecycle (Phase 1) | LazyBoy | +| Docker resources | LazyBoy | +| Remote desktop/noVNC | LazyBoy | +| Computer observation | Cua | +| Native UI control | Cua | +| Pointer/keyboard | Cua | +| Browser semantic control | Cua | +| OS-specific accessibility | Cua | +| Trajectory capture | Cua | +| Workflow abstraction from recording | LazyBoy | + +This is the central design decision. + +Do not invert it. + +--- + +# 46. Final Principle + +LazyBoy's competitive/product value is: + +```text +Agent workspace +Multi-agent +Persistent computers +Human takeover +Memory +Schedules +Vault +Skills +Demo → reusable workflow +Web/mobile UX +Model choice +``` + +The goal of using Cua is to stop spending LazyBoy engineering effort on: + +```text +X11 automation +AT-SPI edge cases +CDP plumbing +screen capture +OS-specific input +window enumeration +cross-platform UI automation +``` + +Build LazyBoy **on top of** Cua. + +Do not turn LazyBoy **into** Cua. + +--- + +# References + +LazyBoy: + +- https://code.30cm.net/daniel.w/lazyBoy +- https://code.30cm.net/daniel.w/lazyBoy/src/branch/main/docs/architecture.md +- https://code.30cm.net/daniel.w/lazyBoy/src/branch/main/crates/control +- https://code.30cm.net/daniel.w/lazyBoy/src/branch/main/crates/control/src/sandbox.rs +- https://code.30cm.net/daniel.w/lazyBoy/src/branch/main/crates/control/src/actions.rs +- https://code.30cm.net/daniel.w/lazyBoy/src/branch/main/crates/sandbox + +Cua: + +- https://github.com/trycua/cua +- https://cua.ai/docs +- https://cua.ai/docs/how-to-guides/driver/install +- https://cua.ai/docs/concepts/choose-a-cua-driver-integration +- https://cua.ai/docs/how-to-guides/driver/connect-your-agent +- https://cua.ai/docs/reference/cua-driver/cli-reference +- https://cua.ai/docs/reference/cua-driver/mcp-tools +- https://cua.ai/docs/how-to-guides/driver/record-and-render-a-trajectory +- https://cua.ai/docs/how-to-guides/driver/restrict-tool-access diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index 7cf2c66..b17acf9 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -15,18 +15,36 @@ import visibility from "react-useanimations/lib/visibility"; import visibility2 from "react-useanimations/lib/visibility2"; import searchToX from "react-useanimations/lib/searchToX"; import { api, ApiError } from "./api"; +import { createCoalescer, subscribeToSession } from "./live"; +import { HANDOFF_MS, VEIL_FADE_MS, handoffRemaining, keepScreenUrl, nextVeil, viewOnlyFor, viewerPath, type Veil } from "./handoff"; import { Avatar, AvatarLookProvider, AvatarStack, BLOBATAR_BACKGROUNDS, BLOBATAR_EXPRESSIONS, BLOBATAR_SHAPES, DEFAULT_LOOK, persistBlobatarShape, readAvatarLooks, resolveBlobatarShape, writeAvatarLook, type AvatarBackground, type AvatarExpression, type AvatarLook } from "./avatar"; import { dateLocale, getLocale, listJoin, setLocale, t, useLocale, type MessageKey } from "./i18n"; import type { AvatarShape, Bot, ComputerMode, ComputerStatus, FileSkill, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, MessageFile, ModelProviderId, Playbook, PlaybookInput, PlaybookStep, Room, RoomMember, Session, TaughtSkill, VoiceSettings, WorkspaceSettings } from "./types"; import { ChatMarkdown, CopyMessageButton } from "./markdown"; +import { RunProbe, errorActions, errorTitle } from "./run-monitor"; import { ScheduleEditor, ScheduleList, cronFromPreset, defaultCronPreset, presetFromCron, scheduleWhen, type CronPreset, type ScheduleItem } from "./schedule"; import { CallOverlay, PhoneIcon } from "./call"; import { VoiceSettingsDialog } from "./voice-settings"; -const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,busyStep:null,usingComputer:false,waitingRunId:null,waitingSessionId:null,queuedRuns:0,display:null,profileMode:"per-bot",screenAvailable:false}; +const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",sharedInput:true,controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,busyStep:null,usingComputer:false,waitingRunId:null,waitingSessionId:null,queuedRuns:0,display:null,profileMode:"per-bot",screenAvailable:false}; const SESSION_STORE="lazyboy.sessionByBot"; const PANE_STORE="lazyboy.rightPane"; const WORKSPACE_STORE="lazyboy.workspace"; +// Refresh policy: the session's event stream decides when the transcript is +// stale, so the timer is only a backstop - it runs slowly while the stream is +// up and speeds back up if the stream drops. One turn can move several events, +// which is why they settle into a single refresh instead of one fetch each. +// The backstop is also the only freshness guarantee for the computer banner: +// computers have no events of their own, so this interval is how long a state +// change nobody sent a message for (idle park, a crash, another user taking +// control) can stay invisible. Four seconds is still lighter than the pre-push +// tick, which paid for the whole transcript plus a heartbeat every two. +const EVENT_SETTLE_MS=120; +const LIVE_POLL_MS=4000; +const FALLBACK_POLL_MS=2000; +// A heartbeat buys a 15 minute control lease and keeps the 10 minute idle +// cutoff from parking the computer, so a minute between beats is a wide margin. +const HEARTBEAT_MS=60000; type RightPart="computer"|"memory"|"settings"|"plugins"|"accounts"; type AccountDialog="phone"|"settings"|"model"|"voice"|"about"|"help"|"feedback"|null; function readSessionStore():Record{try{const raw=localStorage.getItem(SESSION_STORE);return raw?JSON.parse(raw) as Record:{}}catch{return {}}} @@ -36,6 +54,11 @@ type WorkspacePrefs={name:string;showHidden:boolean}; function isDefaultWorkspaceName(name?:string|null){return !name||name==="Local workspace"||name==="本機工作區"} function readWorkspace():WorkspacePrefs{try{const raw=localStorage.getItem(WORKSPACE_STORE);if(!raw)return{name:t("localWorkspace"),showHidden:false};const value=JSON.parse(raw) as {name?:string;showHidden?:boolean};const name=value.name?.trim();return{name:isDefaultWorkspaceName(name)?t("localWorkspace"):name||t("localWorkspace"),showHidden:Boolean(value.showHidden)}}catch{return{name:t("localWorkspace"),showHidden:false}}} +function MessageTime({value}:{value:string}){ + const date=new Date(value); + if(!Number.isFinite(date.getTime()))return null; + return ; +} function WorkspaceAvatar({name}:{name:string}){const parts=name.trim().split(/\s+/).filter(Boolean);const initials=(parts.length>1?parts.map(part=>part[0]).join(""):parts[0]?.slice(0,2)||"LB").slice(0,2).toUpperCase();return } function modeLabel(mode:ComputerMode){return mode==="team"?t("sharedComputer"):t("privateComputer")} function stateLabel(state:ComputerStatus["state"]){return ({stopped:t("stopped"),booting:t("booting"),running:t("running"),suspended:t("suspended"),error:t("error")})[state]} @@ -67,7 +90,9 @@ function localizeStep(step?:string|null){ } function localizeError(message:string){ if(message==="示範進行中：先按「完成示範」或「取消」，再送訊息。")return t("teachInProgress"); - if(message==="AI 回應逾時（120 秒）")return t("aiTimeout"); + if(message==="AI 回應逾時（150 秒）")return t("aiTimeout"); + if(message==="run is not retryable")return t("errorRetryFailed"); + if(message==="Stop the task first")return t("takeoverBusy"); return message; } function hudLabel(computer:ComputerStatus,connecting:boolean,handingOff:boolean){ @@ -88,7 +113,9 @@ function readAsBase64(file:File){return new Promise((resolve,reject)=>{c function formatBytes(size:number){if(size<1024)return `${size} B`;if(size<1024*1024)return `${Math.round(size/102.4)/10} KB`;return `${Math.round(size/104857.6)/10} MB`} function fileExt(name:string){const dot=name.lastIndexOf(".");const ext=dot>=0?name.slice(dot+1).replace(/[^a-z0-9]/gi,""):"";return (ext||"FILE").slice(0,4).toUpperCase()} function messageFiles(blocks:unknown):MessageFile[]{if(!Array.isArray(blocks))return [];return blocks.flatMap(block=>{if(!block||typeof block!=="object")return [];const value=block as {kind?:string;name?:string;mimeType?:string;size?:number};if(value.kind!=="file"&&value.kind!=="image")return [];return [{kind:value.kind,name:value.name||"file",mimeType:value.mimeType,size:value.size}]})} -function chipBlocks(blocks:unknown){if(!Array.isArray(blocks))return [] as {kind:string;site?:string;why?:string;name?:string;human?:string;cron?:string}[];return blocks.flatMap(block=>{if(!block||typeof block!=="object")return [];const value=block as {kind?:string;site?:string;why?:string;name?:string;human?:string;cron?:string};if(value.kind==="login"||value.kind==="schedule"||value.kind==="scheduleRun")return [value];return []})} +type MessageChip={kind:string;site?:string;why?:string;name?:string;human?:string;cron?:string;reason?:string;turns?:number;limit?:number;code?:string;retryable?:boolean;runId?:string;turn?:number;step?:string|null}; +function chipBlocks(blocks:unknown){if(!Array.isArray(blocks))return [] as MessageChip[];return blocks.flatMap(block=>{if(!block||typeof block!=="object")return [];const value=block as MessageChip;if(value.kind==="login"||value.kind==="schedule"||value.kind==="scheduleRun"||value.kind==="resume"||value.kind==="error")return [value];return []})} +function resumeTitle(reason?:string){return reason==="budget_exhausted"?t("resumeBudget"):reason==="loop_detected"?t("resumeLoop"):t("resumeMidTask")} function isAutoAttachCaption(body:string,files:MessageFile[]){const text=body.trim();if(!files.length)return false;if(!text)return true;return files.some(file=>text===file.name||text===`附件 ${file.name}`||text===`Attached ${file.name}`||text===t("attachedFile",{name:file.name}))} function isAutoScheduleCaption(body:string,chips:{kind?:string}[]){if(!chips.some(chip=>chip.kind==="schedule"||chip.kind==="scheduleRun"))return false;const text=body.trim();return /^\[排程(試跑)?\]/.test(text)||/^\[Schedule( test)?\]/i.test(text)} function FileCard({file,preview,onRemove}:{file:{name:string;size?:number};preview?:string|null;onRemove?:()=>void}){const ext=fileExt(file.name);return
{preview?:}
{file.name}{typeof file.size==="number"?formatBytes(file.size):ext}
{onRemove&&}
} @@ -113,7 +140,7 @@ export function App(){ const [createOpen,setCreateOpen]=useState(false); const [createMenuOpen,setCreateMenuOpen]=useState(false); const [groupOpen,setGroupOpen]=useState(false); const [deleteOpen,setDeleteOpen]=useState(false); const [computerOpen,setComputerOpen]=useState(false); const paneStart=readPaneStore(); const [rightCollapsed,setRightCollapsed]=useState(paneStart.collapsed); const [rightPart,setRightPart]=useState(paneStart.part); - const [sessionMenuOpen,setSessionMenuOpen]=useState(false); const [clearOpen,setClearOpen]=useState(false); const [sessionToDelete,setSessionToDelete]=useState(null); const [remembered,setRemembered]=useState>({}); + const [sessionMenuOpen,setSessionMenuOpen]=useState(false); const [clearOpen,setClearOpen]=useState(false); const [sessionToDelete,setSessionToDelete]=useState(null); const [remembered,setRemembered]=useState>({}); const [resumedChips,setResumedChips]=useState>({}); const [retriedRuns,setRetriedRuns]=useState>({}); const [mobileNav,setMobileNav]=useState(false); const [error,setError]=useState(null); const [busy,setBusy]=useState(false); useEffect(()=>{ if(!mobileNav)return; @@ -140,6 +167,11 @@ export function App(){ const desktopFrameRef=useRef(null); const holderRef=useRef(computer.controlHolder); const paneBotRef=useRef(null); const skipHandoffRef=useRef(true); const [desktopReady,setDesktopReady]=useState(false); const [handingOff,setHandingOff]=useState(false); + // The holder the server has not agreed to yet, when the current handoff + // started, and a guard so a fast double click cannot fight itself over the + // mouse. All three exist so the click, not the round trip, owns the answer. + const expectedHolderRef=useRef(null); const handoffAtRef=useRef(0); const controlBusyRef=useRef(false); const [controlBusy,setControlBusy]=useState(false); + const [veil,setVeil]=useState({label:null,leaving:false}); const [pendingFiles,setPendingFiles]=useState([]); const messageEndRef=useRef(null); const sentHistoryRef=useRef([]); const historyIndexRef=useRef(null); const historyDraftRef=useRef(""); @@ -157,7 +189,11 @@ export function App(){ const lastMessageId=messages[messages.length-1]?.id||""; // The bot is parked in waiting_takeover: nothing moves (including queued // messages) until the human releases the screen, so say so loudly. + const desktopInteractive=computer.screenAvailable&&!viewOnlyFor(computer.controlHolder,computer.sharedInput); const pausedForUser=computer.takeoverRequested&&workingMembers.length===0&&(!computer.waitingSessionId||computer.waitingSessionId===activeSessionId||Boolean(activeRoom)); + // Only the newest assistant reply can still be "continued": an older pause + // belongs to a run that has already moved on. + const lastAssistantSeq=messages.reduce((max,message)=>message.role==="assistant"?Math.max(max,message.seq??0):max,0); // Teaching by demonstration: one recording at a time per bot, then a draft // playbook the human names and saves before it becomes a real skill. const teaching=skills.find(skill=>skill.status==="recording")||null; @@ -190,7 +226,18 @@ export function App(){ const status=await api(`/api/computer/${computerBot}/status`);if(refreshSeq!==refreshSeqRef.current)return; const [nextMessages,screen,nextSkills]=await Promise.all([messagesJob,status.state==="running"?api<{url:string|null}>(`/api/computer/${computerBot}/screen`).catch(()=>({url:null})):Promise.resolve({url:null}),skillsJob]); if(refreshSeq!==refreshSeqRef.current)return; - setBusyMembers(nextBusy);setComputer(status);setMessages(nextMessages);setSkills(nextSkills);setScreenUrl(status.botId===computerBot?screen.url:null) + setBusyMembers(nextBusy);setComputer(status);setMessages(nextMessages);setSkills(nextSkills); + // A missing url is not a dead desktop. While it boots or wakes the mounted + // frame keeps its VNC session instead of blacking out and reconnecting. + setScreenUrl(current=>status.botId===computerBot?keepScreenUrl(current,screen.url,status.state):null); + // Agreement with the server is what ends a handoff. The beat has a floor so + // the swap is still visible on a LAN, and the ceiling timer still covers a + // reply that never lands. + if(expectedHolderRef.current&&status.controlHolder===expectedHolderRef.current){ + expectedHolderRef.current=null; + const wait=handoffRemaining(handoffAtRef.current,Date.now()); + if(wait>0)window.setTimeout(()=>setHandingOff(false),wait);else setHandingOff(false); + } },[activeId,activeRoomId,activeSessionId]); useEffect(()=>{loadBots().catch(e=>{if(e instanceof ApiError&&e.status===401)setAuthRequired(true);else setError(localizeError(e.message))})},[loadBots]); useEffect(()=>{if(!voiceSettings?.enabled)setCallOpen(false)},[voiceSettings?.enabled]); @@ -201,13 +248,30 @@ export function App(){ useEffect(()=>{historyIndexRef.current=null;historyDraftRef.current="";setPendingFiles(current=>{current.forEach(file=>file.preview&&URL.revokeObjectURL(file.preview));return []})},[activeSessionId]); useEffect(()=>{if(!plusOpen)setSkillQuery("")},[plusOpen]); useEffect(()=>{messageEndRef.current?.scrollIntoView({block:"end",behavior:"smooth"})},[activeSessionId,lastMessageId,workingMembers.length,pausedForUser]); - useEffect(()=>{if(!activeSessionId||(!activeId&&!activeRoomId)){refreshSeqRef.current+=1;setMessages([]);setScreenUrl(null);if(!activeId&&!activeRoomId)setComputer(blankComputer);return}setScreenUrl(null);refresh().catch(e=>setError(localizeError(e.message)));const timer=setInterval(()=>{refresh().catch(()=>{});const beat=roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||activeId;if(beat)api(`/api/computer/${beat}/heartbeat`,{method:"POST",body:"{}"}).catch(()=>{})},2000);return()=>{clearInterval(timer);refreshSeqRef.current+=1}},[activeId,activeRoomId,activeSessionId,refresh]); - useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard?.writeText(text).then(()=>setClipboardStatus(t("clipboardSynced"))).catch(()=>setClipboardStatus(t("clipboardSyncBlocked")))}if(event.data.type==="lazyboy-copy-request"&&computer.controlHolder==="user")void copySelection();if(event.data.type==="lazyboy-paste-text"&&typeof event.data.text==="string")pasteText(event.data.text);if(event.data.type==="lazyboy-mobile-key"&&typeof event.data.key==="string"&&["Return","BackSpace","Tab","Escape","Left","Right","Up","Down"].includes(event.data.key))queueDesktopInput({kind:"key",key:event.data.key});if(event.data.type==="lazyboy-paste-request"&&computer.controlHolder==="user")setClipboardOpen(true);if(event.data.type==="lazyboy-desktop-ready")setDesktopReady(true);if(event.data.type==="lazyboy-desktop-lost")setDesktopReady(false)};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)}); + // Live transcript: the event stream says when to look, so a reply appears as + // soon as it is written. Polling stays as a backstop that cannot starve the + // database, and a hidden tab stops re-reading until it comes back to the + // foreground - which is itself a "what happened" moment worth refreshing on. + useEffect(()=>{ + if(!activeSessionId||(!activeId&&!activeRoomId)){refreshSeqRef.current+=1;setMessages([]);setScreenUrl(null);if(!activeId&&!activeRoomId)setComputer(blankComputer);return} + setScreenUrl(null); + refresh().catch(e=>setError(localizeError(e.message))); + const settle=createCoalescer(()=>{if(!document.hidden)refresh().catch(()=>{})},EVENT_SETTLE_MS); + let live=false,opened=false,timer=0; + const tick=()=>{if(!document.hidden)refresh().catch(()=>{});timer=window.setTimeout(tick,live?LIVE_POLL_MS:FALLBACK_POLL_MS)}; + timer=window.setTimeout(tick,FALLBACK_POLL_MS); + const heartbeat=window.setInterval(()=>{const beat=roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||activeId;if(beat)api(`/api/computer/${beat}/heartbeat`,{method:"POST",body:"{}"}).catch(()=>{})},HEARTBEAT_MS); + const feed=subscribeToSession(activeSessionId,()=>settle.kick(),{onStatus:connected=>{live=connected;if(connected&&opened)refresh().catch(()=>{});opened=true}}); + const resume=()=>{if(!document.hidden)refresh().catch(()=>{})}; + document.addEventListener("visibilitychange",resume); + return()=>{window.clearTimeout(timer);window.clearInterval(heartbeat);document.removeEventListener("visibilitychange",resume);feed.close();settle.cancel();refreshSeqRef.current+=1}; + },[activeId,activeRoomId,activeSessionId,refresh]); + useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard?.writeText(text).then(()=>setClipboardStatus(t("clipboardSynced"))).catch(()=>setClipboardStatus(t("clipboardSyncBlocked")))}if(event.data.type==="lazyboy-copy-request"&&desktopInteractive)void copySelection();if(event.data.type==="lazyboy-paste-text"&&typeof event.data.text==="string")pasteText(event.data.text);if(event.data.type==="lazyboy-mobile-key"&&typeof event.data.key==="string"&&/^(?:(?:ctrl|alt)\+)?(?:Return|BackSpace|Tab|Escape|Left|Right|Up|Down|[acvz])$/.test(event.data.key))queueDesktopInput({kind:"key",key:event.data.key});if(event.data.type==="lazyboy-paste-request"&&desktopInteractive)setClipboardOpen(true);if(event.data.type==="lazyboy-desktop-ready")setDesktopReady(true);if(event.data.type==="lazyboy-desktop-lost")setDesktopReady(false)};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)}); useEffect(()=>{setDesktopReady(false)},[screenUrl,paneBotId]); - useEffect(()=>{if(skipHandoffRef.current){skipHandoffRef.current=false;holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId;return}if((holderRef.current!==computer.controlHolder||paneBotRef.current!==paneBotId)&&computer.state==="running")setHandingOff(true);holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId},[computer.controlHolder,paneBotId,computer.state]); - useEffect(()=>{if(!handingOff)return;const timer=setTimeout(()=>setHandingOff(false),1600);return()=>clearTimeout(timer)},[handingOff]); - useEffect(()=>{const frame=desktopFrameRef.current;if(!frame?.contentWindow||!screenUrl)return;frame.contentWindow.postMessage({type:"lazyboy-view-only",viewOnly:computer.controlHolder!=="user"},location.origin)},[computer.controlHolder,screenUrl,desktopReady]); - useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||event.data?.type!=="lazyboy-request-control"||!paneBotId)return;void action(()=>api(`/api/computer/${paneBotId}/takeover`,{method:"POST",body:"{}"}))};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)},[paneBotId]); + useEffect(()=>{if(skipHandoffRef.current){skipHandoffRef.current=false;holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId;return}if((holderRef.current!==computer.controlHolder||paneBotRef.current!==paneBotId)&&computer.state==="running"&&!computer.sharedInput)setHandingOff(true);holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId},[computer.controlHolder,paneBotId,computer.state,computer.sharedInput]); + useEffect(()=>{if(!handingOff)return;handoffAtRef.current=Date.now();const timer=setTimeout(()=>{expectedHolderRef.current=null;setHandingOff(false)},HANDOFF_MS);return()=>clearTimeout(timer)},[handingOff]); + useEffect(()=>{const frame=desktopFrameRef.current;if(!frame?.contentWindow||!screenUrl)return;frame.contentWindow.postMessage({type:"lazyboy-view-only",viewOnly:!desktopInteractive},location.origin)},[desktopInteractive,screenUrl,desktopReady]); + useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||event.data?.type!=="lazyboy-request-control"||!paneBotId)return;if(computer.sharedInput){pushViewOnly(!desktopInteractive);return}void setControl("user")};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)},[paneBotId,computer.sharedInput,desktopInteractive]); useEffect(()=>{const close=(event:MouseEvent)=>{const target=event.target;if(target instanceof Element&&target.closest(".create-menu-wrap,.account-wrap,.session-picker,.context-menu,.plus-menu-wrap"))return;setContext(null);setRoomContext(null);setSessionMenuOpen(false);setAccountOpen(false);setCreateMenuOpen(false);setPlusOpen(false)};window.addEventListener("click",close);return()=>window.removeEventListener("click",close)},[]); useEffect(()=>{const onKey=(event:KeyboardEvent)=>{if(event.key!=="Escape")return;setAccountOpen(false);setAccountDialog(null);setCreateMenuOpen(false);setPlusOpen(false);setTeachOpen(false);setEditingSkillId(null);setSessionMenuOpen(false);setContext(null);setRoomContext(null)};window.addEventListener("keydown",onKey);return()=>window.removeEventListener("keydown",onKey)},[]); useEffect(()=>{setWorkspaceName(name=>isDefaultWorkspaceName(name)?t("localWorkspace"):name)},[locale]); @@ -220,10 +284,9 @@ export function App(){ async function openLoginScreen(){ const id=paneBot?.id||active?.id;if(!id)return; setRightPart("computer");setRightCollapsed(false);setComputerOpen(true); - await action(async()=>{ - try{await api(`/api/computer/${id}/boot`,{method:"POST",body:"{}"})}catch{/* already up */} - await api(`/api/computer/${id}/takeover`,{method:"POST",body:"{}"}).catch(()=>{}); - }); + await action(async()=>{try{await api(`/api/computer/${id}/boot`,{method:"POST",body:"{}"})}catch{/* already up */}}); + // Opening the shared screen never pauses a running task. + if(!computer.sharedInput)await setControl("user",id); } async function reloadSchedules(){if(!paneBotId)return;setSchedules(await api(`/api/bots/${paneBotId}/schedules`))} async function saveScheduleDraft(){ @@ -261,13 +324,18 @@ export function App(){ function composerKeyDown(event:ReactKeyboardEvent){if(event.nativeEvent.isComposing||event.key==="Process")return;if(slashSuggestions.length){if(event.key==="Escape"){event.preventDefault();setSlashDismissed(true);return}if(event.key==="ArrowDown"||event.key==="ArrowUp"){event.preventDefault();setSlashIndex(index=>(index+(event.key==="ArrowDown"?1:slashSuggestions.length-1))%slashSuggestions.length);return}if(event.key==="Tab"||(event.key==="Enter"&&!event.shiftKey)){event.preventDefault();setDraft(`/${slashSuggestions[slashIndex%slashSuggestions.length].name} `);return}}const history=sentHistoryRef.current;if(event.key==="ArrowUp"&&history.length>0&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionStart===0)){event.preventDefault();if(historyIndexRef.current===null){historyDraftRef.current=draft;historyIndexRef.current=history.length-1}else historyIndexRef.current=Math.max(0,historyIndexRef.current-1);setDraft(history[historyIndexRef.current]);return}if(event.key==="ArrowDown"&&historyIndexRef.current!==null&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionEnd===event.currentTarget.value.length)){event.preventDefault();if(historyIndexRef.current(sessionsPath,{method:"POST",body:JSON.stringify({title:t("newConversation")})});writeSessionStore(sessionStoreKey,session.id);const next=await api(sessionsPath);setSessions(next);setActiveSessionId(session.id);setMessages([])}catch(e){setError(e instanceof Error?localizeError(e.message):t("operationFailed"))}finally{setBusy(false)}} + // A run that stopped to ask keeps its harness state, so the answer is just a + // normal message: the backend folds it into the paused run instead of + // starting a new task. + async function continueRun(messageId:string){if(!activeSessionId||sendingRef.current||busy)return;sendingRef.current=true;const text=t("resumeSent");try{await action(async()=>{await api(`/api/sessions/${activeSessionId}/messages`,{method:"POST",body:JSON.stringify({text,clientNonce:clientNonce()})});setResumedChips(current=>({...current,[messageId]:true}));await loadSessions()})}finally{sendingRef.current=false}} + async function retryRun(runId?:string){if(!runId||busy)return;await action(()=>api(`/api/runs/${runId}/retry`,{method:"POST",body:"{}"}));setRetriedRuns(current=>({...current,[runId]:true}))} async function clearSession(){if(!activeSessionId)return;setClearOpen(false);setSessionMenuOpen(false);await action(async()=>{await api(`/api/sessions/${activeSessionId}/messages`,{method:"DELETE"});setMessages([]);await loadSessions()})} async function deleteSession(id:string){if(!sessionsPath||!sessionStoreKey)return;setSessionMenuOpen(false);setBusy(true);setError(null);try{await api(`/api/sessions/${id}`,{method:"DELETE"});const next=await api(sessionsPath);setSessions(next);const pick=id===activeSessionId||!next.some(session=>session.id===activeSessionId)?next[0]?.id||null:activeSessionId;setActiveSessionId(pick);if(pick)writeSessionStore(sessionStoreKey,pick)}catch(e){setError(e instanceof Error?localizeError(e.message):t("operationFailed"))}finally{setBusy(false)}} async function rememberMessage(message:Message){const botId=message.speakerBotId||active?.id||activeRoom?.members[0]?.id;if(!botId||!message.body.trim())return;try{await api(`/api/bots/${botId}/memories`,{method:"POST",body:JSON.stringify({content:message.body,sessionId:activeSessionId})});setRemembered(current=>({...current,[message.id]:true}))}catch(e){setError(e instanceof Error?localizeError(e.message):t("rememberFailed"))}} function pasteText(text:string){queueDesktopInput({kind:"clipboard",text})} function queueDesktopInput(input:{kind:"clipboard";text:string}|{kind:"key";key:string}){ const botId=paneBotId; - if(!botId||computer.controlHolder!=="user")return; + if(!botId||!desktopInteractive)return; pasteQueueRef.current=pasteQueueRef.current.catch(()=>{}).then(async()=>{ if(currentPaneRef.current!==botId)return; await api(`/api/computer/${botId}/input`,{method:"POST",body:JSON.stringify(input)}); @@ -291,22 +359,71 @@ export function App(){ async function logout(){setAccountOpen(false);await api("/api/session",{method:"DELETE",body:"{}"}).catch(()=>{});setBots([]);setRooms([]);setMcpServers([]);setActiveId(null);setActiveRoomId(null);setAuthRequired(true)} async function changeComputer(operation:"boot"|"restart"|"stop"){ const botId=paneBotId;if(!botId)return; - const previous=computer;setScreenUrl(null);setDesktopReady(false); + const previous=computer; + setDesktopReady(false); + // Stopping drops the pixels; starting keeps them. The viewer url never + // changes, so the frame mounts straight away and noVNC dials in the moment + // X answers, instead of booting only after the reply has already arrived. + if(operation==="stop")setScreenUrl(null);else setScreenUrl(current=>current||viewerPath(botId)); setComputer(current=>({...current,state:operation==="stop"?current.state:operation==="boot"&¤t.state==="suspended"?"suspended":"booting"})); - try{const status=await api(`/api/computer/${botId}/${operation}`,{method:"POST",body:"{}",signal:AbortSignal.timeout(120_000)});if(currentPaneRef.current===botId)setComputer(status)} + try{ + const status=await api(`/api/computer/${botId}/${operation}`,{method:"POST",body:"{}",signal:AbortSignal.timeout(120_000)}); + if(currentPaneRef.current!==botId)return; + setComputer(status); + // Anything but running and the warm frame is a lie: drop it so the panel + // shows what is really there instead of a desktop that will never return. + if(status.state!=="running")setScreenUrl(null); + } catch(error){ const status=await api(`/api/computer/${botId}/status`,{signal:AbortSignal.timeout(5_000)}).catch(()=>previous); - if(currentPaneRef.current===botId)setComputer(status); + if(currentPaneRef.current===botId){setComputer(status);if(status.state!=="running")setScreenUrl(null)} throw error; } } const startBoot=()=>changeComputer("boot"); const stopComputer=()=>changeComputer("stop"); const restartComputer=()=>changeComputer("restart"); + /** Tell the viewer, this instant, whether human input counts. The lease is + * advisory inside the container, so this is the gate that actually moves the + * mouse, and nobody should wait on a round trip to be able to click. */ + function pushViewOnly(viewOnly:boolean){desktopFrameRef.current?.contentWindow?.postMessage({type:"lazyboy-view-only",viewOnly},location.origin)} + /** Take or release the screen. The button, the badge, the veil and the mouse + * all move on the local copy; the server only has to agree with what is + * already on screen, and if it refuses the read-back puts the mouse back. */ + async function setControl(holder:ComputerStatus["controlHolder"],botId:string|null=paneBotId){ + if(!botId||controlBusyRef.current)return; + controlBusyRef.current=true;setControlBusy(true); + expectedHolderRef.current=holder; + setComputer(current=>({...current,controlHolder:holder,takeoverRequested:holder==="user"?false:current.takeoverRequested})); + if(!computer.sharedInput)setHandingOff(true); + pushViewOnly(viewOnlyFor(holder,computer.sharedInput)); + try{await api(`/api/computer/${botId}/${holder==="user"?"takeover":"release"}`,{method:"POST",body:"{}"})} + catch(error){setError(localizeError(error instanceof Error?error.message:t("operationFailed")))} + finally{ + // Read the truth back once: a refused takeover has to give the mouse up + // again, and the status answer is who holds it now. + await refresh().catch(()=>{});expectedHolderRef.current=null;controlBusyRef.current=false;setControlBusy(false); + } + } + useEffect(() => { + const viewport = window.visualViewport; + const update = () => { + document.documentElement.style.setProperty("--visible-height", `${viewport?.height || window.innerHeight}px`); + document.documentElement.style.setProperty("--visible-top", `${viewport?.offsetTop || 0}px`); + }; + update(); viewport?.addEventListener("resize", update); viewport?.addEventListener("scroll", update); + window.addEventListener("resize", update); + return () => { viewport?.removeEventListener("resize", update); viewport?.removeEventListener("scroll", update); window.removeEventListener("resize", update); }; + }, []); const connecting=computer.state==="running"&&Boolean(screenUrl)&&!desktopReady; const overlayLabel=hudLabel(computer,connecting,handingOff); + // The veil fades in with its label and fades back out through the same + // mascot, so a fast handoff reads as a finished gesture rather than a frame + // that was dropped. Status blips never get to strobe it. + useEffect(()=>{setVeil(current=>nextVeil(overlayLabel,current))},[overlayLabel]); + useEffect(()=>{if(!veil.leaving)return;const timer=setTimeout(()=>setVeil({label:null,leaving:false}),VEIL_FADE_MS);return()=>clearTimeout(timer)},[veil.leaving]); const frame=screenUrl?