From f6585d216dcd76d1972ff0d188e3aa782457facf Mon Sep 17 00:00:00 2001 From: daniel wang Date: Mon, 7 Sep 2026 14:23:38 +0000 Subject: [PATCH] fix job web --- .env.example | 4 + Cargo.lock | 5 +- Makefile | 5 + a.md | 2208 +++++++++++++++++++++++++++ crates/api/src/file_skills.rs | 90 +- crates/api/src/main.rs | 4 +- crates/api/src/retention.rs | 151 +- crates/api/src/routes.rs | 4 +- crates/api/src/runs.rs | 122 +- crates/api/src/sessions.rs | 14 +- crates/api/src/skills.rs | 72 +- crates/api/src/state.rs | 5 +- crates/api/src/tools.rs | 239 ++- crates/api/src/vault.rs | 15 +- crates/api/src/voice.rs | 18 +- crates/api/src/voice_call.rs | 30 +- crates/api/src/web_static.rs | 6 +- crates/control/Cargo.toml | 3 + crates/control/src/a11y.rs | 5 +- crates/control/src/actions.rs | 17 +- crates/control/src/cdp.rs | 43 +- crates/control/src/controller.rs | 215 +++ crates/control/src/cua/browser.rs | 602 ++++++++ crates/control/src/cua/client.rs | 238 +++ crates/control/src/cua/mod.rs | 532 +++++++ crates/control/src/cua/record.rs | 283 ++++ crates/control/src/cua/translate.rs | 219 +++ crates/control/src/legacy.rs | 300 ++++ crates/control/src/lib.rs | 7 + crates/control/src/observe.rs | 38 +- crates/control/src/process.rs | 71 + crates/control/src/sandbox.rs | 119 ++ crates/controld/Cargo.toml | 2 - crates/controld/src/main.rs | 319 ++-- crates/harness/src/execution.rs | 38 +- crates/harness/src/lib.rs | 2 +- crates/harness/src/voice.rs | 126 +- crates/sandbox/src/docker.rs | 92 +- crates/sandbox/src/fake.rs | 50 +- crates/supervisor/src/docker.rs | 128 +- crates/supervisor/src/main.rs | 101 +- docker-compose.yml | 1 + docs/cua-compatibility.md | 87 ++ docs/development.md | 7 + docs/operations.md | 1 + image/computer/Dockerfile | 22 + image/computer/cua-smoke-gtk.py | 64 + image/computer/cua-smoke.html | 31 + image/computer/lazyboy-browser | 1 + image/computer/lazyboy-screen | 43 + scripts/cua-smoke-inner.py | 785 ++++++++++ scripts/cua-smoke-test.sh | 92 ++ 52 files changed, 7136 insertions(+), 540 deletions(-) create mode 100644 a.md create mode 100644 crates/control/src/controller.rs create mode 100644 crates/control/src/cua/browser.rs create mode 100644 crates/control/src/cua/client.rs create mode 100644 crates/control/src/cua/mod.rs create mode 100644 crates/control/src/cua/record.rs create mode 100644 crates/control/src/cua/translate.rs create mode 100644 crates/control/src/legacy.rs create mode 100644 crates/control/src/process.rs create mode 100644 docs/cua-compatibility.md create mode 100755 image/computer/cua-smoke-gtk.py create mode 100644 image/computer/cua-smoke.html create mode 100755 scripts/cua-smoke-inner.py create mode 100755 scripts/cua-smoke-test.sh diff --git a/.env.example b/.env.example index eab494b..bb84dcd 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,10 @@ LAZYBOY_COMPUTER_CPUS=2 LAZYBOY_COMPUTER_PIDS=2048 # Only affects the Agent desktop container. Disabled by default. LAZYBOY_COMPUTER_SUDO=false +# Computer-control backend inside each desktop container. Rebuild/recreate +# computers after changing. `cua` is Cua Driver (default); `legacy` is +# CDP/AT-SPI/xdotool rollback. +LAZYBOY_COMPUTER_DRIVER=cua # Linux only (optional): point this at the host's LXCFS root to make htop/free # report the per-Agent cgroup quota. Leave the default empty directory on macOS. LAZYBOY_LXCFS_ROOT=./data/lxcfs diff --git a/Cargo.lock b/Cargo.lock index b945f05..ed14eaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1936,6 +1936,7 @@ name = "lazyboy-control" version = "0.1.0" dependencies = [ "async-trait", + "base64 0.22.1", "chrono", "hex", "image", @@ -1944,6 +1945,8 @@ dependencies = [ "serde_json", "sha2", "thiserror", + "tokio", + "tracing", ] [[package]] @@ -1951,8 +1954,6 @@ name = "lazyboy-controld" version = "0.1.0" dependencies = [ "axum", - "base64 0.22.1", - "lazyboy-contracts", "lazyboy-control", "serde_json", "tokio", diff --git a/Makefile b/Makefile index 3a27ceb..dd19507 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ DATA_DIR ?= ./data .PHONY: help env env-force \ up logs ps health down purge \ computer postgres postgres-down pg-collation \ + cua-smoke \ build build-api build-supervisor build-controld \ fmt fmt-check clippy lint audit test clean \ web \ @@ -31,6 +32,7 @@ help: ## Show this help @echo "" @echo " Individual pieces:" @echo " make computer Build the heavy Debian desktop image (lazyboy/computer:local)" + @echo " make cua-smoke Run Cua Driver smoke test in a disposable desktop container" @echo " make postgres Start only postgres (127.0.0.1:5434) and wait for ready" @echo " make postgres-down Stop postgres" @echo " make pg-collation Repair a Postgres collation version mismatch (see docs)" @@ -93,6 +95,9 @@ purge: ## Stop containers and delete the postgres data volume computer: ## Build the Debian desktop image used to spawn bot computers docker build -f image/computer/Dockerfile -t $(COMPUTER_IMAGE) . +cua-smoke: computer ## Run the Cua Driver smoke test inside a disposable desktop container + ./scripts/cua-smoke-test.sh --docker --image $(COMPUTER_IMAGE) + postgres: ## Start only postgres and wait until it is ready $(COMPOSE) -f docker-compose.yml -f docker-compose.dev.yml up -d postgres @echo "waiting for postgres (127.0.0.1:5434) to be ready..." diff --git a/a.md b/a.md new file mode 100644 index 0000000..fe27848 --- /dev/null +++ b/a.md @@ -0,0 +1,2208 @@ +# LazyBoy → Cua Driver Migration Plan + +> **Purpose:** This document is an implementation specification for a coding agent. +> +> Repository: `https://code.30cm.net/daniel.w/lazyBoy` +> +> Cua: `https://github.com/trycua/cua` +> +> Cua docs: `https://cua.ai/docs` +> +> **Main principle:** Do **NOT** rewrite LazyBoy into Cua. Keep LazyBoy as the agent/workspace/product layer and replace the low-level computer-control implementation with Cua Driver incrementally. + +--- + +## 0. Agent Instructions + +You are modifying **LazyBoy**, a self-hosted AI agent workspace. + +The goal is to migrate the low-level GUI/browser computer-control implementation from custom CDP / AT-SPI / X11 code to **Cua Driver**, without breaking LazyBoy's existing product architecture. + +### Non-negotiable rules + +1. **Do not perform a Big Bang rewrite.** +2. **Do not remove the legacy computer-control implementation in the first PR.** +3. Keep LazyBoy's current public Agent tool schema stable unless absolutely necessary. +4. Keep: + - `computer_observe` + - `computer_act` + - `browser` + - `shell` + - file tools + - takeover flow + - memory + - schedule + - vault + - skills/playbooks +5. Cua must initially be an **implementation detail behind LazyBoy abstractions**. +6. Do not expose all Cua MCP tools directly to the LLM. +7. Do not replace `DockerSandbox`, `Supervisor`, noVNC, or persistent bot homes during Phase 1. +8. Add a feature flag / backend selector so legacy behavior can be restored immediately. +9. Every migrated action must have observable verification. +10. Do not guess Cua API/tool names from this document. + +### Cua API freshness rule + +Cua changes quickly. + +Before implementing anything: + +```bash +cua-driver --version +cua-driver doctor +cua-driver list-tools +``` + +For every Cua tool you plan to call: + +```bash +cua-driver describe +``` + +Use the schema reported by the installed Cua Driver version as the source of truth. + +Do not hard-code assumptions from old blog posts, old examples, or this document when the installed version differs. + +--- + +# 1. Current LazyBoy Architecture + +The current architecture is approximately: + +```text +User + │ + ▼ +React Web UI + │ + ▼ +crates/api + │ + ├── Agent loop + ├── Sessions / Runs + ├── Memory + ├── Schedule + ├── Vault + ├── Skills + ├── MCP + └── Tool definitions + │ + ▼ +SandboxProvider + │ + ▼ +crates/sandbox + │ + ▼ +DockerSandbox + │ + ▼ +crates/supervisor + │ + ▼ +Linux Desktop Container + │ + ├── XFCE + ├── Chromium + ├── Xvfb + ├── x11vnc + ├── websockify + └── controld + │ + ├── CDP + ├── AT-SPI + ├── X11 + └── screenshots +``` + +Relevant current components: + +```text +apps/web +crates/api +crates/contracts +crates/control +crates/controld +crates/harness +crates/sandbox +crates/supervisor +``` + +The existing `SandboxProvider` abstraction is an important migration seam and should be preserved. + +Current methods include approximately: + +```rust +async fn provision(...) +async fn prepare(...) +async fn capabilities(...) +async fn ensure_screen(...) +async fn reconnect(...) +async fn suspend(...) +async fn resume(...) +async fn execute(...) +async fn observe(...) +async fn act(...) +async fn connect_screen(...) +async fn list_files(...) +async fn read_file(...) +async fn write_file(...) +async fn stop(...) +async fn destroy(...) +``` + +Do not collapse this abstraction. + +--- + +# 2. Target Architecture + +Phase 1 target: + +```text + LazyBoy Web + │ + ▼ + LazyBoy API + │ + LazyBoy Agent Loop + │ + ┌────────────────┼────────────────┐ + │ │ │ + ▼ ▼ ▼ + Memory Skills Schedule + │ + ▼ + LazyBoy Tool Layer + │ + computer / browser / shell + │ + ▼ + ComputerController + abstraction + │ │ + ┌───────────┘ └────────────┐ + ▼ ▼ + LegacyController CuaController + │ │ + CDP / AT-SPI / X11 Cua Driver + │ + ┌──────────────┼──────────────┐ + ▼ ▼ ▼ + X11 AT-SPI Browser + │ + ▼ + Existing Desktop + Container +``` + +### Important + +Cua is initially used as the **computer-control backend**. + +It is **not** the Agent runtime. + +It is **not** the memory system. + +It is **not** the scheduler. + +It is **not** the user-facing product. + +It is **not** the first-phase sandbox lifecycle owner. + +--- + +# 3. What Must Stay in LazyBoy + +The following are LazyBoy product responsibilities and must remain owned by LazyBoy. + +## Keep unchanged unless required + +### Frontend + +```text +apps/web +``` + +Keep: + +- Chat UI +- remote desktop +- noVNC +- mobile controls +- takeover UI +- bot management +- session UI + +### API / Agent layer + +```text +crates/api +``` + +Keep: + +- Agent loop +- model interaction +- run management +- session management +- tool policy +- takeover workflow +- credential flow +- memory +- schedules +- skills +- MCP integrations + +### Harness + +```text +crates/harness +``` + +Keep model-provider support. + +Cua Driver must not decide which model LazyBoy uses. + +### Supervisor + +```text +crates/supervisor +``` + +Phase 1: keep current behavior. + +Keep: + +- Docker lifecycle +- pause/resume +- resource limits +- persisted bot homes +- screen lifecycle + +### Sandbox + +```text +crates/sandbox +``` + +Phase 1: keep `DockerSandbox`. + +Do **not** replace it with Cua Sandbox yet. + +--- + +# 4. What Cua Should Replace + +The long-term goal is to reduce LazyBoy-owned OS automation. + +Current files that are candidates for replacement or simplification: + +```text +crates/control/src/a11y.py +crates/control/src/a11y.rs + +crates/control/src/cdp.py +crates/control/src/cdp.rs + +crates/control/src/x11.rs +crates/control/src/screen.rs +crates/control/src/overlay.rs +``` + +Do not delete them during the initial migration. + +Instead place them behind a legacy backend. + +The responsibilities that should gradually move to Cua: + +- screenshots +- application/window enumeration +- accessibility tree +- native UI element actions +- pointer input +- keyboard input +- scrolling +- browser state +- browser semantic actions +- window-scoped control +- recording / trajectory capture + +--- + +# 5. Do NOT Expose Raw Cua to the LLM + +LazyBoy currently has a relatively compact Agent-facing tool surface. + +Preserve this. + +Preferred model: + +```text +LLM + │ + ▼ +LazyBoy tools + │ + ├── computer_observe + ├── computer_act + ├── browser + ├── shell + ├── file tools + └── request_takeover + │ + ▼ +LazyBoy policy / safety / vault / state + │ + ▼ +Cua adapter + │ + ▼ +Cua Driver +``` + +Avoid this: + +```text +LLM + │ + ▼ +50+ raw Cua tools + │ + ▼ +OS +``` + +Reasons: + +- larger tool schemas consume context +- LazyBoy loses policy control +- LazyBoy loses stable abstraction +- Cua version changes would leak into prompts +- takeover behavior becomes harder to control +- credential handling becomes harder to constrain +- Agent behavior becomes coupled to Cua implementation details + +Cua should initially behave like a device driver. + +--- + +# 6. Introduce a ComputerController Abstraction + +Create a low-level computer-control abstraction separate from sandbox lifecycle. + +Suggested location: + +```text +crates/control/src/controller.rs +``` + +Possible interface: + +```rust +#[async_trait::async_trait] +pub trait ComputerController: Send + Sync { + async fn health(&self, ctx: &ControlContext) + -> Result; + + async fn observe( + &self, + request: ObserveRequest, + ctx: &ControlContext, + ) -> Result; + + async fn act( + &self, + request: ActionRequest, + ctx: &ControlContext, + ) -> Result; + + async fn browser( + &self, + request: BrowserRequest, + ctx: &ControlContext, + ) -> Result; + + async fn start_recording( + &self, + request: RecordingRequest, + ctx: &ControlContext, + ) -> Result; + + async fn stop_recording( + &self, + ctx: &ControlContext, + ) -> Result; +} +``` + +Names may be adjusted to match existing LazyBoy contracts. + +Do not introduce unnecessary abstractions if equivalent types already exist. + +--- + +# 7. Backend Implementations + +Implement: + +```text +LegacyController +CuaController +``` + +Suggested layout: + +```text +crates/control/src/ +├── controller.rs +├── legacy/ +│ ├── mod.rs +│ ├── a11y.rs +│ ├── cdp.rs +│ ├── x11.rs +│ └── screen.rs +└── cua/ + ├── mod.rs + ├── client.rs + ├── translate.rs + ├── observe.rs + ├── actions.rs + ├── browser.rs + └── recording.rs +``` + +Do not spend the first PR moving all old files if it creates noisy diffs. + +It is acceptable to initially keep existing file locations and only add: + +```text +controller.rs +cua.rs +``` + +Refactor structure after behavior is stable. + +--- + +# 8. Backend Configuration + +Add a configuration value: + +```bash +LAZYBOY_COMPUTER_DRIVER=legacy +``` + +or: + +```bash +LAZYBOY_COMPUTER_DRIVER=cua +``` + +Default during initial rollout: + +```bash +LAZYBOY_COMPUTER_DRIVER=legacy +``` + +After Cua passes production-equivalent validation, the default may become: + +```bash +LAZYBOY_COMPUTER_DRIVER=cua +``` + +Do not remove the legacy option until Cua has passed the migration acceptance suite. + +If configuration already has a typed settings system, add this there instead of reading environment variables throughout the code. + +Suggested enum: + +```rust +pub enum ComputerDriver { + Legacy, + Cua, +} +``` + +--- + +# 9. Recommended Cua Integration Mode + +LazyBoy is an application with built-in computer-use. + +Preferred order for experimentation: + +## POC + +Use the Cua Driver CLI / daemon boundary first if it allows fast validation. + +Examples: + +```bash +cua-driver call '' +``` + +or a controlled local process interface. + +This proves Cua works inside the existing Linux desktop container. + +## Production integration + +Prefer a typed/stable integration boundary. + +Evaluate, based on the installed Cua release: + +1. direct in-process SDK if suitable for LazyBoy's process model +2. private worker / daemon +3. CLI JSON calls as fallback + +Do not choose MCP merely because Agent frameworks often use MCP. + +LazyBoy is embedding computer use inside a product. The internal integration does not have to look like the Agent-facing integration. + +### Rust note + +Cua Driver's core runtime is Rust-based, but public application SDK support may differ by release. + +Do not write custom unsafe bindings unless there is a strong reason. + +Prefer an officially supported application integration surface. + +--- + +# 10. Phase 0 — Discovery / Compatibility Check + +Before changing LazyBoy behavior, create a short engineering report. + +The coding agent must verify the actual runtime environment. + +Inside a real LazyBoy desktop container: + +```bash +echo "$DISPLAY" +ps aux +env | sort +``` + +Confirm: + +- X11 display exists +- XFCE session exists +- AT-SPI bus is available +- Chromium exists +- Cua Driver can start +- Cua Driver can capture the current display +- Cua Driver can enumerate apps/windows +- Cua Driver can operate in Xvfb +- Cua Driver can access Chromium in this environment + +Install Cua Driver using the current official installation method. + +Then run: + +```bash +cua-driver --version +cua-driver doctor +cua-driver list-tools +``` + +Save results in: + +```text +docs/cua-compatibility.md +``` + +Include: + +- installed Cua version +- Linux distribution +- display server +- Cua doctor output summary +- supported Cua tools relevant to LazyBoy +- missing dependencies +- known limitations + +### Stop condition + +If Cua cannot reliably control the existing `XFCE + Xvfb` environment, do **not** proceed with architecture replacement. + +Instead document the blocker first. + +--- + +# 11. Phase 1 — Five-Action POC + +Do not begin by migrating the whole control crate. + +First prove these five capabilities inside the existing LazyBoy desktop: + +1. screenshot +2. accessibility/window observation +3. click native GUI element +4. type text +5. browser semantic action + +Optional sixth: + +6. scroll + +Create a standalone POC path. + +Example: + +```text +scripts/cua-smoke-test.sh +``` + +or: + +```text +crates/control/examples/cua_smoke.rs +``` + +### POC flow + +Suggested test: + +```text +1. Open a native/simple XFCE application. +2. Observe window state. +3. Click an accessible element. +4. Type text. +5. Launch/use Chromium. +6. Navigate to example.com. +7. Read browser state. +8. Trigger a semantic browser interaction if available. +9. Capture final screenshot. +``` + +### POC success criteria + +All five core actions succeed 10 consecutive times without: + +- wrong window actions +- stale element actions +- focus corruption +- unexplained timeouts +- leaving Cua processes behind +- breaking noVNC control + +Do not migrate production paths until this passes. + +--- + +# 12. Phase 2 — Migrate `computer_observe` + +First production migration target: + +```text +computer_observe +``` + +Current LazyBoy observation must remain compatible with the Agent. + +Do not change the Agent prompt contract unless required. + +Cua result must be translated into: + +```rust +ComputerObservation +``` + +Preserve existing concepts where possible: + +- screenshot/image +- dimensions +- cursor +- active window +- UI elements + +### Translation layer + +Implement: + +```text +Cua state + ↓ +CuaObservationAdapter + ↓ +LazyBoy ComputerObservation +``` + +Do not expose Cua-native short-lived element identifiers directly as permanent LazyBoy identifiers. + +Element references may only be valid for a specific observation. + +Treat each observation as a snapshot. + +### Observation ID + +Strongly consider adding or preserving an observation/snapshot identifier. + +Example: + +```rust +pub struct ComputerObservation { + pub observation_id: String, + ... +} +``` + +If changing the contract is too invasive, keep this internal first. + +This will later help detect stale actions. + +--- + +# 13. Phase 3 — Migrate `computer_act` + +Preserve the existing LazyBoy `ComputerAction` DSL. + +Current behavior such as: + +- click +- move +- pointer down/up +- hover +- drag +- type +- keyboard +- wait +- semantic references + +should remain Agent-facing LazyBoy concepts. + +Create a translator: + +```text +LazyBoy ComputerAction + │ + ▼ + CuaActionTranslator + │ + ▼ + Cua Driver +``` + +Example conceptual mapping: + +```text +LazyBoy click(element) + → semantic Cua action when possible + +LazyBoy click(x, y) + → pixel Cua action + +LazyBoy type(text) + → Cua text input + +LazyBoy key(...) + → Cua key action + +LazyBoy scroll(...) + → Cua scroll +``` + +Exact tool names MUST come from: + +```bash +cua-driver list-tools +cua-driver describe ... +``` + +### Preserve semantic-first behavior + +Prefer: + +```text +Accessibility / browser semantic action +``` + +over: + +```text +screen coordinate +``` + +Coordinate action should be fallback, not default. + +--- + +# 14. Preserve LazyBoy Action Safety Logic + +Do not delete useful behavior from: + +```text +crates/control/src/actions.rs +``` + +Existing logic includes concepts such as: + +- action batch limits +- coordinate validation +- element lookup +- stale click prevention +- browser semantic-routing preference +- double-click expansion +- drag normalization + +These should become policy/translation logic above Cua. + +Desired layering: + +```text +Agent request + │ + ▼ +LazyBoy validation + │ + ▼ +LazyBoy action policy + │ + ▼ +Cua translation + │ + ▼ +Cua Driver +``` + +Cua is not a replacement for LazyBoy product policy. + +--- + +# 15. Stale Element Handling + +This is critical. + +Do not assume: + +```text +element 12 +``` + +from one Cua observation refers to the same UI element later. + +The migration must treat element references as snapshot-scoped. + +Preferred flow: + +```text +observe + ↓ +snapshot A + ↓ +Agent selects element + ↓ +act against snapshot A + ↓ +UI changes + ↓ +observe again + ↓ +snapshot B +``` + +If the UI changed materially, do not retry an old semantic reference blindly. + +### Retry rule + +When an action fails: + +1. re-observe +2. re-resolve the target +3. retry with bounded count + +Never loop the same stale action indefinitely. + +--- + +# 16. Phase 4 — Migrate Browser Control + +LazyBoy currently separates browser actions from generic computer actions. + +Keep that separation for the Agent. + +Agent-facing: + +```text +browser +``` + +Internal: + +```text +LazyBoy BrowserRequest + │ + ▼ +Cua Browser Adapter + │ + ▼ +Cua Driver browser tools +``` + +Cua browser control should replace custom CDP behavior gradually. + +### Important + +Preserve the current policy: + +```text +When semantic browser state is available, +prefer browser semantic actions over pixel clicking Chromium. +``` + +Do not make browser automation less reliable during migration. + +### Test cases + +At minimum test: + +- navigate URL +- inspect page +- click semantic element +- type into input +- scroll +- multiple tabs if LazyBoy currently depends on them +- file picker transition +- page refresh +- browser restart +- authenticated persistent profile + +--- + +# 17. Browser Profile Compatibility + +LazyBoy persists browser profiles per computer/bot. + +This is product-critical. + +Do not let Cua silently replace LazyBoy's profile with an ephemeral managed browser unless explicitly intended. + +Verify: + +- existing Chromium profile path remains usable +- cookies survive container pause/resume +- login sessions survive LazyBoy restart behavior as expected +- takeover user and Agent see the same session +- Cua attaches to the correct browser/window + +If Cua requires an explicit browser preparation/attach step, integrate that into LazyBoy lifecycle. + +Do not auto-create a separate profile that breaks existing saved logins. + +--- + +# 18. Phase 5 — Recording Integration + +LazyBoy has an important feature: + +```text +Human demonstration + ↓ +record context/actions + ↓ +model generates playbook + ↓ +future run resolves current UI +``` + +Preserve this architecture. + +Do **not** downgrade it into raw coordinate replay. + +Cua recording should be used as richer source data. + +Target: + +```text +Human / Agent demonstration + │ + ▼ + Cua trajectory + │ + ├── before state + ├── action + ├── after state + ├── screenshots + └── optional video + │ + ▼ + LazyBoy Skill Compiler + │ + ▼ + Semantic Playbook +``` + +Cua trajectory is evidence. + +LazyBoy skill/playbook is the reusable automation. + +### Do not do + +```text +record x=312,y=441 +replay x=312,y=441 forever +``` + +### Do + +Store semantic intent when possible: + +```text +Click the "Sign in" button +``` + +Then resolve it on the current screen during replay. + +--- + +# 19. Takeover Must Keep Working + +User takeover is a core LazyBoy feature. + +Migration acceptance requires: + +```text +Agent running + ↓ +request_takeover + ↓ +Agent stops issuing input + ↓ +User controls noVNC desktop + ↓ +User releases takeover + ↓ +Agent re-observes + ↓ +Agent resumes from current state +``` + +### Required rule + +After takeover ends: + +**Always perform a fresh observation before the Agent performs another UI action.** + +Never reuse pre-takeover element references. + +--- + +# 20. Credential / Vault Boundary + +Cua must not get broad access to LazyBoy secrets by default. + +Keep credential policy in LazyBoy. + +Preferred flow: + +```text +Agent wants login + │ + ▼ +LazyBoy checks: +- allowed domain +- saved credential exists +- HTTPS / policy + │ + ▼ +LazyBoy authorizes injection + │ + ▼ +Cua performs allowed typing/action +``` + +Do not make the Cua integration read the entire Vault. + +Secrets should not appear: + +- in command-line arguments +- in logs +- in Cua debug output +- in trajectory metadata +- in screenshots longer than unavoidable +- in error messages + +Audit recording behavior around passwords. + +If recording is active, ensure sensitive typing can be masked or recording paused. + +--- + +# 21. `controld` Migration Strategy + +Do not delete `crates/controld` during Phase 1. + +It is a useful compatibility boundary. + +Current conceptual API: + +```text +POST /observe +POST /act +``` + +Recommended first migration: + +```text +POST /observe + ↓ +selected ComputerController + ↓ +CuaController or LegacyController +``` + +```text +POST /act + ↓ +LazyBoy validation + ↓ +selected ComputerController + ↓ +CuaController or LegacyController +``` + +This keeps: + +```text +API +SandboxProvider +Supervisor +``` + +largely unchanged. + +Later, if Cua integration makes `controld` unnecessary, remove it in a dedicated architectural PR. + +Do not mix that cleanup into the initial migration. + +--- + +# 22. Docker Image Changes + +Add Cua dependencies to the LazyBoy desktop image. + +The coding agent must locate the actual Dockerfile(s) used for desktop computers. + +Do not assume the path. + +Changes may include: + +- Cua Driver install +- required X11 packages +- AT-SPI dependencies +- ffmpeg if recording/video is enabled +- runtime directories +- permissions +- PATH configuration + +Run: + +```bash +cua-driver doctor +``` + +inside the built desktop container as part of the smoke test. + +### Image versioning + +Pin a known-working Cua version for reproducible builds. + +Do not install uncontrolled nightly builds in the default production image. + +Optionally allow: + +```bash +LAZYBOY_CUA_CHANNEL=stable +LAZYBOY_CUA_VERSION= +``` + +or equivalent build args. + +--- + +# 23. Cua Process Lifecycle + +Do not start one uncontrolled global Cua process for every LazyBoy computer unless architecture requires it. + +Determine which process should own: + +- display connection +- accessibility session +- browser connection +- recordings +- Cua lifecycle + +For LazyBoy's existing architecture, the safest initial design is usually: + +```text +one desktop container + │ + ├── XFCE/Xvfb + ├── Chromium + ├── noVNC + ├── LazyBoy controld + └── Cua runtime/driver +``` + +The driver should only see/control that computer's desktop session. + +### Isolation + +Bot A must never control Bot B's display. + +Tests must explicitly verify isolation. + +--- + +# 24. Health Checks + +Add a Cua health check. + +Possible information: + +```rust +pub struct ControllerHealth { + pub backend: String, + pub version: Option, + pub healthy: bool, + pub degraded: bool, + pub details: Vec, +} +``` + +Surface useful failures: + +- driver not installed +- X11 unavailable +- AT-SPI unavailable +- screenshot unavailable +- browser unavailable +- incompatible Cua version + +Do not return generic: + +```text +computer failed +``` + +when a meaningful diagnosis is available. + +--- + +# 25. Fallback Behavior + +During migration: + +```text +LAZYBOY_COMPUTER_DRIVER=legacy +``` + +must work. + +For `cua` mode, avoid silent fallback for individual actions unless explicitly designed. + +Bad: + +```text +Cua click failed +→ silently xdotool click +``` + +This makes failures impossible to debug. + +Preferred: + +```text +Cua action failed +→ return classified error +→ Agent re-observes / retries / requests takeover +``` + +A feature-flag-level fallback is acceptable. + +An invisible per-action fallback is not. + +--- + +# 26. Logging / Observability + +Add structured logs around Cua calls. + +Include: + +- operation ID +- run ID +- bot ID +- screen ID +- backend +- Cua tool +- duration +- success/failure +- action route if Cua reports it +- observation ID if available + +Never log secret text. + +For typing actions: + +```text +text="" +length=12 +``` + +not: + +```text +text="actual-password" +``` + +--- + +# 27. Metrics + +If LazyBoy has metrics infrastructure, add: + +```text +computer_action_total +computer_action_failed_total +computer_observe_duration_ms +computer_action_duration_ms +computer_browser_action_duration_ms +computer_stale_reference_total +computer_takeover_total +cua_driver_restart_total +``` + +Useful labels: + +```text +backend +action_type +route +result +``` + +Avoid high-cardinality IDs such as `run_id` as metric labels. + +--- + +# 28. Testing Strategy + +## Unit tests + +Test translation only. + +Examples: + +```text +LazyBoy click semantic ref → expected Cua request + +LazyBoy coordinate click → expected Cua request + +LazyBoy type → expected Cua request + +Cua observation → LazyBoy ComputerObservation + +unsupported Cua response → classified error +``` + +Use fake/mocked Cua responses. + +--- + +## Contract tests + +The Agent-facing tool results should remain equivalent between: + +```text +LegacyController +CuaController +``` + +for common scenarios. + +Test: + +- observation shape +- action result shape +- error behavior +- browser results +- takeover interaction + +--- + +## Integration tests + +Run against a real desktop container. + +Test native GUI: + +```text +open app +observe +click +type +verify application state +``` + +Do not validate only that Cua returned `"ok"`. + +Validate independent application state. + +--- + +## Browser E2E + +Use a deterministic local test page rather than an external website. + +Create fixtures for: + +- button +- input +- checkbox +- select +- scroll area +- delayed DOM update +- modal +- new tab +- canvas fallback if needed + +Verify actual DOM/application state. + +--- + +# 29. Migration Acceptance Suite + +Cua backend is not considered ready until all of the following pass. + +## Desktop + +- [ ] screenshot works +- [ ] window enumeration works +- [ ] active window works +- [ ] accessibility elements work +- [ ] semantic click works +- [ ] coordinate click works +- [ ] type text works +- [ ] hotkey works +- [ ] scroll works +- [ ] drag works if supported +- [ ] desktop stays usable through noVNC + +## Browser + +- [ ] attach to correct Chromium +- [ ] use persistent LazyBoy profile +- [ ] navigate +- [ ] observe DOM/browser state +- [ ] semantic click +- [ ] type +- [ ] scroll +- [ ] refresh +- [ ] profile survives pause/resume +- [ ] same browser is visible to human takeover + +## Lifecycle + +- [ ] fresh container +- [ ] existing persisted container +- [ ] pause +- [ ] resume +- [ ] stop +- [ ] restart +- [ ] concurrent bots +- [ ] no cross-bot control + +## Agent flow + +- [ ] `computer_observe` +- [ ] `computer_act` +- [ ] `browser` +- [ ] `shell` +- [ ] takeover +- [ ] resume after takeover +- [ ] skill recording +- [ ] scheduled run + +--- + +# 30. Performance Benchmark + +Before defaulting to Cua, compare against legacy. + +Create a benchmark report: + +```text +docs/cua-benchmark.md +``` + +Measure: + +| Scenario | Legacy | Cua | Winner | +| -------------------- | -----: | --: | ------ | +| screenshot | | | | +| observe desktop | | | | +| native click | | | | +| type text | | | | +| browser snapshot | | | | +| browser click | | | | +| 20-step browser task | | | | + +Also measure: + +- tool-call count +- bytes returned to model +- screenshot count +- total model-visible observation size +- end-to-end task completion time +- failure rate + +The goal is not only faster pointer execution. + +The real goal is: + +```text +fewer Agent turns ++ +less context ++ +higher task success rate +``` + +--- + +# 31. Rollout Plan + +## Step A + +Add: + +```text +ComputerController +LegacyController +``` + +No behavior change. + +All tests must pass. + +--- + +## Step B + +Add Cua smoke test. + +No production traffic. + +--- + +## Step C + +Implement: + +```text +CuaController.observe +``` + +Feature flagged. + +--- + +## Step D + +Implement simple actions: + +```text +click +type +keyboard +scroll +``` + +--- + +## Step E + +Migrate browser state/actions. + +--- + +## Step F + +Add recording integration. + +--- + +## Step G + +Run acceptance suite and benchmark. + +--- + +## Step H + +Change default: + +```text +legacy +→ +cua +``` + +but keep legacy rollback. + +--- + +## Step I + +After stable operation, delete obsolete low-level code in separate PRs. + +Possible deletion candidates: + +```text +a11y.py +a11y.rs +cdp.py +cdp.rs +x11.rs +screen implementation portions +``` + +Only delete code proven unused. + +--- + +# 32. Suggested PR Sequence + +Keep PRs small. + +### PR 1 + +```text +refactor(control): introduce pluggable ComputerController +``` + +- add interface +- wrap legacy +- no behavior change +- tests + +### PR 2 + +```text +build(desktop): install and validate Cua Driver +``` + +- image changes +- doctor +- smoke test +- compatibility doc + +### PR 3 + +```text +feat(control): add Cua observation backend +``` + +### PR 4 + +```text +feat(control): route computer actions through Cua +``` + +### PR 5 + +```text +feat(browser): add Cua browser adapter +``` + +### PR 6 + +```text +feat(skills): ingest Cua trajectories for demonstrations +``` + +### PR 7 + +```text +test(control): add Cua acceptance and benchmark suite +``` + +### PR 8 + +```text +chore(control): make Cua the default backend +``` + +### Later + +```text +chore(control): remove obsolete legacy OS automation +``` + +--- + +# 33. Error Model + +Map Cua failures to typed LazyBoy errors. + +Suggested categories: + +```rust +pub enum ControlError { + DriverUnavailable, + DriverUnhealthy, + DisplayUnavailable, + AccessibilityUnavailable, + BrowserUnavailable, + TargetNotFound, + StaleReference, + PermissionDenied, + Timeout, + Unsupported, + Busy, + InvalidAction, + Internal(String), +} +``` + +Do not expose Cua raw errors directly to the Agent when a stable LazyBoy error can represent them. + +Log the raw cause internally. + +--- + +# 34. Retry Policy + +Do not blindly retry UI actions. + +Recommended: + +### Observation failure + +Retry a small bounded number if transport/runtime error appears transient. + +### Semantic target not found + +```text +re-observe +→ re-resolve +→ retry once +``` + +### Stale target + +```text +re-observe +→ never retry original snapshot ref directly +``` + +### Pixel miss + +Do not repeat the same click indefinitely. + +Preserve LazyBoy's stale-click protection concept. + +### Driver crash + +Restart driver/runtime if safe, then require a fresh observation. + +--- + +# 35. Screenshot Policy + +Avoid sending screenshots to the model when structured UI state is sufficient. + +Prefer: + +```text +semantic / accessibility observation +``` + +Use screenshots when: + +- layout matters +- semantic state is incomplete +- canvas/WebGL is involved +- visual verification is needed + +This reduces: + +- latency +- model context +- vision token cost +- accidental secret exposure + +--- + +# 36. Cua Permission Policy + +If the chosen Cua deployment mode supports permission policies, use allow-list behavior. + +LazyBoy only needs a subset of Cua capabilities. + +Allow only required operations. + +Example conceptual set: + +```text +health/doctor-like observation +window/app listing +screenshot +window state +click +type +key +scroll +browser state +browser click +browser type +browser navigation +recording +``` + +Do not allow unrelated capabilities automatically. + +Use actual current Cua tool names from the installed version. + +--- + +# 37. Security Requirements + +- Cua must not access the host Docker socket from Agent desktops. +- Each bot must remain isolated. +- Keep LazyBoy supervisor on internal network. +- Keep `controld` localhost/internal where applicable. +- Do not expose Cua daemon ports publicly. +- Do not log passwords/tokens. +- Verify trajectory storage permissions. +- Verify screenshots are covered by LazyBoy retention policy. +- Do not allow an Agent to change Cua policies. +- Do not let the Agent select another bot's display/session. +- Cua executable/version should be controlled by LazyBoy image/build process. + +--- + +# 38. Multi-Screen Considerations + +LazyBoy already has screen concepts: + +```text +screen_lease_id +screen_id +screen_slot +display +``` + +Do not discard these. + +Cua must be bound to the LazyBoy-selected display/screen. + +Verify multi-screen behavior before enabling Cua for multi-screen bots. + +If Cua does not safely support LazyBoy's current multi-screen semantics: + +- support one screen first +- return explicit capability information +- do not fake support + +--- + +# 39. Cua Capability Mapping + +LazyBoy should expose backend capabilities. + +Suggested internal structure: + +```rust +pub struct ComputerCapabilities { + pub multi_screen: bool, + pub semantic_desktop: bool, + pub semantic_browser: bool, + pub pixel_actions: bool, + pub recording: bool, + pub background_input: bool, +} +``` + +Only add fields if needed and compatible with existing contract evolution. + +Use capability checks instead of OS-name conditionals where possible. + +Bad: + +```rust +if linux { + ... +} +``` + +Better: + +```rust +if capabilities.semantic_browser { + ... +} +``` + +--- + +# 40. Future Phase — Cua Sandbox + +Do **not** implement this during the Driver migration. + +Later, LazyBoy may support: + +```text +SandboxProvider + │ + ├── LazyBoyDocker + └── CuaSandbox +``` + +Possible future computer types: + +```text +Lightweight Linux +Linux VM +Windows VM +macOS VM +Cloud Computer +``` + +This should be a separate project after Cua Driver migration is stable. + +The current `SandboxProvider` abstraction should make this possible. + +--- + +# 41. Future Phase — Windows / macOS + +One strategic reason for Cua is avoiding custom implementations for: + +```text +Linux AT-SPI/X11 +Windows UIA/input/capture +macOS AX/input/capture +``` + +Do not add Windows/macOS support during the initial Linux migration. + +First ensure the LazyBoy control contract is OS-neutral. + +Then add backend capabilities and sandbox providers later. + +--- + +# 42. Definition of Done — Phase 1 + +Phase 1 is complete when: + +1. LazyBoy can run with: + +```bash +LAZYBOY_COMPUTER_DRIVER=legacy +``` + +and: + +```bash +LAZYBOY_COMPUTER_DRIVER=cua +``` + +2. Existing Agent tool schemas remain compatible. + +3. Cua works in the existing LazyBoy Linux desktop container. + +4. `computer_observe` works through Cua. + +5. `computer_act` core actions work through Cua. + +6. Browser semantic operations work through Cua. + +7. noVNC human takeover still works. + +8. Persistent Chromium sessions still work. + +9. Docker pause/resume still works. + +10. Multi-bot isolation is verified. + +11. Legacy backend remains available for rollback. + +12. No secret is exposed in logs or recording metadata. + +13. Acceptance tests pass. + +14. Benchmark results are documented. + +--- + +# 43. First Task for the Coding Agent + +Do this first and nothing larger: + +## Task + +Create a branch: + +```text +feat/cua-driver-poc +``` + +Then: + +1. inspect current LazyBoy computer-control flow +2. identify actual desktop Dockerfile/image entrypoint +3. install a pinned stable Cua Driver in that image +4. make no Agent-facing schema changes +5. create a Cua smoke test that runs inside a LazyBoy desktop +6. verify: + - `cua-driver --version` + - `cua-driver doctor` + - screenshot + - window/accessibility observation + - native click + - typing + - Chromium/browser state +7. write results to: + +```text +docs/cua-compatibility.md +``` + +8. stop after the POC +9. do not delete any legacy control code +10. report blockers before starting the controller refactor + +### Expected output + +The first PR should answer only: + +> "Can Cua Driver reliably control the existing LazyBoy XFCE + Xvfb desktop container?" + +If the answer is yes, proceed to the next PR. + +--- + +# 44. Second Task After POC Passes + +Create: + +```text +ComputerController +``` + +with: + +```text +LegacyController +CuaController +``` + +Initially make all production requests use: + +```text +LegacyController +``` + +Then route only test/flagged traffic to: + +```text +CuaController +``` + +Do not migrate browser and recording in the same PR. + +--- + +# 45. Architecture Decision + +The intended ownership after migration is: + +| Concern | Owner | +| ----------------------------------- | ------- | +| Product UI | LazyBoy | +| Agent loop | LazyBoy | +| Model provider | LazyBoy | +| Memory | LazyBoy | +| Schedules | LazyBoy | +| Vault | LazyBoy | +| Takeover | LazyBoy | +| Skills/playbooks | LazyBoy | +| Sandbox lifecycle (Phase 1) | LazyBoy | +| Docker resources | LazyBoy | +| Remote desktop/noVNC | LazyBoy | +| Computer observation | Cua | +| Native UI control | Cua | +| Pointer/keyboard | Cua | +| Browser semantic control | Cua | +| OS-specific accessibility | Cua | +| Trajectory capture | Cua | +| Workflow abstraction from recording | LazyBoy | + +This is the central design decision. + +Do not invert it. + +--- + +# 46. Final Principle + +LazyBoy's competitive/product value is: + +```text +Agent workspace +Multi-agent +Persistent computers +Human takeover +Memory +Schedules +Vault +Skills +Demo → reusable workflow +Web/mobile UX +Model choice +``` + +The goal of using Cua is to stop spending LazyBoy engineering effort on: + +```text +X11 automation +AT-SPI edge cases +CDP plumbing +screen capture +OS-specific input +window enumeration +cross-platform UI automation +``` + +Build LazyBoy **on top of** Cua. + +Do not turn LazyBoy **into** Cua. + +--- + +# References + +LazyBoy: + +- https://code.30cm.net/daniel.w/lazyBoy +- https://code.30cm.net/daniel.w/lazyBoy/src/branch/main/docs/architecture.md +- https://code.30cm.net/daniel.w/lazyBoy/src/branch/main/crates/control +- https://code.30cm.net/daniel.w/lazyBoy/src/branch/main/crates/control/src/sandbox.rs +- https://code.30cm.net/daniel.w/lazyBoy/src/branch/main/crates/control/src/actions.rs +- https://code.30cm.net/daniel.w/lazyBoy/src/branch/main/crates/sandbox + +Cua: + +- https://github.com/trycua/cua +- https://cua.ai/docs +- https://cua.ai/docs/how-to-guides/driver/install +- https://cua.ai/docs/concepts/choose-a-cua-driver-integration +- https://cua.ai/docs/how-to-guides/driver/connect-your-agent +- https://cua.ai/docs/reference/cua-driver/cli-reference +- https://cua.ai/docs/reference/cua-driver/mcp-tools +- https://cua.ai/docs/how-to-guides/driver/record-and-render-a-trajectory +- https://cua.ai/docs/how-to-guides/driver/restrict-tool-access diff --git a/crates/api/src/file_skills.rs b/crates/api/src/file_skills.rs index 915fb0b..1a6059d 100644 --- a/crates/api/src/file_skills.rs +++ b/crates/api/src/file_skills.rs @@ -17,7 +17,9 @@ fn valid_name(name: &str) -> bool { && name != "goal" && name != "skills" && name != "help" - && name.bytes().all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + && name + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') } fn parse_file(path: PathBuf, name: String) -> Option { @@ -38,23 +40,51 @@ fn parse_file(path: PathBuf, name: String) -> Option { } else { ("", text.trim().to_string()) }; - let description = frontmatter.lines().find_map(|line| { - let (key, value) = line.split_once(':')?; - (key.trim() == "description") - .then(|| value.trim().trim_matches(|ch| ch == '"' || ch == '\'').to_string()) - }).unwrap_or_else(|| instructions.lines().next().unwrap_or("自訂技能").chars().take(120).collect()); - (!instructions.is_empty()).then_some(FileSkill { name, description, instructions }) + let description = frontmatter + .lines() + .find_map(|line| { + let (key, value) = line.split_once(':')?; + (key.trim() == "description").then(|| { + value + .trim() + .trim_matches(|ch| ch == '"' || ch == '\'') + .to_string() + }) + }) + .unwrap_or_else(|| { + instructions + .lines() + .next() + .unwrap_or("自訂技能") + .chars() + .take(120) + .collect() + }); + (!instructions.is_empty()).then_some(FileSkill { + name, + description, + instructions, + }) } pub fn list(data_dir: &str) -> Vec { let root = Path::new(data_dir).join("skills"); - let Ok(entries) = std::fs::read_dir(&root) else { return Vec::new() }; - let mut skills = entries.filter_map(Result::ok).filter_map(|entry| { - let file_type = entry.file_type().ok()?; - if !file_type.is_dir() { return None; } - let name = entry.file_name().to_string_lossy().to_string(); - valid_name(&name).then(|| parse_file(entry.path().join("SKILL.md"), name)).flatten() - }).collect::>(); + let Ok(entries) = std::fs::read_dir(&root) else { + return Vec::new(); + }; + let mut skills = entries + .filter_map(Result::ok) + .filter_map(|entry| { + let file_type = entry.file_type().ok()?; + if !file_type.is_dir() { + return None; + } + let name = entry.file_name().to_string_lossy().to_string(); + valid_name(&name) + .then(|| parse_file(entry.path().join("SKILL.md"), name)) + .flatten() + }) + .collect::>(); skills.sort_by(|left, right| left.name.cmp(&right.name)); skills } @@ -62,15 +92,25 @@ pub fn list(data_dir: &str) -> Vec { pub fn slash(prompt: &str, data_dir: &str) -> Option<(FileSkill, String)> { let mut words = prompt.trim().splitn(2, char::is_whitespace); let command = words.next()?.strip_prefix('/')?; - if !valid_name(command) { return None; } - let skill = list(data_dir).into_iter().find(|skill| skill.name == command)?; + if !valid_name(command) { + return None; + } + let skill = list(data_dir) + .into_iter() + .find(|skill| skill.name == command)?; Some((skill, words.next().unwrap_or("").trim().to_string())) } pub fn index(data_dir: &str) -> String { let skills = list(data_dir); - if skills.is_empty() { return String::new(); } - let lines = skills.iter().map(|skill| format!("/{:<18} {}", skill.name, skill.description)).collect::>().join("\n"); + if skills.is_empty() { + return String::new(); + } + let lines = skills + .iter() + .map(|skill| format!("/{:<18} {}", skill.name, skill.description)) + .collect::>() + .join("\n"); format!("可用的檔案技能（唯讀）：\n{lines}") } @@ -81,12 +121,20 @@ mod tests { #[test] fn loads_only_safe_skill_names_and_resolves_arguments() { - let root = std::env::temp_dir().join(format!("lazyboy-file-skills-{}", uuid::Uuid::new_v4())); + let root = + std::env::temp_dir().join(format!("lazyboy-file-skills-{}", uuid::Uuid::new_v4())); fs::create_dir_all(root.join("skills/open-site")).unwrap(); - fs::write(root.join("skills/open-site/SKILL.md"), "---\ndescription: Open a site\n---\nUse the browser.\n").unwrap(); + fs::write( + root.join("skills/open-site/SKILL.md"), + "---\ndescription: Open a site\n---\nUse the browser.\n", + ) + .unwrap(); fs::create_dir_all(root.join("skills/goal")).unwrap(); let root_text = root.to_str().unwrap(); - assert_eq!(slash("/open-site example.com", root_text).unwrap().1, "example.com"); + assert_eq!( + slash("/open-site example.com", root_text).unwrap().1, + "example.com" + ); assert!(slash("/goal do it", root_text).is_none()); assert!(slash("/missing", root_text).is_none()); let _ = fs::remove_dir_all(root); diff --git a/crates/api/src/main.rs b/crates/api/src/main.rs index 250a12b..208de0e 100644 --- a/crates/api/src/main.rs +++ b/crates/api/src/main.rs @@ -57,7 +57,9 @@ async fn main() { }); let retention_state = state.clone(); - tokio::spawn(async move { retention::retention_loop(retention_state).await; }); + tokio::spawn(async move { + retention::retention_loop(retention_state).await; + }); let worker_state = state.clone(); tokio::spawn(async move { diff --git a/crates/api/src/retention.rs b/crates/api/src/retention.rs index c12fde0..1d161b1 100644 --- a/crates/api/src/retention.rs +++ b/crates/api/src/retention.rs @@ -1,25 +1,60 @@ //! Bounded maintenance of expendable diagnostics, never user-authored content. -use std::time::{Duration}; -use sqlx::PgPool; use crate::state::AppState; +use sqlx::PgPool; +use std::time::Duration; fn days(name: &str, default: i32) -> i32 { - std::env::var(name).ok().and_then(|v| v.parse::().ok()) - .filter(|v| (1..=3650).contains(v)).unwrap_or(default) + std::env::var(name) + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|v| (1..=3650).contains(v)) + .unwrap_or(default) } pub async fn retention_loop(state: AppState) { let recording_days = days("LAZYBOY_RECORDING_RETENTION_DAYS", 30); let rules = [ - ("events", include_str!("retention/events.sql"), days("LAZYBOY_EVENT_RETENTION_DAYS", 30)), - ("checkpoints", include_str!("retention/checkpoints.sql"), days("LAZYBOY_CHECKPOINT_RETENTION_DAYS", 7)), - ("runs", include_str!("retention/runs.sql"), days("LAZYBOY_RUN_RETENTION_DAYS", 90)), - ("run_activity", include_str!("retention/run_activity.sql"), days("LAZYBOY_RUN_ACTIVITY_RETENTION_DAYS", 7)), - ("recordings", include_str!("retention/recordings.sql"), recording_days), - ("revisions", include_str!("retention/revisions.sql"), days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90)), - ("deleted_memories", include_str!("retention/deleted_memories.sql"), days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90)), + ( + "events", + include_str!("retention/events.sql"), + days("LAZYBOY_EVENT_RETENTION_DAYS", 30), + ), + ( + "checkpoints", + include_str!("retention/checkpoints.sql"), + days("LAZYBOY_CHECKPOINT_RETENTION_DAYS", 7), + ), + ( + "runs", + include_str!("retention/runs.sql"), + days("LAZYBOY_RUN_RETENTION_DAYS", 90), + ), + ( + "run_activity", + include_str!("retention/run_activity.sql"), + days("LAZYBOY_RUN_ACTIVITY_RETENTION_DAYS", 7), + ), + ( + "recordings", + include_str!("retention/recordings.sql"), + recording_days, + ), + ( + "revisions", + include_str!("retention/revisions.sql"), + days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90), + ), + ( + "deleted_memories", + include_str!("retention/deleted_memories.sql"), + days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90), + ), ("leases", include_str!("retention/leases.sql"), 7), - ("profile_locks", include_str!("retention/profile_locks.sql"), 7), + ( + "profile_locks", + include_str!("retention/profile_locks.sql"), + 7, + ), ]; loop { for (name, query, age) in rules { @@ -27,20 +62,48 @@ pub async fn retention_loop(state: AppState) { // Limit both transaction size and work per hour; defer excess backlog. for _ in 0..20 { match batch(state.pool(), query, age).await { - Ok(count) => { removed += count; if count < 1000 { break; } } - Err(error) => { tracing::warn!(name, %error, "retention batch failed; retry next hour"); break; } + Ok(count) => { + removed += count; + if count < 1000 { + break; + } + } + Err(error) => { + tracing::warn!(name, %error, "retention batch failed; retry next hour"); + break; + } } tokio::time::sleep(Duration::from_millis(100)).await; } - if removed > 0 { tracing::info!(name, rows=removed, "retention cleaned expired diagnostics"); } + if removed > 0 { + tracing::info!( + name, + rows = removed, + "retention cleaned expired diagnostics" + ); + } } if let Err(error) = clean_frames(&state, recording_days).await { tracing::warn!(%error, "recording file retention failed; retry next hour"); } - if let Ok(bytes) = sqlx::query_scalar::<_, i64>("SELECT pg_database_size(current_database())").fetch_one(state.pool()).await { - let warn_mb = std::env::var("LAZYBOY_DB_WARN_MB").ok().and_then(|v| v.parse::().ok()).filter(|v| *v > 0 && *v < 1_000_000).unwrap_or(1024); + if let Ok(bytes) = + sqlx::query_scalar::<_, i64>("SELECT pg_database_size(current_database())") + .fetch_one(state.pool()) + .await + { + let warn_mb = std::env::var("LAZYBOY_DB_WARN_MB") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|v| *v > 0 && *v < 1_000_000) + .unwrap_or(1024); tracing::info!(bytes, "database size after retention"); - if bytes > warn_mb * 1024 * 1024 { tracing::warn!(bytes, warn_mb, "database exceeds configured size warning; review retained conversations and memories"); } + if bytes > warn_mb * 1024 * 1024 { + tracing::warn!( + bytes, + warn_mb, + "database exceeds configured size warning; review retained conversations and memories" + ); + } } tokio::time::sleep(Duration::from_secs(3600)).await; } @@ -49,16 +112,31 @@ pub async fn retention_loop(state: AppState) { async fn batch(pool: &PgPool, query: &str, age: i32) -> Result { let mut tx = pool.begin().await?; // One maintenance writer, even when multiple API processes start together. - let acquired: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(72189431)").fetch_one(&mut *tx).await?; - if !acquired { return Ok(0); } - sqlx::query("SET LOCAL statement_timeout = '10s'").execute(&mut *tx).await?; - sqlx::query("SET LOCAL lock_timeout = '1s'").execute(&mut *tx).await?; - let result = sqlx::query(query).bind(age).bind(1000_i64).execute(&mut *tx).await?; + let acquired: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(72189431)") + .fetch_one(&mut *tx) + .await?; + if !acquired { + return Ok(0); + } + sqlx::query("SET LOCAL statement_timeout = '10s'") + .execute(&mut *tx) + .await?; + sqlx::query("SET LOCAL lock_timeout = '1s'") + .execute(&mut *tx) + .await?; + let result = sqlx::query(query) + .bind(age) + .bind(1000_i64) + .execute(&mut *tx) + .await?; tx.commit().await?; Ok(result.rows_affected()) } -async fn clean_frames(state: &AppState, age: i32) -> Result<(), Box> { +async fn clean_frames( + state: &AppState, + age: i32, +) -> Result<(), Box> { let root = std::path::Path::new(&state.data_dir).join("teach"); let mut dirs = match tokio::fs::read_dir(&root).await { Ok(dirs) => dirs, @@ -67,20 +145,35 @@ async fn clean_frames(state: &AppState, age: i32) -> Result<(), Box= 1000 { break; } + if cleaned >= 1000 { + break; + } // Ignore symlinks and unexpected names; never traverse a user's home. - if !entry.file_type().await?.is_dir() { continue; } + if !entry.file_type().await?.is_dir() { + continue; + } let id = entry.file_name().to_string_lossy().into_owned(); - if uuid::Uuid::parse_str(&id).is_err() { continue; } + if uuid::Uuid::parse_str(&id).is_err() { + continue; + } let eligible: Option = sqlx::query_scalar( "SELECT status IN ('saved','failed','draft') AND updated_at < now() - make_interval(days => $2) FROM taught_skills WHERE id=$1" ).bind(&id).bind(age).fetch_optional(state.pool()).await?; - let old_orphan = eligible.is_none() && entry.metadata().await?.modified()?.elapsed().unwrap_or(Duration::ZERO) > Duration::from_secs(age as u64 * 86400); + let old_orphan = eligible.is_none() + && entry + .metadata() + .await? + .modified()? + .elapsed() + .unwrap_or(Duration::ZERO) + > Duration::from_secs(age as u64 * 86400); if eligible == Some(true) || old_orphan { tokio::fs::remove_dir_all(entry.path()).await?; cleaned += 1; } } - if cleaned > 0 { tracing::info!(cleaned, "removed expired teaching frame directories"); } + if cleaned > 0 { + tracing::info!(cleaned, "removed expired teaching frame directories"); + } Ok(()) } diff --git a/crates/api/src/routes.rs b/crates/api/src/routes.rs index e450b69..d6027ce 100644 --- a/crates/api/src/routes.rs +++ b/crates/api/src/routes.rs @@ -609,7 +609,9 @@ async fn stop( State(state): State, Path(id): Path, ) -> Result, (StatusCode, Json)> { - let actor = actor(&state).await.map_err(|status| (status, Json(json!({"message":"無法取得工作區"}))))?; + let actor = actor(&state) + .await + .map_err(|status| (status, Json(json!({"message":"無法取得工作區"}))))?; computer::stop(&state, &actor, &id) .await .map(|status| Json(serde_json::to_value(status).unwrap())) diff --git a/crates/api/src/runs.rs b/crates/api/src/runs.rs index 7badf9f..29ec6b3 100644 --- a/crates/api/src/runs.rs +++ b/crates/api/src/runs.rs @@ -1,7 +1,7 @@ use lazyboy_harness::execution::{ - ExecutionMode, GoalOutcome, MAX_NUDGES_GOAL, MAX_NUDGES_PLAIN, NEEDS_INPUT_MARKER, StopReason, - VERIFY_BEFORE_DONE, asks_for_input, goal_outcome, goal_request, stop_reason, GOAL_CONTINUE, - GOAL_INSTRUCTIONS, + ExecutionMode, GOAL_CONTINUE, GOAL_INSTRUCTIONS, GoalOutcome, MAX_NUDGES_GOAL, + MAX_NUDGES_PLAIN, NEEDS_INPUT_MARKER, StopReason, VERIFY_BEFORE_DONE, asks_for_input, + goal_outcome, goal_request, stop_reason, }; use lazyboy_harness::policy::{ActionObserved, LoopGuard, RunPolicy, Verdict}; use std::sync::Arc; @@ -26,6 +26,8 @@ use crate::tools::{ToolCtx, ToolOutcome, dispatch, tool_definitions}; const SCREENSHOT_CAPTION: &str = "Desktop screenshot (1280x800) with yellow numbered marks. Click by those element ids. The live VNC view has no marks."; +const TAKEOVER_RESUME_PROMPT: &str = "The user finished collaborating and released control. Continue the original task from the CURRENT screen. Do not restart from scratch. Element ids and page refs from before the handoff are invalid; use only the fresh observation below."; + const SYSTEM_CHAT: &str = "You are this bot's assistant. This message is conversation — a greeting, small talk, a question you can answer from knowledge, planning, or explaining. Reply in text only. Do not try to use the desktop, browser, files, or shell, and do not narrate that you are checking a screen. You have a Linux desktop for later if the user asks you to operate it; this turn does not need it. @@ -179,7 +181,10 @@ pub async fn send( checkpoint=checkpoint-'awaitResume', updated_at=now() WHERE id=$1 AND status IN ('waiting_input','waiting_takeover')", ) - .bind(&run_id).execute(&mut *tx).await.map_err(|error| error.to_string())?; + .bind(&run_id) + .execute(&mut *tx) + .await + .map_err(|error| error.to_string())?; } let message_id = Uuid::new_v4().to_string(); let seq: i32 = sqlx::query_scalar( @@ -472,7 +477,7 @@ async fn execute_run( "run", json!({"event": "started", "task": crate::monitor::snippet(prompt, 160)}), ) - .await; + .await; let bot = state .db @@ -625,16 +630,20 @@ async fn execute_run( None }; let initial_prompt = if goal_mode { - format!("Execute this goal until it is verified complete:\n{}", goal_text) + format!( + "Execute this goal until it is verified complete:\n{}", + goal_text + ) } else if let Some((skill, args)) = &file_skill { - format!("Run the /{} skill with these arguments: {}", skill.name, args) + format!( + "Run the /{} skill with these arguments: {}", + skill.name, args + ) } else { prompt.to_string() }; let mut first = if resume_after_takeover { - vec![UserContent::text( - "The user finished collaborating and released control. Continue the original task from the CURRENT screen. Do not restart from scratch.", - )] + vec![UserContent::text(TAKEOVER_RESUME_PROMPT)] } else { vec![UserContent::text(initial_prompt)] }; @@ -660,7 +669,11 @@ async fn execute_run( if !resume_after_takeover { // The user named a taught skill: hand the model the full playbook up // front so it does not have to guess or call use_skill first. - if let Some(skill) = if goal_mode || file_skill.is_some() { None } else { crate::skills::skill_for_prompt(state.pool(), bot_id, prompt).await } { + if let Some(skill) = if goal_mode || file_skill.is_some() { + None + } else { + crate::skills::skill_for_prompt(state.pool(), bot_id, prompt).await + } { first.push(UserContent::text(crate::skills::format_playbook_for_run( &skill, ))); @@ -689,8 +702,12 @@ async fn execute_run( .any(|block| block.get("kind").and_then(Value::as_str) == Some("file")); // Greetings and small talk must not even *see* desktop tools: models // otherwise "check the screen" or `ls` the home on "hi" and boot Docker. - let chat_only = - !resume_after_takeover && !goal_mode && file_skill.is_none() && skill_check.is_none() && !workspace_file && is_plain_chat(prompt); + let chat_only = !resume_after_takeover + && !goal_mode + && file_skill.is_none() + && skill_check.is_none() + && !workspace_file + && is_plain_chat(prompt); if chat_only { // Memory is recalled separately and injected into the preamble below. // Do not expose even memory tools here: a plain greeting must be one @@ -761,7 +778,11 @@ async fn execute_run( // steering input. Keep the run alive and deliver each new message once // before the next model turn instead of waiting for a second run to win // the bot lease. - let mut steering_seq = checkpoint.get("steeringSeq").and_then(Value::as_i64).map(|seq| seq as i32).unwrap_or(current_seq); + let mut steering_seq = checkpoint + .get("steeringSeq") + .and_then(Value::as_i64) + .map(|seq| seq as i32) + .unwrap_or(current_seq); let mut used_gui = false; let mut did_work = false; // One verification demand per run: enough to catch "I'm done" that isn't, @@ -1057,7 +1078,11 @@ async fn execute_run( goal_outcome(&final_text), GoalOutcome::Complete | GoalOutcome::NeedsInput ); - let nudge_limit = if goal_mode { MAX_NUDGES_GOAL } else { MAX_NUDGES_PLAIN }; + let nudge_limit = if goal_mode { + MAX_NUDGES_GOAL + } else { + MAX_NUDGES_PLAIN + }; let mut verify_chosen = false; let nudge = if goal_mode { match goal_outcome(&final_text) { @@ -1114,9 +1139,7 @@ async fn execute_run( if used_gui || skill_check.is_some() { prepare_run_computer(state, actor, bot_id, run_id, &ctx, true).await?; } - if (used_gui || skill_check.is_some()) - && ctx.gui_block.lock().unwrap().is_none() - { + if (used_gui || skill_check.is_some()) && ctx.gui_block.lock().unwrap().is_none() { set_run_step(state, run_id, "computer_observe: 重新確認畫面").await; let outcome = dispatch(&ctx, "computer_observe", &json!({})).await; content.push(UserContent::text(outcome.text)); @@ -1511,7 +1534,11 @@ async fn execute_run( state, thread_id, run_id, - if needs_input { "run.paused" } else { "run.completed" }, + if needs_input { + "run.paused" + } else { + "run.completed" + }, turns, screenshots, screenshot_bytes, @@ -1618,10 +1645,16 @@ async fn complete_with_retry( for attempt in 0..3 { if attempt > 0 { let failure = crate::monitor::classify_run_error(&last); - set_run_step(trace.state, trace.run_id, &format!( - "{} 正在自動重試模型（第 {}/3 次）；保留已完成的操作。", - failure.headline, attempt + 1 - )).await; + set_run_step( + trace.state, + trace.run_id, + &format!( + "{} 正在自動重試模型（第 {}/3 次）；保留已完成的操作。", + failure.headline, + attempt + 1 + ), + ) + .await; } let started = std::time::Instant::now(); let result = tokio::time::timeout( @@ -1779,7 +1812,10 @@ fn shrink_checkpoint(history: &mut Vec, pending: &mut Message) { cap_parts(pending); history.iter_mut().for_each(cap_parts); let fits = |turns: &[Message]| { - json!({"harnessHistory": turns, "harnessPending": pending}).to_string().len() <= LIMIT + json!({"harnessHistory": turns, "harnessPending": pending}) + .to_string() + .len() + <= LIMIT }; while !fits(history) && history.len() > 1 { history.remove(0); @@ -1810,7 +1846,10 @@ async fn save_harness_checkpoint( // Only a checkpoint that still cannot fit after shrinking fails closed, and // loudly: the run keeps its uncertain-effects flag instead of freezing. if value.to_string().len() > 1024 * 1024 { - tracing::error!(run_id, "harness checkpoint still exceeds 1 MB after shrinking"); + tracing::error!( + run_id, + "harness checkpoint still exceeds 1 MB after shrinking" + ); return Ok(()); } let result=sqlx::query("UPDATE runs SET checkpoint=COALESCE(checkpoint,'{}'::jsonb)||$3,updated_at=now() WHERE id=$1 AND lease_owner=$2 AND status='running'") @@ -2090,7 +2129,9 @@ async fn pause_for_answer( } } else if !asks_for_input(req.draft) { // A model's optimistic draft must not hide a harness-detected stall. - format!("{draft}\n\n任務尚未確認完成。停止原因：{stall}。已保存操作進度；回覆下一步指示或接管確認現況後可繼續。") + format!( + "{draft}\n\n任務尚未確認完成。停止原因：{stall}。已保存操作進度；回覆下一步指示或接管確認現況後可繼續。" + ) } else { draft }; @@ -2959,15 +3000,17 @@ fn describe_step(name: &str, args: &Value) -> String { } "shell" => { // The terminal does four different things; the feed says which. - let session = get("session") - .filter(|name| !name.trim().is_empty() && *name != "main"); + let session = get("session").filter(|name| !name.trim().is_empty() && *name != "main"); let suffix = session.map(|name| format!(" ·{name}")).unwrap_or_default(); if args.get("reset").and_then(Value::as_bool).unwrap_or(false) { format!("重開終端機{suffix}") } else if let Some(keys) = get("keys") { format!("輸入 {}{suffix}", short(Some(keys), 20)) } else { - match get("command").or(get("cmd")).filter(|line| !line.trim().is_empty()) { + match get("command") + .or(get("cmd")) + .filter(|line| !line.trim().is_empty()) + { Some(command) => format!("{}{suffix}", short(Some(command), 60)), None => format!("讀終端機{suffix}"), } @@ -3114,8 +3157,8 @@ fn tool_status(timed_out: bool, pause: bool, text: &str) -> &'static str { fn action_changes_state(name: &str, args: &Value) -> bool { match name { "computer_act" | "shell" | "write_file" | "launch_app" | "open_path" - | "create_schedule" | "cancel_schedule" | "remember" | "forget_memory" | "use_saved_login" - | "request_takeover" => true, + | "create_schedule" | "cancel_schedule" | "remember" | "forget_memory" + | "use_saved_login" | "request_takeover" => true, "browser" => matches!( args.get("action").and_then(Value::as_str), Some("click") | Some("type") | Some("navigate") | Some("press") @@ -3179,13 +3222,24 @@ async fn record_run_metrics( #[cfg(test)] mod tests { use super::{ - RunHalt, SCREENSHOT_CAPTION, describe_step, drop_history_screenshots, halt_from_status, - history_window_start, is_plain_chat, retryable_run_error, screenshot_parts, tool_needs_gui, - tool_needs_sandbox, tool_status, + RunHalt, SCREENSHOT_CAPTION, TAKEOVER_RESUME_PROMPT, describe_step, + drop_history_screenshots, halt_from_status, history_window_start, is_plain_chat, + retryable_run_error, screenshot_parts, tool_needs_gui, tool_needs_sandbox, tool_status, }; use rig_core::completion::message::{Message, UserContent}; use serde_json::json; + #[test] + fn takeover_resume_invalidates_pre_handoff_refs() { + assert!( + TAKEOVER_RESUME_PROMPT + .contains("Element ids and page refs from before the handoff are invalid") + ); + assert!(TAKEOVER_RESUME_PROMPT.contains("fresh observation")); + assert!(tool_needs_gui("browser")); + assert!(tool_needs_gui("computer_observe")); + } + #[test] fn the_trail_reads_a_tool_result_as_ok_error_timeout_or_pause() { assert_eq!(tool_status(false, false, "clicked element #12"), "ok"); diff --git a/crates/api/src/sessions.rs b/crates/api/src/sessions.rs index e2c7be2..cdb6beb 100644 --- a/crates/api/src/sessions.rs +++ b/crates/api/src/sessions.rs @@ -390,7 +390,14 @@ async fn events( // afterwards would open a window in which a commit could knock on a channel // this reader is not listening to yet. let wakes = state.wakes.subscribe(); - let stream_state = (state, id, actor, after, Vec::<(i32, String, Value)>::new(), wakes); + let stream_state = ( + state, + id, + actor, + after, + Vec::<(i32, String, Value)>::new(), + wakes, + ); let output = stream::unfold( stream_state, |(state, id, actor, mut after, mut pending, mut wakes)| async move { @@ -402,10 +409,7 @@ async fn events( .event(kind) .json_data(payload) .unwrap_or_else(|_| Event::default().event("error").data("{}")); - return Some(( - Ok(event), - (state, id, actor, after, pending, wakes), - )); + return Some((Ok(event), (state, id, actor, after, pending, wakes))); } match sqlx::query_as::<_, (i32, String, Value)>( "SELECT e.seq,e.type,e.payload FROM events e diff --git a/crates/api/src/skills.rs b/crates/api/src/skills.rs index 5bbe53a..7b784a1 100644 --- a/crates/api/src/skills.rs +++ b/crates/api/src/skills.rs @@ -19,8 +19,7 @@ use axum::{Json, Router}; use base64::Engine; use chrono::{DateTime, TimeDelta, Utc}; use lazyboy_control::{ - AdapterContext, CommandRequest, ComputerRef, cdp_record_command_on, cdp_record_stop_command, - frame_signature, signatures_similar, teach_recorder_output, + AdapterContext, ComputerRef, RecordingRequest, frame_signature, signatures_similar, }; use rig_core::completion::message::{ AssistantContent, ImageDetail, ImageMediaType, Message, UserContent, @@ -282,17 +281,14 @@ async fn start_skill( })?; let ctx = computer::adapter_context_for(&actor, &bot_id, "teach", screen.as_ref(), None); - let display = ctx.display.clone().unwrap_or_else(|| ":1".into()); - let argv = cdp_record_command_on(&display, ctx.profile_path.as_deref(), &skill_id); if let Err(error) = state .sandbox - .execute( + .start_recording( &computer_ref, - CommandRequest { - argv, - cwd: None, - timeout_ms: Some(10_000), - stdin: None, + RecordingRequest { + skill_id: skill_id.clone(), + display: ctx.display.clone(), + profile_path: ctx.profile_path.clone(), }, &ctx, ) @@ -889,6 +885,14 @@ async fn record_loop( } } +fn recording_request(skill_id: &str, ctx: &AdapterContext) -> RecordingRequest { + RecordingRequest { + skill_id: skill_id.to_string(), + display: ctx.display.clone(), + profile_path: ctx.profile_path.clone(), + } +} + async fn stop_recorder( state: &AppState, computer_ref: &ComputerRef, @@ -897,16 +901,7 @@ async fn stop_recorder( ) { let _ = state .sandbox - .execute( - computer_ref, - CommandRequest { - argv: cdp_record_stop_command(skill_id), - cwd: None, - timeout_ms: Some(5_000), - stdin: None, - }, - ctx, - ) + .stop_recording(computer_ref, recording_request(skill_id, ctx), ctx) .await; } @@ -916,34 +911,17 @@ async fn collect_browser_events( ctx: &AdapterContext, skill_id: &str, ) -> Vec { - let out = teach_recorder_output(skill_id); - let result = state + match state .sandbox - .execute( - computer_ref, - CommandRequest { - argv: vec![ - "sh".into(), - "-c".into(), - "cat \"$0\" 2>/dev/null; rm -f \"$0\"".into(), - out, - ], - cwd: None, - timeout_ms: Some(10_000), - stdin: None, - }, - ctx, - ) - .await; - let Ok(result) = result else { - return Vec::new(); - }; - result - .stdout - .lines() - .filter_map(|line| serde_json::from_str::(line).ok()) - .filter(|event| event.get("t").and_then(Value::as_str) != Some("recorder")) - .collect() + .collect_recording(computer_ref, recording_request(skill_id, ctx), ctx) + .await + { + Ok(result) => result.events, + Err(error) => { + tracing::warn!("teach {skill_id}: collect recording failed: {error}"); + Vec::new() + } + } } /// Stop recording, gather the trace, hand the desktop back and distil the diff --git a/crates/api/src/state.rs b/crates/api/src/state.rs index cfa7ff2..d96e31d 100644 --- a/crates/api/src/state.rs +++ b/crates/api/src/state.rs @@ -47,7 +47,10 @@ pub struct CallLease { impl Drop for CallLease { fn drop(&mut self) { let mut map = self.registry.lock(); - if map.get(&self.bot_id).is_some_and(|held| *held == self.call_id) { + if map + .get(&self.bot_id) + .is_some_and(|held| *held == self.call_id) + { map.remove(&self.bot_id); } } diff --git a/crates/api/src/tools.rs b/crates/api/src/tools.rs index ef9efac..c893b3d 100644 --- a/crates/api/src/tools.rs +++ b/crates/api/src/tools.rs @@ -4,10 +4,10 @@ use lazyboy_contracts::{ ComputerAction, ComputerMode, ComputerObservation, PointerType, UiElement, }; use lazyboy_control::{ - ActionError, ActionRequest, AdapterContext, CdpPage, CommandRequest, ComputerRef, - SandboxProvider, a11y_command_on, apply_element_targets, browser_gui_block, cdp_command_on, - click_fingerprint, element_id, format_ui_elements, frames_match, merge_ui_elements, - cdp_stdin_command_on, overlay_elements, parse_a11y_page, parse_cdp_page, parse_computer_actions, + ActionError, ActionRequest, AdapterContext, BrowserRequest, CdpPage, CommandRequest, + ComputerRef, SandboxProvider, a11y_command_on, apply_element_targets, browser_gui_block, + cdp_stdin_command_on, click_fingerprint, element_id, format_ui_elements, frames_match, + merge_ui_elements, overlay_elements, parse_a11y_page, parse_cdp_page, parse_computer_actions, resolve_bot_workspace_cwd, resolve_bot_workspace_path, should_block_stale_click, }; use rig_core::completion::ToolDefinition; @@ -454,9 +454,11 @@ fn vision_guard(ctx: &ToolCtx) -> Option { } fn observation_text(note: &str, observation: &ComputerObservation, unchanged: bool) -> String { - let label = if observation.elements.iter().any(|element| { - matches!(element.kind.as_deref(), Some("dom") | Some("a11y")) - }) { + let label = if observation + .elements + .iter() + .any(|element| matches!(element.kind.as_deref(), Some("dom") | Some("a11y"))) + { "Clickable controls" } else { "Clickable windows" @@ -481,7 +483,11 @@ async fn observe(ctx: &ToolCtx) -> ToolOutcome { if let Some(blocked) = vision_guard(ctx) { return blocked; } - match ctx.sandbox.observe(&ctx.computer_ref(), &ctx.adapter()).await { + match ctx + .sandbox + .observe(&ctx.computer_ref(), &ctx.adapter()) + .await + { Ok(observation) => { let (observation, note) = attach_ui_elements(ctx, observation, "computer observed").await; @@ -509,7 +515,11 @@ async fn wait_then_observe(ctx: &ToolCtx, args: &Value) -> ToolOutcome { if vision_guard(ctx).is_some() { return text_outcome(format!("waited {seconds:.0}s")); } - match ctx.sandbox.observe(&ctx.computer_ref(), &ctx.adapter()).await { + match ctx + .sandbox + .observe(&ctx.computer_ref(), &ctx.adapter()) + .await + { Ok(observation) => { let note = format!("waited {seconds:.0}s"); let (observation, note) = attach_ui_elements(ctx, observation, ¬e).await; @@ -559,11 +569,7 @@ async fn a11y_snapshot(ctx: &ToolCtx, include_browser: bool) -> Option lazyboy_control::A11yPage { @@ -607,41 +613,26 @@ async fn cdp_snapshot(ctx: &ToolCtx, ensure: bool) -> Option { async fn cdp_call(ctx: &ToolCtx, request: Value) -> CdpPage { let adapter = ctx.adapter(); - let display = adapter.display.as_deref().unwrap_or(":1"); - // Clicks may sit through a page's stay timer (CLICK_WAIT_MS in cdp.py). - let timeout_ms = if request.get("action").and_then(Value::as_str) == Some("click") { - let wait = request - .get("waitMs") - .and_then(Value::as_u64) - .unwrap_or(45_000) - .min(120_000); - wait + 25_000 - } else { - 20_000 - }; - let argv = cdp_command_on(display, adapter.profile_path.as_deref(), &request); + let mut browser_request: BrowserRequest = + serde_json::from_value(request.clone()).unwrap_or_default(); + if browser_request.action.is_empty() { + browser_request.action = request + .get("action") + .and_then(Value::as_str) + .unwrap_or("snapshot") + .to_string(); + } + if let Some(ensure) = request.get("ensure").and_then(Value::as_bool) { + browser_request.ensure = ensure; + } + browser_request.display = adapter.display.clone(); + browser_request.profile_path = adapter.profile_path.clone(); match ctx .sandbox - .execute( - &ctx.computer_ref(), - CommandRequest { - argv, - cwd: None, - timeout_ms: Some(timeout_ms), - stdin: None, - }, - &ctx.adapter(), - ) + .browser(&ctx.computer_ref(), browser_request, &adapter) .await { - Ok(result) => { - let raw = if result.stdout.trim().is_empty() { - result.stderr - } else { - result.stdout - }; - parse_cdp_page(&raw) - } + Ok(page) => page, Err(error) => CdpPage { ok: false, error: Some(error.to_string()), @@ -652,33 +643,55 @@ async fn cdp_call(ctx: &ToolCtx, request: Value) -> CdpPage { fn is_connection_check(page: &CdpPage) -> bool { let text = format!("{} {}", page.title, page.text).to_lowercase(); - page.ok && (text.contains("需要確認您的連線是安全") || (text.contains("cloudflare") && [ - "verify you are human", "verifying you are human", "checking your browser", - "checking if the site connection is secure", "needs to review the security", - "驗證您是人類", "验证您是人类", "確認您的連線是安全", "確認您的人類身分", - ].iter().any(|marker| text.contains(marker)))) + page.ok + && (text.contains("需要確認您的連線是安全") + || (text.contains("cloudflare") + && [ + "verify you are human", + "verifying you are human", + "checking your browser", + "checking if the site connection is secure", + "needs to review the security", + "驗證您是人類", + "验证您是人类", + "確認您的連線是安全", + "確認您的人類身分", + ] + .iter() + .any(|marker| text.contains(marker)))) } fn connection_takeover(ctx: &ToolCtx, reason: &str) -> ToolOutcome { *ctx.takeover_requested.lock().unwrap() = true; ToolOutcome { - text: reason.into(), image: None, pause: true, + text: reason.into(), + image: None, + pause: true, blocks: login_blocks(&json!({"reason":reason,"site":"網站連線驗證", "why":"完成驗證後，繼續原本的瀏覽任務。"})), } } async fn connection_check(ctx: &ToolCtx, args: &Value) -> ToolOutcome { - if let Some(blocked) = vision_guard(ctx) { return blocked; } - let (Some(x), Some(y)) = (args.get("x").and_then(Value::as_u64), args.get("y").and_then(Value::as_u64)) else { - return text_outcome("Take a fresh computer_observe and provide the visible checkbox's non-negative screen x/y."); + if let Some(blocked) = vision_guard(ctx) { + return blocked; + } + let (Some(x), Some(y)) = ( + args.get("x").and_then(Value::as_u64), + args.get("y").and_then(Value::as_u64), + ) else { + return text_outcome( + "Take a fresh computer_observe and provide the visible checkbox's non-negative screen x/y.", + ); }; if x > i32::MAX as u64 || y > i32::MAX as u64 { return text_outcome("Checkbox coordinates are out of range."); } let page = cdp_call(ctx, json!({"action":"snapshot","ensure":false})).await; if !is_connection_check(&page) { - return text_outcome("No supported Cloudflare connection-check page was confirmed. Re-observe; use request_takeover for other CAPTCHA, login or 2FA. No click was sent."); + return text_outcome( + "No supported Cloudflare connection-check page was confirmed. Re-observe; use request_takeover for other CAPTCHA, login or 2FA. No click was sent.", + ); } let already_attempted = { let mut attempted = ctx.connection_check_attempted.lock().unwrap(); @@ -694,10 +707,20 @@ async fn connection_check(ctx: &ToolCtx, args: &Value) -> ToolOutcome { Err(error) => return text_outcome(error.to_string()), }; let adapter = ctx.adapter(); - let result = ctx.sandbox.act(&ctx.computer_ref(), ActionRequest { - actions, observe:false, settle_ms:350, - display:adapter.display.clone(), profile_path:adapter.profile_path.clone(), - }, &adapter).await; + let result = ctx + .sandbox + .act( + &ctx.computer_ref(), + ActionRequest { + actions, + observe: false, + settle_ms: 350, + display: adapter.display.clone(), + profile_path: adapter.profile_path.clone(), + }, + &adapter, + ) + .await; if result.is_err() { return connection_takeover(ctx, "無法確認驗證點擊是否完成，請接管檢查。"); } @@ -708,12 +731,18 @@ async fn connection_check(ctx: &ToolCtx, args: &Value) -> ToolOutcome { // can also remove the checkbox. Return evidence for the task check. if after.ok && !is_connection_check(&after) && !after.text.trim().is_empty() { let mut outcome = observe(ctx).await; - outcome.text = format!("Connection-check markers disappeared. This is NOT proof of success. Confirm the requested content is actually visible before continuing; if a challenge/error remains, request_takeover.\n{}\n{}", - browser_result_text("snapshot", &after), outcome.text); + outcome.text = format!( + "Connection-check markers disappeared. This is NOT proof of success. Confirm the requested content is actually visible before continuing; if a challenge/error remains, request_takeover.\n{}\n{}", + browser_result_text("snapshot", &after), + outcome.text + ); return outcome; } } - connection_takeover(ctx, "已嘗試一次驗證並等待，仍無法確認通過。請接管完成驗證，之後繼續原任務。") + connection_takeover( + ctx, + "已嘗試一次驗證並等待，仍無法確認通過。請接管完成驗證，之後繼續原任務。", + ) } async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome { @@ -802,7 +831,11 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome { blocks: Vec::new(), }; } - match ctx.sandbox.observe(&ctx.computer_ref(), &ctx.adapter()).await { + match ctx + .sandbox + .observe(&ctx.computer_ref(), &ctx.adapter()) + .await + { Ok(observation) => { let (observation, note) = attach_ui_elements(ctx, observation, &text).await; pack_observation(ctx, ¬e, observation) @@ -819,7 +852,10 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome { fn browser_result_text(action: &str, page: &CdpPage) -> String { format!( "browser {action}\nPage: {} {}\nClickable page elements: {}\nVisible text:\n{}", - page.title, page.url, format_ui_elements(&page.elements), page.text + page.title, + page.url, + format_ui_elements(&page.elements), + page.text ) } @@ -1201,13 +1237,11 @@ async fn shell(ctx: &ToolCtx, args: &Value) -> ToolOutcome { Ok(result) => result, Err(error) => return text_outcome(error.to_string()), }; - text_outcome(format!( - "{}\n{}", - result.stdout.trim_end(), - result.stderr.trim_end() + text_outcome( + format!("{}\n{}", result.stdout.trim_end(), result.stderr.trim_end()) + .trim_end() + .to_string(), ) - .trim_end() - .to_string()) } async fn list_files(ctx: &ToolCtx, args: &Value) -> ToolOutcome { @@ -1295,7 +1329,12 @@ async fn write_file(ctx: &ToolCtx, args: &Value) -> ToolOutcome { }; match ctx .sandbox - .write_file(&ctx.computer_ref(), &stored, content.as_bytes(), &ctx.adapter()) + .write_file( + &ctx.computer_ref(), + &stored, + content.as_bytes(), + &ctx.adapter(), + ) .await { Ok(()) => ToolOutcome { @@ -1438,7 +1477,9 @@ async fn list_saved_accounts(ctx: &ToolCtx) -> ToolOutcome { }) .collect(); if slim.is_empty() { - text_outcome("No saved logins. Ask the human to add one under 帳號, or call request_takeover so they can sign in on the screen.") + text_outcome( + "No saved logins. Ask the human to add one under 帳號, or call request_takeover so they can sign in on the screen.", + ) } else { text_outcome(json!({"accounts": slim}).to_string()) } @@ -1640,13 +1681,33 @@ mod connection_check_tests { use super::*; #[test] fn recognizes_connection_wall_but_not_cloudflare_footer() { - for text in ["Cloudflare 驗證您是人類", "Cloudflare Verify you are human", "Dcard 需要確認您的連線是安全的"] { - assert!(is_connection_check(&CdpPage { ok:true, text:text.into(), ..Default::default() })); + for text in [ + "Cloudflare 驗證您是人類", + "Cloudflare Verify you are human", + "Dcard 需要確認您的連線是安全的", + ] { + assert!(is_connection_check(&CdpPage { + ok: true, + text: text.into(), + ..Default::default() + })); } - for text in ["Article text. Protected by Cloudflare", "Sign in with your password", ""] { - assert!(!is_connection_check(&CdpPage { ok:true, text:text.into(), ..Default::default() })); + for text in [ + "Article text. Protected by Cloudflare", + "Sign in with your password", + "", + ] { + assert!(!is_connection_check(&CdpPage { + ok: true, + text: text.into(), + ..Default::default() + })); } - assert!(!is_connection_check(&CdpPage { ok:false, text:"Cloudflare Verify you are human".into(), ..Default::default() })); + assert!(!is_connection_check(&CdpPage { + ok: false, + text: "Cloudflare Verify you are human".into(), + ..Default::default() + })); } } @@ -1686,29 +1747,41 @@ mod shell_session_tests { argv(json!({"session":"build","keys":"C-c"})), ["lazyboy-shell", "keys", "build", "C-c"] ); - assert_eq!(argv(json!({"reset":true})), ["lazyboy-shell", "reset", "main"]); + assert_eq!( + argv(json!({"reset":true})), + ["lazyboy-shell", "reset", "main"] + ); assert!(shell_argv(&json!({"keys":" "}), "main", None).is_err()); } #[test] fn cwd_moves_this_command_into_a_directory_and_survives_quotes() { - let argv = shell_argv(&json!({"command":"make\ntest"}), "main", Some("/tmp/a b'c")) - .expect("argv"); + let argv = + shell_argv(&json!({"command":"make\ntest"}), "main", Some("/tmp/a b'c")).expect("argv"); assert_eq!(argv[4], "cd -- '/tmp/a b'\\''c' && {\nmake\ntest\n}"); } #[test] fn waits_stay_inside_the_range_the_desktop_can_honour() { assert_eq!(shell_wait_ms(&json!({"wait_ms":900})), 1_000); - assert_eq!(shell_wait_ms(&json!({"wait_ms":900_000})), SHELL_WAIT_MS_MAX); + assert_eq!( + shell_wait_ms(&json!({"wait_ms":900_000})), + SHELL_WAIT_MS_MAX + ); assert_eq!(shell_wait_ms(&json!({})), SHELL_WAIT_MS_DEFAULT); assert_eq!(shell_log_lines(&json!({"log_lines":0})), 1); } #[test] fn only_a_missing_script_falls_back_to_one_shot() { - assert!(shell_script_missing(127, "bash: lazyboy-shell: command not found")); - assert!(!shell_script_missing(127, "bash: whatever: command not found")); + assert!(shell_script_missing( + 127, + "bash: lazyboy-shell: command not found" + )); + assert!(!shell_script_missing( + 127, + "bash: whatever: command not found" + )); assert!(!shell_script_missing(1, "lazyboy-shell: nope")); } } diff --git a/crates/api/src/vault.rs b/crates/api/src/vault.rs index a482139..0653bea 100644 --- a/crates/api/src/vault.rs +++ b/crates/api/src/vault.rs @@ -331,8 +331,14 @@ fn vault_key() -> Result<[u8; 32], String> { let material = std::env::var("LAZYBOY_VAULT_KEY") .ok() .filter(|value| !value.is_empty()) - .or_else(|| std::env::var("LAZYBOY_APP_TOKEN").ok().filter(|v| !v.is_empty())) - .ok_or_else(|| "set LAZYBOY_VAULT_KEY or LAZYBOY_APP_TOKEN to encrypt saved passwords".to_string())?; + .or_else(|| { + std::env::var("LAZYBOY_APP_TOKEN") + .ok() + .filter(|v| !v.is_empty()) + }) + .ok_or_else(|| { + "set LAZYBOY_VAULT_KEY or LAZYBOY_APP_TOKEN to encrypt saved passwords".to_string() + })?; let digest = Sha256::digest(material.as_bytes()); let mut key = [0u8; 32]; key.copy_from_slice(&digest); @@ -383,7 +389,10 @@ mod tests { #[test] fn host_strips_urls() { - assert_eq!(normalize_host("https://mail.google.com/inbox", "Gmail"), "mail.google.com"); + assert_eq!( + normalize_host("https://mail.google.com/inbox", "Gmail"), + "mail.google.com" + ); assert_eq!(normalize_host("", "Gmail"), "gmail"); } } diff --git a/crates/api/src/voice.rs b/crates/api/src/voice.rs index 433c56d..e692159 100644 --- a/crates/api/src/voice.rs +++ b/crates/api/src/voice.rs @@ -15,7 +15,10 @@ use crate::voice_call; pub fn router() -> Router { Router::new() - .route("/api/voice/settings", get(get_settings).patch(update_settings)) + .route( + "/api/voice/settings", + get(get_settings).patch(update_settings), + ) .route("/api/sessions/{id}/call", get(voice_call::call_ws)) } @@ -193,7 +196,9 @@ async fn update_settings( if model_id.is_empty() || voice_id.is_empty() { return Err(StatusCode::BAD_REQUEST); } - if catalog_voices(provider).iter().all(|(id, _)| *id != voice_id) + if catalog_voices(provider) + .iter() + .all(|(id, _)| *id != voice_id) && provider != VoiceProvider::Scripted { return Err(StatusCode::BAD_REQUEST); @@ -210,7 +215,14 @@ async fn update_settings( }; state .db - .update_voice_settings(&actor, input.enabled, provider.as_str(), model_id, voice_id, api_key) + .update_voice_settings( + &actor, + input.enabled, + provider.as_str(), + model_id, + voice_id, + api_key, + ) .await .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; get_settings(State(state)).await diff --git a/crates/api/src/voice_call.rs b/crates/api/src/voice_call.rs index 34dc94b..4193425 100644 --- a/crates/api/src/voice_call.rs +++ b/crates/api/src/voice_call.rs @@ -1,11 +1,11 @@ use std::sync::Arc; use std::time::Duration; +use axum::Json; use axum::extract::ws::{Message as AxumMessage, WebSocket, WebSocketUpgrade}; use axum::extract::{Path, State}; use axum::http::StatusCode; use axum::response::IntoResponse; -use axum::Json; use futures_util::{SinkExt, StreamExt}; use lazyboy_contracts::SessionAttachment; use lazyboy_harness::{VoiceConnectRequest, VoiceEvent, VoiceSocket, create_voice}; @@ -115,7 +115,9 @@ async fn prepare_call( .ok_or((StatusCode::NOT_FOUND, "workspace not found".into()))?; let (provider, resolved) = resolve_space_voice(&space).map_err(|message| (StatusCode::CONFLICT, message))?; - let history = recent_text_history(state, session_id).await.unwrap_or_default(); + let history = recent_text_history(state, session_id) + .await + .unwrap_or_default(); let mut connect = VoiceConnectRequest::from_resolved( &resolved, voice_instructions(&bot_name, &bot_instructions), @@ -291,7 +293,8 @@ async fn handle_provider_event( }; user_partial.clear(); if !body.is_empty() { - let _ = persist_transcript(state, session_id, "user", &body, &prep.call_id).await; + let _ = + persist_transcript(state, session_id, "user", &body, &prep.call_id).await; send_json( client_write, json!({"type":"transcript","role":"user","text":body,"final":true}), @@ -340,8 +343,16 @@ async fn handle_provider_event( name, arguments, } => { - let output = dispatch_voice_tool(state, actor, &prep.bot_id, session_id, &prep.call_id, &name, &arguments) - .await; + let output = dispatch_voice_tool( + state, + actor, + &prep.bot_id, + session_id, + &prep.call_id, + &name, + &arguments, + ) + .await; let _ = provider .send(VoiceEvent::FunctionCallOutput { call_id, @@ -543,8 +554,7 @@ async fn watch_computer( ) .await?; if let Some(line) = speakable_progress(&previous, &snapshot) { - let urgent = snapshot.get("status").and_then(Value::as_str) - == Some("takeover") + let urgent = snapshot.get("status").and_then(Value::as_str) == Some("takeover") || snapshot.get("status").and_then(Value::as_str) == Some("failed"); if urgent || last_spoken_at.elapsed() > Duration::from_secs(6) { let _ = provider.send(VoiceEvent::SpeakNow { text: line }).await; @@ -652,6 +662,10 @@ mod tests { ); assert!(speakable_progress(&takeover.to_string(), &takeover).is_none()); let idle = json!({"status":"idle"}); - assert!(speakable_progress("{\"status\":\"running\"}", &idle).unwrap().contains("做完")); + assert!( + speakable_progress("{\"status\":\"running\"}", &idle) + .unwrap() + .contains("做完") + ); } } diff --git a/crates/api/src/web_static.rs b/crates/api/src/web_static.rs index 1cd894e..a726f75 100644 --- a/crates/api/src/web_static.rs +++ b/crates/api/src/web_static.rs @@ -62,10 +62,8 @@ mod tests { use std::fs; fn scratch(name: &str) -> PathBuf { - let path = std::env::temp_dir().join(format!( - "lazyboy-web-static-{}-{name}", - std::process::id() - )); + let path = + std::env::temp_dir().join(format!("lazyboy-web-static-{}-{name}", std::process::id())); let _ = fs::remove_dir_all(&path); fs::create_dir_all(&path).unwrap(); path diff --git a/crates/control/Cargo.toml b/crates/control/Cargo.toml index 68bd0d3..bcd51c0 100644 --- a/crates/control/Cargo.toml +++ b/crates/control/Cargo.toml @@ -16,6 +16,9 @@ hex.workspace = true chrono.workspace = true async-trait = "0.1" image.workspace = true +tokio.workspace = true +tracing.workspace = true +base64.workspace = true [lints] workspace = true diff --git a/crates/control/src/a11y.rs b/crates/control/src/a11y.rs index e13c5ac..f5de9c6 100644 --- a/crates/control/src/a11y.rs +++ b/crates/control/src/a11y.rs @@ -139,7 +139,10 @@ mod tests { assert!(argv.contains(&"DISPLAY=:2".into())); assert!(argv.iter().any(|item| item.contains("python3"))); assert!(argv.last().unwrap().contains("\"display\":\":2\"")); - assert!(argv.iter().any(|item| item.contains("Atspi") || item.contains("atspi"))); + assert!( + argv.iter() + .any(|item| item.contains("Atspi") || item.contains("atspi")) + ); } #[test] diff --git a/crates/control/src/actions.rs b/crates/control/src/actions.rs index e5d42b3..59b8299 100644 --- a/crates/control/src/actions.rs +++ b/crates/control/src/actions.rs @@ -33,9 +33,12 @@ pub enum ActionError { /// instead of failing the click. pub fn element_id(value: Option<&Value>) -> Option { match value? { - Value::Number(number) => number - .as_u64() - .or_else(|| number.as_f64().filter(|f| *f >= 0.0).map(|f| f.round() as u64)), + Value::Number(number) => number.as_u64().or_else(|| { + number + .as_f64() + .filter(|f| *f >= 0.0) + .map(|f| f.round() as u64) + }), Value::String(text) => text .trim() .trim_start_matches(['#', '[']) @@ -611,9 +614,7 @@ mod tests { assert!(actions[0].get("x").is_none()); let parsed = parse_computer_actions(&actions).unwrap(); match &parsed[0] { - ComputerAction::Ref { - ref_kind, verb, .. - } => { + ComputerAction::Ref { ref_kind, verb, .. } => { assert_eq!(ref_kind, "dom"); assert_eq!(*verb, RefVerb::Click); } @@ -636,9 +637,7 @@ mod tests { }]; let blocked = browser_gui_block(&json!([{"kind":"click","x":40,"y":80}]), &elements); assert!(blocked.unwrap().contains("browser")); - assert!( - browser_gui_block(&json!([{"kind":"click","element":1}]), &elements).is_none() - ); + assert!(browser_gui_block(&json!([{"kind":"click","element":1}]), &elements).is_none()); } #[test] diff --git a/crates/control/src/cdp.rs b/crates/control/src/cdp.rs index e14f4ce..e4f7dd8 100644 --- a/crates/control/src/cdp.rs +++ b/crates/control/src/cdp.rs @@ -1,4 +1,5 @@ use lazyboy_contracts::UiElement; +use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use crate::x11::parse_ui_elements; @@ -52,11 +53,7 @@ pub fn cdp_stdin_command_on(display: &str, profile: Option<&str>) -> Vec if let Some(profile) = profile.filter(|value| !value.is_empty()) { env.push(format!("LAZYBOY_BROWSER_PROFILE={profile}")); } - env.extend([ - "python3".into(), - "-c".into(), - CDP_PY.into(), - ]); + env.extend(["python3".into(), "-c".into(), CDP_PY.into()]); env } @@ -70,6 +67,23 @@ pub fn teach_recorder_output(skill_id: &str) -> String { format!("/tmp/{}.jsonl", teach_recorder_tag(skill_id)) } +pub fn sanitize_skill_id(skill_id: &str) -> String { + let cleaned: String = skill_id + .chars() + .filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_') + .take(80) + .collect(); + if cleaned.is_empty() { + "unknown".into() + } else { + cleaned + } +} + +pub fn teach_trajectory_dir(skill_id: &str) -> String { + format!("/tmp/lazyboy/teach-{}", sanitize_skill_id(skill_id)) +} + /// Detached, long-running CDP recorder for a human demonstration. The script /// is handed to `sh` as a positional argument so no shell quoting touches it. pub fn cdp_record_command_on(display: &str, profile: Option<&str>, skill_id: &str) -> Vec { @@ -87,7 +101,8 @@ pub fn cdp_record_command_on(display: &str, profile: Option<&str>, skill_id: &st vec![ "sh".into(), "-c".into(), - "setsid nohup env DISPLAY=\"$0\" python3 -c \"$1\" \"$2\" >/dev/null 2>&1 /dev/null 2>&1 Vec { vec!["pkill".into(), "-f".into(), teach_recorder_tag(skill_id)] } -#[derive(Debug, Clone, PartialEq, Default)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct CdpPage { pub ok: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, + #[serde(default)] pub url: String, + #[serde(default)] pub title: String, + #[serde(default)] pub text: String, + #[serde(default)] pub restarted: bool, /// Seconds the click waited for a disabled control to become enabled. + #[serde(default, skip_serializing_if = "Option::is_none")] pub waited_seconds: Option, + #[serde(default)] pub elements: Vec, } @@ -164,6 +187,12 @@ mod tests { assert_eq!(devtools_port("3"), 9224); } + #[test] + fn trajectory_dir_strips_path_chars() { + assert_eq!(teach_trajectory_dir("abc/../x"), "/tmp/lazyboy/teach-abcx"); + assert_eq!(sanitize_skill_id(""), "unknown"); + } + #[test] fn command_passes_port_profile_and_script() { let argv = cdp_command_on( diff --git a/crates/control/src/controller.rs b/crates/control/src/controller.rs new file mode 100644 index 0000000..c833e92 --- /dev/null +++ b/crates/control/src/controller.rs @@ -0,0 +1,215 @@ +use std::str::FromStr; +use std::sync::Arc; + +use async_trait::async_trait; +use thiserror::Error; + +use crate::cua::CuaController; +use crate::legacy::LegacyController; +use crate::{ + ActionRequest, ActionResult, BrowserRequest, CdpPage, RecordingRequest, RecordingResult, + RecordingSession, +}; +use lazyboy_contracts::ComputerObservation; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ControlContext { + pub display: String, + pub profile_path: Option, +} + +impl ControlContext { + pub fn new(display: impl Into, profile_path: Option) -> Self { + Self { + display: display.into(), + profile_path, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ControllerHealth { + pub backend: String, + pub version: Option, + pub healthy: bool, + pub degraded: bool, + pub details: Vec, +} + +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ControlError { + #[error("computer driver is not installed")] + DriverUnavailable, + #[error("computer driver is unhealthy")] + DriverUnhealthy, + #[error("display is unavailable")] + DisplayUnavailable, + #[error("accessibility is unavailable")] + AccessibilityUnavailable, + #[error("browser is unavailable")] + BrowserUnavailable, + #[error("target not found")] + TargetNotFound, + #[error("stale UI reference; take a fresh observation")] + StaleReference, + #[error("permission denied")] + PermissionDenied, + #[error("computer action timed out")] + Timeout, + #[error("unsupported computer action")] + Unsupported, + #[error("computer is busy")] + Busy, + #[error("{0}")] + InvalidAction(String), + #[error("{0}")] + Internal(String), +} + +impl ControlError { + pub fn internal(text: impl Into) -> Self { + let text = text.into(); + Self::Internal(truncate_error(&text)) + } + + pub fn is_client_error(&self) -> bool { + matches!( + self, + Self::TargetNotFound + | Self::StaleReference + | Self::Unsupported + | Self::InvalidAction(_) + | Self::PermissionDenied + ) + } +} + +fn truncate_error(text: &str) -> String { + const LIMIT: usize = 800; + let trimmed = text.trim(); + if trimmed.len() <= LIMIT { + trimmed.to_string() + } else { + format!("{}…", &trimmed[..LIMIT]) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ComputerDriver { + Legacy, + Cua, +} + +impl ComputerDriver { + pub const ENV: &'static str = "LAZYBOY_COMPUTER_DRIVER"; + + pub fn from_env() -> Self { + match std::env::var(Self::ENV) { + Ok(value) if value.trim().is_empty() => Self::Cua, + Ok(value) => match value.parse() { + Ok(driver) => driver, + Err(error) => { + tracing::error!("{error}; using cua"); + Self::Cua + } + }, + Err(_) => Self::Cua, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Legacy => "legacy", + Self::Cua => "cua", + } + } + + pub fn controller(self) -> Arc { + match self { + Self::Legacy => Arc::new(LegacyController), + Self::Cua => Arc::new(CuaController::default()), + } + } +} + +impl FromStr for ComputerDriver { + type Err = UnknownComputerDriver; + + fn from_str(value: &str) -> Result { + match value.trim().to_ascii_lowercase().as_str() { + "legacy" => Ok(Self::Legacy), + "cua" => Ok(Self::Cua), + other => Err(UnknownComputerDriver(other.to_string())), + } + } +} + +#[derive(Debug, Error, Clone, PartialEq, Eq)] +#[error("unknown computer driver {0:?}; expected legacy or cua")] +pub struct UnknownComputerDriver(pub String); + +#[async_trait] +pub trait ComputerController: Send + Sync { + fn backend(&self) -> ComputerDriver; + + async fn health(&self, ctx: &ControlContext) -> Result; + + async fn observe(&self, ctx: &ControlContext) -> Result; + + async fn act( + &self, + request: &ActionRequest, + ctx: &ControlContext, + ) -> Result; + + async fn browser( + &self, + request: &BrowserRequest, + ctx: &ControlContext, + ) -> Result; + + async fn start_recording( + &self, + request: &RecordingRequest, + ctx: &ControlContext, + ) -> Result; + + async fn stop_recording( + &self, + request: &RecordingRequest, + ctx: &ControlContext, + ) -> Result<(), ControlError>; + + async fn collect_recording( + &self, + request: &RecordingRequest, + ctx: &ControlContext, + ) -> Result; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_driver_names() { + assert_eq!( + "legacy".parse::().unwrap(), + ComputerDriver::Legacy + ); + assert_eq!( + "CUA".parse::().unwrap(), + ComputerDriver::Cua + ); + assert!("xdotool".parse::().is_err()); + assert_eq!(ComputerDriver::Cua.as_str(), "cua"); + } + + #[test] + fn invalid_browser_action_is_a_client_error() { + let error = ControlError::InvalidAction( + "browser navigate only accepts http, https, or about URLs".into(), + ); + assert!(error.is_client_error()); + } +} diff --git a/crates/control/src/cua/browser.rs b/crates/control/src/cua/browser.rs new file mode 100644 index 0000000..87d24ad --- /dev/null +++ b/crates/control/src/cua/browser.rs @@ -0,0 +1,602 @@ +use lazyboy_contracts::UiElement; +use serde_json::{Value, json}; +use tokio::time::{Duration, sleep}; + +use super::ListedWindow; +use super::client::{CuaClient, first_array_of_objects}; +use crate::controller::ControlError; +use crate::process::spawn_detached; +use crate::{BrowserRequest, CdpPage, launch_argv_on}; + +pub const SESSION: &str = "lazyboy"; + +#[derive(Debug, Clone)] +pub struct BrowserBind { + pub pid: u64, + pub window_id: u64, + pub target_id: String, + pub tab_id: String, +} + +pub fn is_cua_ref(selector: &str) -> bool { + let mut parts = selector.split(':'); + matches!( + (parts.next(), parts.next(), parts.next()), + (Some(prefix), Some(index), None) + if prefix.starts_with('p') + && prefix[1..].chars().all(|ch| ch.is_ascii_digit()) + && !prefix[1..].is_empty() + && index.chars().all(|ch| ch.is_ascii_digit()) + && !index.is_empty() + ) +} + +pub fn allowed_navigate_url(url: &str) -> bool { + url.starts_with("http://") || url.starts_with("https://") || url.starts_with("about:") +} + +pub fn page_from_semantic(value: &Value) -> CdpPage { + let page = value.get("page").unwrap_or(value); + let url = page + .get("url") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let title = page + .get("title") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let outline = value.get("outline").and_then(Value::as_str).unwrap_or(""); + let mut elements = Vec::new(); + for (index, item) in first_array_of_objects(value, "ref").into_iter().enumerate() { + let Some(element) = element_from_ref(index as u32 + 1, item) else { + continue; + }; + elements.push(element); + } + let ok = value.get("status").and_then(Value::as_str) != Some("refused") + && value.get("ok").and_then(Value::as_bool) != Some(false); + CdpPage { + ok, + error: if ok { + None + } else { + value + .get("message") + .or_else(|| value.get("error")) + .and_then(Value::as_str) + .map(str::to_string) + }, + url, + title, + text: outline.to_string(), + restarted: false, + waited_seconds: None, + elements, + } +} + +fn element_from_ref(id: u32, item: &Value) -> Option { + let selector = item.get("ref").and_then(Value::as_str)?.to_string(); + if selector.is_empty() { + return None; + } + let name = item + .get("name") + .or_else(|| item.get("label")) + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let role = item + .get("role") + .and_then(Value::as_str) + .filter(|role| !role.is_empty()) + .map(str::to_string); + let visibility = item + .get("visibility") + .and_then(Value::as_str) + .unwrap_or("in_viewport"); + let (x, y, w, h) = match item.get("frame") { + Some(frame) => ( + number(frame, "x").unwrap_or(0), + number(frame, "y").unwrap_or(0), + number(frame, "w") + .or_else(|| number(frame, "width")) + .unwrap_or(0), + number(frame, "h") + .or_else(|| number(frame, "height")) + .unwrap_or(0), + ), + None if visibility == "in_viewport" => (0, 0, 1, 1), + None => (0, 0, 0, 0), + }; + Some(UiElement { + id, + title: if name.is_empty() { + selector.clone() + } else { + name + }, + x, + y, + w, + h, + selector: Some(selector), + kind: Some("dom".into()), + role, + }) +} + +fn number(value: &Value, key: &str) -> Option { + value + .get(key) + .and_then(Value::as_u64) + .or_else(|| { + value + .get(key) + .and_then(Value::as_f64) + .filter(|n| *n >= 0.0) + .map(|n| n as u64) + }) + .map(|n| n as u32) +} + +pub fn find_ref<'a>(page: &'a CdpPage, selector: &'a str) -> Option<&'a str> { + if is_cua_ref(selector) { + return page + .elements + .iter() + .find(|element| element.selector.as_deref() == Some(selector)) + .and_then(|element| element.selector.as_deref()) + .or(Some(selector)); + } + if let Ok(id) = selector.parse::() { + return page + .elements + .iter() + .find(|element| element.id == id) + .and_then(|element| element.selector.as_deref()); + } + let needle = selector + .trim_start_matches(['#', '.', '[']) + .trim_end_matches(']') + .to_ascii_lowercase(); + page.elements.iter().find_map(|element| { + let title = element.title.to_ascii_lowercase(); + let role = element + .role + .clone() + .unwrap_or_default() + .to_ascii_lowercase(); + if title.contains(&needle) || role == needle { + element.selector.as_deref() + } else { + None + } + }) +} + +pub fn chromium_window(windows: &[ListedWindow]) -> Option<&ListedWindow> { + windows.iter().find(|window| { + let blob = format!("{} {}", window.title, window.app_name).to_ascii_lowercase(); + blob.contains("chrom") + }) +} + +fn ids_from(value: &Value) -> Option<(String, String)> { + let mut target_id = None; + let mut tab_id = None; + let mut nodes = Vec::new(); + super::client::walk(value, &mut nodes); + for node in nodes { + if target_id.is_none() { + target_id = node + .get("target_id") + .and_then(Value::as_str) + .map(str::to_string); + } + if tab_id.is_none() { + tab_id = node + .get("tab_id") + .and_then(Value::as_str) + .map(str::to_string); + } + if let Some(tabs) = node.get("tabs").and_then(Value::as_array) + && let Some(first) = tabs.first() + && tab_id.is_none() + { + tab_id = first + .get("tab_id") + .or_else(|| first.get("id")) + .and_then(Value::as_str) + .map(str::to_string); + } + } + Some((target_id?, tab_id?)) +} + +pub async fn ensure_bind( + client: &CuaClient, + display: &str, + profile: Option<&str>, + ensure: bool, + windows: &[ListedWindow], +) -> Result { + let mut listed = windows.to_vec(); + if chromium_window(&listed).is_none() && ensure { + if let Some(argv) = launch_argv_on(display, profile, "browser", None) { + spawn_detached(&argv) + .await + .map_err(ControlError::internal)?; + } + for _ in 0..24 { + sleep(Duration::from_millis(250)).await; + listed = super::CuaController::list_windows_now(client, display).await?; + if chromium_window(&listed).is_some() { + break; + } + } + } + let window = chromium_window(&listed) + .cloned() + .ok_or(ControlError::BrowserUnavailable)?; + attach(client, display, &window).await +} + +async fn attach( + client: &CuaClient, + display: &str, + window: &ListedWindow, +) -> Result { + let pid = window.pid; + let window_id = window.id; + let prepare = client + .call( + display, + "browser_prepare", + &json!({ + "pid": pid, + "window_id": window_id, + "session": SESSION, + "strategy": { "kind": "existing_profile" }, + "allow_launch": false, + }), + &[], + ) + .await; + if let Err(error) = &prepare { + let text = error.to_string(); + if !text.contains("consent") && !text.contains("prepare") && !text.contains("grant") { + tracing::warn!(error = %error, "browser_prepare failed"); + } + } + let state = client + .call( + display, + "get_browser_state", + &json!({ + "pid": pid, + "window_id": window_id, + "session": SESSION, + "include_screenshot": false, + }), + &[], + ) + .await?; + let (target_id, tab_id) = ids_from(&state).ok_or(ControlError::BrowserUnavailable)?; + Ok(BrowserBind { + pid, + window_id, + target_id, + tab_id, + }) +} + +pub async fn snapshot( + client: &CuaClient, + display: &str, + bind: &BrowserBind, +) -> Result { + let value = client + .call( + display, + "get_browser_state", + &json!({ + "target_id": bind.target_id, + "tab_id": bind.tab_id, + "session": SESSION, + "snapshot_format": "semantic_v2", + "include_screenshot": false, + }), + &[], + ) + .await?; + Ok(page_from_semantic(&value)) +} + +pub async fn run( + client: &CuaClient, + display: &str, + profile: Option<&str>, + request: &BrowserRequest, + windows: &[ListedWindow], + bind: &mut Option, +) -> Result { + if request.action == "probe" { + return Ok(CdpPage { + ok: chromium_window(windows).is_some(), + ..CdpPage::default() + }); + } + let attached = match bind.as_ref() { + Some(current) => current.clone(), + None => { + let attached = ensure_bind(client, display, profile, request.ensure, windows).await?; + *bind = Some(attached.clone()); + attached + } + }; + if request.action == "ensure" { + return Ok(CdpPage { + ok: true, + ..CdpPage::default() + }); + } + match request.action.as_str() { + "snapshot" => snapshot(client, display, &attached).await, + "wait" => { + let ms = request.ms.unwrap_or(400).min(5000); + sleep(Duration::from_millis(ms)).await; + snapshot(client, display, &attached).await + } + "navigate" => { + let url = request.url.as_deref().unwrap_or(""); + if !allowed_navigate_url(url) { + return Err(ControlError::InvalidAction( + "browser navigate only accepts http, https, or about URLs".into(), + )); + } + client + .call( + display, + "browser_navigate", + &json!({ + "target_id": attached.target_id, + "tab_id": attached.tab_id, + "session": SESSION, + "url": url, + }), + &[], + ) + .await?; + sleep(Duration::from_millis(800)).await; + snapshot(client, display, &attached).await + } + "click" => click(client, display, &attached, request).await, + "type" => type_into(client, display, &attached, request).await, + "press" => { + let key = map_press_key(request.key.as_deref().unwrap_or("return")); + client + .call( + display, + "press_key", + &json!({ + "key": key, + "pid": attached.pid, + "window_id": attached.window_id, + "session": SESSION, + }), + &[], + ) + .await?; + sleep(Duration::from_millis(200)).await; + snapshot(client, display, &attached).await + } + other => Err(ControlError::InvalidAction(format!( + "unsupported browser action {other}" + ))), + } +} + +async fn click( + client: &CuaClient, + display: &str, + bind: &BrowserBind, + request: &BrowserRequest, +) -> Result { + let selector = request + .selector + .as_deref() + .ok_or_else(|| ControlError::InvalidAction("browser click needs a selector".into()))?; + let wait_ms = request.wait_ms.unwrap_or(45_000).min(120_000); + let deadline = tokio::time::Instant::now() + Duration::from_millis(wait_ms); + let mut waited = 0.0f64; + loop { + let page = snapshot(client, display, bind).await?; + let Some(r#ref) = find_ref(&page, selector) else { + return Ok(CdpPage { + ok: false, + error: Some( + "element gone: the page changed and ids were renumbered. Use the fresh element list in this result." + .into(), + ), + url: page.url, + title: page.title, + text: page.text, + elements: page.elements, + ..CdpPage::default() + }); + }; + let r#ref = r#ref.to_string(); + match client + .call( + display, + "browser_click", + &json!({ + "target_id": bind.target_id, + "tab_id": bind.tab_id, + "session": SESSION, + "ref": r#ref, + "input_route": "dom_event", + }), + &[], + ) + .await + { + Ok(_) => { + sleep(Duration::from_millis(250)).await; + let mut after = snapshot(client, display, bind).await?; + if waited >= 1.0 { + after.waited_seconds = Some((waited * 10.0).round() / 10.0); + } + return Ok(after); + } + Err(error) if tokio::time::Instant::now() < deadline => { + waited += 0.5; + tracing::info!(error = %error, "browser click retrying"); + sleep(Duration::from_millis(500)).await; + } + Err(error) => return Err(error), + } + } +} + +async fn type_into( + client: &CuaClient, + display: &str, + bind: &BrowserBind, + request: &BrowserRequest, +) -> Result { + let text = request.text.clone().unwrap_or_default(); + tracing::info!(backend = "cua", tool = "browser_type", length = text.len()); + if let Some(selector) = request.selector.as_deref() { + let page = snapshot(client, display, bind).await?; + let Some(r#ref) = find_ref(&page, selector) else { + return Ok(CdpPage { + ok: false, + error: Some("target field is unavailable; no text inserted".into()), + url: page.url, + title: page.title, + text: page.text, + elements: page.elements, + ..CdpPage::default() + }); + }; + let r#ref = r#ref.to_string(); + client + .call( + display, + "browser_type", + &json!({ + "target_id": bind.target_id, + "tab_id": bind.tab_id, + "session": SESSION, + "ref": r#ref, + "text": text, + "replace": true, + }), + &[], + ) + .await?; + } else if !text.is_empty() { + client + .call( + display, + "type_text", + &json!({ + "text": text, + "pid": bind.pid, + "window_id": bind.window_id, + "session": SESSION, + }), + &[], + ) + .await?; + } + sleep(Duration::from_millis(200)).await; + snapshot(client, display, bind).await +} + +fn map_press_key(key: &str) -> String { + match key.to_ascii_lowercase().as_str() { + "enter" | "return" => "return".into(), + "esc" | "escape" => "escape".into(), + other => other.to_ascii_lowercase(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_snapshot_scoped_refs() { + assert!(is_cua_ref("p1:1")); + assert!(is_cua_ref("p28:12")); + assert!(!is_cua_ref("#submit")); + assert!(!is_cua_ref("button.primary")); + assert!(!is_cua_ref("p:1")); + } + + #[test] + fn semantic_snapshot_becomes_cdp_page() { + let raw = json!({ + "status": "ok", + "outline": "- button \"Smoke Click\"\n- textbox \"Smoke Entry\"", + "page": { "title": "LazyBoy Cua Smoke", "url": "http://127.0.0.1:8765/cua-smoke.html" }, + "refs": [ + { "name": "Smoke Click", "ref": "p1:1", "role": "button", "visibility": "in_viewport" }, + { "name": "Smoke Entry", "ref": "p1:2", "role": "textbox", "visibility": "in_viewport" } + ] + }); + let page = page_from_semantic(&raw); + assert!(page.ok); + assert_eq!(page.title, "LazyBoy Cua Smoke"); + assert_eq!(page.elements.len(), 2); + assert_eq!(page.elements[0].id, 1); + assert_eq!(page.elements[0].selector.as_deref(), Some("p1:1")); + assert_eq!(page.elements[0].kind.as_deref(), Some("dom")); + assert_eq!(find_ref(&page, "1"), Some("p1:1")); + assert_eq!(find_ref(&page, "p1:1"), Some("p1:1")); + assert_eq!(find_ref(&page, "Smoke Entry"), Some("p1:2")); + } + + #[test] + fn navigate_accepts_http_https_about_only() { + assert!(allowed_navigate_url("https://example.com")); + assert!(allowed_navigate_url("http://127.0.0.1:8765/cua-smoke.html")); + assert!(allowed_navigate_url("about:blank")); + assert!(!allowed_navigate_url("file:///tmp/x.html")); + assert!(!allowed_navigate_url("javascript:alert(1)")); + assert!(!allowed_navigate_url("")); + } + + #[test] + fn chromium_window_matches_title_or_app() { + let chrome = ListedWindow { + id: 9, + pid: 334, + title: "LazyBoy Cua Smoke - Chromium".into(), + app_name: "Chromium".into(), + x: 0, + y: 0, + w: 1280, + h: 800, + z: 2, + }; + let terminal = ListedWindow { + id: 3, + pid: 20, + title: "終端機".into(), + app_name: "xfce4-terminal".into(), + x: 10, + y: 10, + w: 400, + h: 300, + z: 1, + }; + assert!(chromium_window(&[terminal.clone(), chrome.clone()]).is_some()); + assert!(chromium_window(&[terminal]).is_none()); + } +} diff --git a/crates/control/src/cua/client.rs b/crates/control/src/cua/client.rs new file mode 100644 index 0000000..080a80a --- /dev/null +++ b/crates/control/src/cua/client.rs @@ -0,0 +1,238 @@ +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use serde_json::Value; +use tokio::process::Command; + +use crate::controller::ControlError; +use crate::screen::normalize_display; + +pub const PRIMARY_SOCKET: &str = "/tmp/lazyboy/cua.sock"; + +#[derive(Debug, Clone)] +pub struct CuaClient { + bin: PathBuf, +} + +impl Default for CuaClient { + fn default() -> Self { + Self { + bin: PathBuf::from("cua-driver"), + } + } +} + +impl CuaClient { + pub fn socket_for_display(display: &str) -> PathBuf { + let number = normalize_display(display) + .trim_start_matches(':') + .to_string(); + if number == "1" { + PathBuf::from(PRIMARY_SOCKET) + } else { + PathBuf::from(format!("/tmp/lazyboy/cua-{number}.sock")) + } + } + + pub fn dbus_file(display: &str) -> PathBuf { + let number = normalize_display(display) + .trim_start_matches(':') + .to_string(); + PathBuf::from(format!("/tmp/lazyboy/screen-{number}.dbus")) + } + + pub async fn version(&self) -> Result { + let output = Command::new(&self.bin) + .arg("--version") + .output() + .await + .map_err(|_| ControlError::DriverUnavailable)?; + if !output.status.success() { + return Err(ControlError::DriverUnavailable); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) + } + + pub async fn status(&self, display: &str) -> Result { + let socket = Self::socket_for_display(display); + if !socket.exists() { + return Err(ControlError::DriverUnavailable); + } + let output = Command::new(&self.bin) + .args(["status", "--socket", &socket.to_string_lossy()]) + .output() + .await + .map_err(|_| ControlError::DriverUnavailable)?; + let text = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + if !output.status.success() || text.to_ascii_lowercase().contains("not running") { + return Err(ControlError::DriverUnhealthy); + } + Ok(text) + } + + pub async fn call( + &self, + screen: &str, + tool: &str, + payload: &Value, + extra: &[&str], + ) -> Result { + let socket = Self::socket_for_display(screen); + if !socket.exists() { + return Err(ControlError::DriverUnavailable); + } + let mut command = Command::new(&self.bin); + command + .env("DISPLAY", normalize_display(screen)) + .env( + "CUA_DRIVER_RS_HOME", + std::env::var("CUA_DRIVER_RS_HOME") + .unwrap_or_else(|_| "/tmp/lazyboy/cua-home".into()), + ) + .args(["call", "--socket", &socket.to_string_lossy()]); + command.args(extra); + command.arg(tool); + command.arg(payload.to_string()); + apply_desktop_bus(&mut command, screen); + let started = Instant::now(); + let output = command + .output() + .await + .map_err(|error| ControlError::internal(error.to_string()))?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let combined = format!("{stdout}\n{stderr}"); + tracing::info!( + backend = "cua", + tool, + screen, + duration_ms = started.elapsed().as_millis() as u64, + success = output.status.success() && !combined.contains('❌') + ); + if !output.status.success() || combined.contains('❌') { + return Err(classify_cua_failure(&combined)); + } + Ok(parse_jsonish(&stdout).unwrap_or(Value::String(stdout.trim().to_string()))) + } +} + +fn apply_desktop_bus(command: &mut Command, display: &str) { + let dbus = CuaClient::dbus_file(display); + if let Ok(address) = std::fs::read_to_string(&dbus) { + let address = address.trim(); + if !address.is_empty() { + command.env("DBUS_SESSION_BUS_ADDRESS", address); + } + } + let number = normalize_display(display) + .trim_start_matches(':') + .to_string(); + let runtime = PathBuf::from(format!("/tmp/lazyboy/screen-{number}.runtime")); + if let Ok(value) = std::fs::read_to_string(runtime) { + let value = value.trim(); + if !value.is_empty() { + command.env("XDG_RUNTIME_DIR", value); + } + } +} + +pub fn parse_jsonish(text: &str) -> Option { + let text = text.trim(); + if text.is_empty() { + return None; + } + if let Ok(value) = serde_json::from_str::(text) { + return Some(value); + } + let start = text.find('{')?; + let end = text.rfind('}')?; + if end > start { + serde_json::from_str(&text[start..=end]).ok() + } else { + None + } +} + +pub fn walk<'a>(value: &'a Value, out: &mut Vec<&'a Value>) { + out.push(value); + match value { + Value::Object(map) => { + for item in map.values() { + walk(item, out); + } + } + Value::Array(items) => { + for item in items { + walk(item, out); + } + } + _ => {} + } +} + +pub fn first_array_of_objects<'a>(value: &'a Value, required: &str) -> Vec<&'a Value> { + let mut nodes = Vec::new(); + walk(value, &mut nodes); + for node in nodes { + if let Some(items) = node.as_array() + && items.iter().any(|item| item.get(required).is_some()) + { + return items.iter().collect(); + } + if let Some(items) = node.get(required).and_then(Value::as_array) + && items.iter().any(Value::is_object) + { + return items.iter().collect(); + } + } + Vec::new() +} + +fn classify_cua_failure(text: &str) -> ControlError { + let lower = text.to_ascii_lowercase(); + if lower.contains("stale") { + ControlError::StaleReference + } else if lower.contains("not_found") || lower.contains("not found") { + ControlError::TargetNotFound + } else if lower.contains("timeout") { + ControlError::Timeout + } else if lower.contains("permission") || lower.contains("consent") { + ControlError::PermissionDenied + } else if lower.contains("unsupported") { + ControlError::Unsupported + } else if Path::new(PRIMARY_SOCKET) + .parent() + .is_some_and(|dir| !dir.exists()) + { + ControlError::DriverUnavailable + } else { + ControlError::internal(text) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn primary_display_uses_well_known_socket() { + assert_eq!( + CuaClient::socket_for_display(":1"), + PathBuf::from(PRIMARY_SOCKET) + ); + assert_eq!( + CuaClient::socket_for_display(":2"), + PathBuf::from("/tmp/lazyboy/cua-2.sock") + ); + } + + #[test] + fn extracts_json_object_from_noisy_stdout() { + let parsed = parse_jsonish("✅ ok\n{\"status\":\"ok\",\"x\":1}\n").unwrap(); + assert_eq!(parsed["status"], "ok"); + } +} diff --git a/crates/control/src/cua/mod.rs b/crates/control/src/cua/mod.rs new file mode 100644 index 0000000..fc18fe9 --- /dev/null +++ b/crates/control/src/cua/mod.rs @@ -0,0 +1,532 @@ +mod browser; +mod client; +mod record; +mod translate; + +use std::collections::HashMap; +use std::path::PathBuf; +use std::time::{SystemTime, UNIX_EPOCH}; + +use async_trait::async_trait; +use lazyboy_contracts::{ActiveWindow, ComputerObservation, CursorPosition, UiElement}; +use serde_json::{Value, json}; +use tokio::time::{Duration, sleep}; + +use crate::controller::{ + ComputerController, ComputerDriver, ControlContext, ControlError, ControllerHealth, +}; +use crate::legacy::apply_action as legacy_apply_action; +use crate::process::spawn_detached; +use crate::{ + ActionRequest, ActionResult, BrowserRequest, CdpPage, RecordingRequest, RecordingResult, + RecordingSession, action_pause_ms, normalize_display, observation_from_png, + observation_with_elements, teach_trajectory_dir, +}; +use client::first_array_of_objects; + +pub use client::CuaClient; +pub use translate::{TranslatedAction, translate_action}; + +#[derive(Debug, Default)] +pub struct CuaController { + client: CuaClient, + browser: tokio::sync::Mutex>, +} + +#[async_trait] +impl ComputerController for CuaController { + fn backend(&self) -> ComputerDriver { + ComputerDriver::Cua + } + + async fn health(&self, ctx: &ControlContext) -> Result { + let version = self.client.version().await.ok(); + match self.client.status(&ctx.display).await { + Ok(status) => Ok(ControllerHealth { + backend: ComputerDriver::Cua.as_str().to_string(), + version, + healthy: true, + degraded: false, + details: vec![status.trim().to_string()], + }), + Err(error) => Ok(ControllerHealth { + backend: ComputerDriver::Cua.as_str().to_string(), + version, + healthy: false, + degraded: true, + details: vec![error.to_string()], + }), + } + } + + async fn observe(&self, ctx: &ControlContext) -> Result { + self.browser + .lock() + .await + .remove(normalize_display(&ctx.display)); + self.observe_display(&ctx.display).await + } + + async fn act( + &self, + request: &ActionRequest, + ctx: &ControlContext, + ) -> Result { + let display = ctx.display.as_str(); + let profile = ctx.profile_path.as_deref(); + let mut completed = 0usize; + for action in &request.actions { + match translate_action(action, display, profile) { + Ok(translated) => self.dispatch(display, translated).await?, + Err(ControlError::Unsupported) => { + legacy_apply_action(display, profile, action).await?; + } + Err(error) => return Err(error), + } + let pause = action_pause_ms(action); + if pause > 0 { + sleep(Duration::from_millis(pause)).await; + } + completed += 1; + } + if request.settle_ms > 0 { + sleep(Duration::from_millis(u64::from(request.settle_ms))).await; + } + let observation = if request.observe { + Some(self.observe_display(display).await?) + } else { + None + }; + Ok(ActionResult { + completed, + observation, + }) + } + + async fn browser( + &self, + request: &BrowserRequest, + ctx: &ControlContext, + ) -> Result { + let key = normalize_display(&ctx.display).to_string(); + let mut windows = self.windows(&ctx.display).await.unwrap_or_default(); + let mut cache = self.browser.lock().await; + for attempt in 0..2 { + let mut bind = cache.remove(&key); + let had_bind = bind.is_some(); + match browser::run( + &self.client, + &ctx.display, + ctx.profile_path.as_deref(), + request, + &windows, + &mut bind, + ) + .await + { + Err(ControlError::BrowserUnavailable) if !had_bind => { + return Ok(CdpPage { + ok: false, + error: Some("cdp unavailable".into()), + ..CdpPage::default() + }); + } + Err(error) + if attempt == 0 + && had_bind + && matches!( + error, + ControlError::BrowserUnavailable + | ControlError::StaleReference + | ControlError::TargetNotFound + ) => + { + windows = self.windows(&ctx.display).await.unwrap_or_default(); + } + other => { + if let Some(current) = bind { + cache.insert(key, current); + } + return map_browser_unavailable(other); + } + } + } + map_browser_unavailable(Err(ControlError::BrowserUnavailable)) + } + + async fn start_recording( + &self, + request: &RecordingRequest, + ctx: &ControlContext, + ) -> Result { + if request.skill_id.trim().is_empty() { + return Err(ControlError::InvalidAction( + "recording needs a skill id".into(), + )); + } + let output_dir = teach_trajectory_dir(&request.skill_id); + let _ = tokio::fs::create_dir_all(&output_dir).await; + let _ = self + .client + .call(&ctx.display, "stop_recording", &json!({}), &[]) + .await; + if let Err(error) = self + .client + .call( + &ctx.display, + "start_recording", + &json!({ "output_dir": output_dir, "record_video": false }), + &[], + ) + .await + { + tracing::warn!(error = %error, "cua start_recording failed"); + } + if let Err(error) = crate::legacy::start_cdp_recorder( + &ctx.display, + ctx.profile_path.as_deref(), + &request.skill_id, + ) + .await + { + tracing::warn!(error = %error, "cdp recorder start failed"); + } + Ok(RecordingSession { + skill_id: request.skill_id.clone(), + output_dir, + }) + } + + async fn stop_recording( + &self, + request: &RecordingRequest, + ctx: &ControlContext, + ) -> Result<(), ControlError> { + let _ = self + .client + .call(&ctx.display, "stop_recording", &json!({}), &[]) + .await; + crate::legacy::stop_cdp_recorder(&request.skill_id).await + } + + async fn collect_recording( + &self, + request: &RecordingRequest, + _ctx: &ControlContext, + ) -> Result { + let mut events = crate::legacy::collect_cdp_events(&request.skill_id).await; + let dir = teach_trajectory_dir(&request.skill_id); + events.extend(record::events_from_dir(std::path::Path::new(&dir))); + events.sort_by_key(|event| event.get("at").and_then(Value::as_i64).unwrap_or(0)); + let _ = tokio::fs::remove_dir_all(&dir).await; + Ok(RecordingResult { events }) + } +} + +impl CuaController { + async fn observe_display(&self, display: &str) -> Result { + let png_path = observe_png_path(display); + let _ = tokio::fs::remove_file(&png_path).await; + self.client + .call( + display, + "get_desktop_state", + &json!({ "screenshot_out_file": png_path.to_string_lossy() }), + &["--screenshot-out-file", &png_path.to_string_lossy()], + ) + .await?; + let png = tokio::fs::read(&png_path) + .await + .map_err(|_| ControlError::Internal("screenshot failed".into()))?; + let _ = tokio::fs::remove_file(&png_path).await; + if png.is_empty() { + return Err(ControlError::Internal("screenshot failed".into())); + } + let (width, height) = png_dimensions(&png); + let cursor = self.cursor(display).await; + let windows = self.windows(display).await.unwrap_or_default(); + let active = windows + .iter() + .max_by_key(|window| window.z) + .map(|window| ActiveWindow { + id: window.id.to_string(), + title: Some(window.title.clone()).filter(|title| !title.is_empty()), + }); + let elements: Vec = windows + .into_iter() + .enumerate() + .map(|(index, window)| UiElement { + id: (index + 1) as u32, + title: window.title.chars().take(80).collect(), + x: window.x, + y: window.y, + w: window.w, + h: window.h, + selector: None, + kind: Some("window".into()), + role: None, + }) + .collect(); + Ok(observation_with_elements( + observation_from_png(png, width, height, cursor, active), + elements, + )) + } + + async fn cursor(&self, display: &str) -> Option { + let value = self + .client + .call(display, "get_cursor_position", &json!({}), &[]) + .await + .ok()?; + cursor_from_value(&value) + } + + async fn windows(&self, display: &str) -> Result, ControlError> { + Self::list_windows_now(&self.client, display).await + } + + pub(crate) async fn list_windows_now( + client: &CuaClient, + display: &str, + ) -> Result, ControlError> { + let value = client + .call( + display, + "list_windows", + &json!({ "on_screen_only": true }), + &[], + ) + .await?; + Ok(parse_listed_windows(&value)) + } + + async fn dispatch( + &self, + display: &str, + translated: TranslatedAction, + ) -> Result<(), ControlError> { + match translated { + TranslatedAction::Sleep { ms } => { + sleep(Duration::from_millis(ms)).await; + Ok(()) + } + TranslatedAction::LegacyArgv { argv, detached } => { + if detached { + spawn_detached(&argv).await.map_err(ControlError::internal) + } else { + crate::process::run_output(&argv) + .await + .map_err(ControlError::internal) + .and_then(|output| { + if output.status.success() { + Ok(()) + } else { + Err(ControlError::internal(String::from_utf8_lossy( + &output.stderr, + ))) + } + }) + } + } + TranslatedAction::FocusTitle { title } => self.focus_title(display, &title).await, + TranslatedAction::Cua { tool, payload } => { + let payload = self.with_window_target(display, tool, payload).await?; + self.client.call(display, tool, &payload, &[]).await?; + Ok(()) + } + } + } + + async fn with_window_target( + &self, + display: &str, + tool: &str, + mut payload: Value, + ) -> Result { + if tool != "mouse_button_down" && tool != "mouse_button_up" { + return Ok(payload); + } + if payload.get("pid").is_some() && payload.get("window_id").is_some() { + return Ok(payload); + } + let window = self + .windows(display) + .await? + .into_iter() + .max_by_key(|window| window.z) + .ok_or(ControlError::TargetNotFound)?; + if let Some(object) = payload.as_object_mut() { + object.insert("pid".into(), json!(window.pid)); + object.insert("window_id".into(), json!(window.id)); + } + Ok(payload) + } + + async fn focus_title(&self, display: &str, title: &str) -> Result<(), ControlError> { + let needle = title.to_ascii_lowercase(); + let windows = self.windows(display).await?; + let window = windows + .into_iter() + .find(|window| window.title.to_ascii_lowercase().contains(&needle)) + .ok_or(ControlError::TargetNotFound)?; + self.client + .call( + display, + "bring_to_front", + &json!({ "pid": window.pid, "window_id": window.id }), + &[], + ) + .await?; + Ok(()) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct ListedWindow { + pub(crate) id: u64, + pub(crate) pid: u64, + pub(crate) title: String, + pub(crate) app_name: String, + pub(crate) x: u32, + pub(crate) y: u32, + pub(crate) w: u32, + pub(crate) h: u32, + pub(crate) z: i64, +} + +fn map_browser_unavailable(result: Result) -> Result { + match result { + Err(ControlError::BrowserUnavailable) => Ok(CdpPage { + ok: false, + error: Some("cdp unavailable".into()), + ..CdpPage::default() + }), + other => other, + } +} + +fn cursor_from_value(value: &Value) -> Option { + let mut nodes = Vec::new(); + client::walk(value, &mut nodes); + for node in nodes { + if let (Some(x), Some(y)) = ( + node.get("x").and_then(Value::as_i64), + node.get("y").and_then(Value::as_i64), + ) { + return Some(CursorPosition { + x: x as i32, + y: y as i32, + }); + } + } + None +} + +fn parse_listed_windows(value: &Value) -> Vec { + first_array_of_objects(value, "window_id") + .into_iter() + .filter_map(listed_window) + .filter(|window| !ignored_window(window)) + .collect() +} + +fn listed_window(value: &Value) -> Option { + let bounds = value.get("bounds"); + let x = number(value, "x").or_else(|| bounds.and_then(|bounds| number(bounds, "x")))?; + let y = number(value, "y").or_else(|| bounds.and_then(|bounds| number(bounds, "y")))?; + let w = number(value, "width") + .or_else(|| number(value, "w")) + .or_else(|| bounds.and_then(|bounds| number(bounds, "width")))?; + let h = number(value, "height") + .or_else(|| number(value, "h")) + .or_else(|| bounds.and_then(|bounds| number(bounds, "height")))?; + if w < 32 || h < 16 { + return None; + } + Some(ListedWindow { + id: number(value, "window_id")?, + pid: number(value, "pid").unwrap_or(0), + title: value + .get("title") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(), + app_name: value + .get("app_name") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(), + x: x as u32, + y: y as u32, + w: w as u32, + h: h as u32, + z: number(value, "z_index").unwrap_or(0) as i64, + }) +} + +fn ignored_window(window: &ListedWindow) -> bool { + let title = window.title.to_ascii_lowercase(); + title.is_empty() || title == "desktop" || title == "xfce4-panel" || title.contains("xfdesktop") +} + +fn number(value: &Value, key: &str) -> Option { + value.get(key).and_then(Value::as_u64).or_else(|| { + value + .get(key) + .and_then(Value::as_f64) + .filter(|n| *n >= 0.0) + .map(|n| n as u64) + }) +} + +fn png_dimensions(bytes: &[u8]) -> (u32, u32) { + image::load_from_memory(bytes) + .map(|image| (image.width(), image.height())) + .unwrap_or((1280, 800)) +} + +fn observe_png_path(display: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or(0); + PathBuf::from(format!( + "/tmp/lazyboy/cua-obs-{}-{nanos}.png", + display.trim_start_matches(':') + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn window_list_skips_panel_and_numbers_from_one() { + let raw = json!([ + { + "window_id": 1, + "pid": 8, + "title": "xfce4-panel", + "x": 0, + "y": 759, + "width": 1280, + "height": 41, + "z_index": 3 + }, + { + "window_id": 9, + "pid": 20, + "title": "終端機", + "app_name": "xfce4-terminal", + "bounds": { "x": 53, "y": 55, "width": 753, "height": 699 }, + "z_index": 1 + } + ]); + let windows = parse_listed_windows(&raw); + assert_eq!(windows.len(), 1); + assert_eq!(windows[0].title, "終端機"); + assert_eq!(windows[0].app_name, "xfce4-terminal"); + assert_eq!(windows[0].w, 753); + } +} diff --git a/crates/control/src/cua/record.rs b/crates/control/src/cua/record.rs new file mode 100644 index 0000000..f42159a --- /dev/null +++ b/crates/control/src/cua/record.rs @@ -0,0 +1,283 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::UNIX_EPOCH; + +use chrono::DateTime; +use serde_json::{Value, json}; + +pub fn events_from_dir(dir: &Path) -> Vec { + let mut turns = match fs::read_dir(dir) { + Ok(entries) => entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.join("action.json").is_file()) + .collect::>(), + Err(_) => return Vec::new(), + }; + turns.sort(); + turns + .into_iter() + .filter_map(|path| event_from_turn(&path)) + .collect() +} + +fn event_from_turn(dir: &Path) -> Option { + let raw = fs::read_to_string(dir.join("action.json")).ok()?; + let value: Value = serde_json::from_str(&raw).ok()?; + let tool = tool_name(&value); + if tool.is_empty() { + return None; + } + let args = arguments(&value); + let at = timestamp_ms(&value).unwrap_or_else(|| file_time_ms(&dir.join("action.json"))); + let el = element_from_args(args); + let mut event = match tool.as_str() { + "click" | "right_click" | "double_click" | "browser_click" => { + json!({ "t": "click", "el": el, "at": at }) + } + "type_text" | "set_value" | "browser_type" => { + let name = element_label(&el); + let role = el.get("role").and_then(Value::as_str).unwrap_or(""); + let raw_text = args + .get("text") + .or_else(|| args.get("value")) + .and_then(Value::as_str) + .unwrap_or(""); + let value = if looks_secret(&name) || looks_secret(role) { + "[已遮罩]" + } else { + raw_text + }; + json!({ "t": "input", "el": el, "value": value, "at": at }) + } + "press_key" | "hotkey" => { + let key = args + .get("key") + .and_then(Value::as_str) + .map(str::to_string) + .or_else(|| { + args.get("keys").and_then(Value::as_array).map(|keys| { + keys.iter() + .filter_map(Value::as_str) + .collect::>() + .join("+") + }) + }) + .unwrap_or_default(); + json!({ "t": "key", "key": key, "el": el, "at": at }) + } + "scroll" => json!({ + "t": "scroll", + "y": args.get("amount").and_then(Value::as_i64).unwrap_or(0), + "at": at + }), + "browser_navigate" => json!({ + "t": "navigate", + "url": args.get("url").and_then(Value::as_str).unwrap_or(""), + "at": at + }), + _ => return None, + }; + if let Some(title) = window_title(dir) { + event["window"] = json!(title); + } + Some(event) +} + +fn tool_name(value: &Value) -> String { + value + .get("tool") + .or_else(|| value.get("name")) + .or_else(|| value.get("action")) + .and_then(Value::as_str) + .unwrap_or("") + .to_ascii_lowercase() +} + +fn arguments(value: &Value) -> &Value { + value + .get("arguments") + .or_else(|| value.get("input")) + .or_else(|| value.get("args")) + .unwrap_or(value) +} + +fn element_from_args(args: &Value) -> Value { + let name = pick( + args, + &[ + "name", + "label", + "ref", + "selector", + "element_token", + "element_index", + ], + ); + let role = pick(args, &["role"]); + json!({ + "name": name, + "label": name, + "role": role, + "text": name, + }) +} + +fn element_label(el: &Value) -> String { + el.get("name") + .or_else(|| el.get("label")) + .or_else(|| el.get("text")) + .and_then(Value::as_str) + .unwrap_or("") + .to_string() +} + +fn pick(value: &Value, keys: &[&str]) -> String { + for key in keys { + if let Some(text) = value.get(*key).and_then(Value::as_str).map(str::trim) + && !text.is_empty() + { + return text.to_string(); + } + if let Some(number) = value.get(*key).and_then(Value::as_i64) { + return number.to_string(); + } + } + String::new() +} + +fn timestamp_ms(value: &Value) -> Option { + if let Some(ms) = value.get("at").and_then(Value::as_i64) { + return Some(ms); + } + let stamp = value + .get("timestamp") + .or_else(|| value.get("ts")) + .or_else(|| value.get("time")) + .and_then(Value::as_str)?; + DateTime::parse_from_rfc3339(stamp) + .ok() + .map(|time| time.timestamp_millis()) +} + +fn file_time_ms(path: &Path) -> i64 { + fs::metadata(path) + .and_then(|meta| meta.modified()) + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .map(|duration| duration.as_millis() as i64) + .unwrap_or(0) +} + +fn window_title(dir: &Path) -> Option { + for name in ["after_state.json", "app_state.json", "before_state.json"] { + let Ok(raw) = fs::read_to_string(dir.join(name)) else { + continue; + }; + let Ok(value) = serde_json::from_str::(&raw) else { + continue; + }; + if let Some(title) = value + .get("title") + .or_else(|| value.get("window_title")) + .or_else(|| value.pointer("/window/title")) + .and_then(Value::as_str) + .map(str::trim) + .filter(|title| !title.is_empty()) + { + return Some(title.to_string()); + } + } + None +} + +pub fn looks_secret(text: &str) -> bool { + let lower = text.to_lowercase(); + [ + "pass", + "pwd", + "密碼", + "token", + "otp", + "驗證碼", + "secret", + "cvv", + ] + .iter() + .any(|needle| lower.contains(needle)) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::SystemTime; + + fn write_turn(root: &Path, name: &str, action: &Value, after: Option<&Value>) { + let dir = root.join(name); + fs::create_dir_all(&dir).unwrap(); + fs::write(dir.join("action.json"), action.to_string()).unwrap(); + if let Some(state) = after { + fs::write(dir.join("after_state.json"), state.to_string()).unwrap(); + } + } + + #[test] + fn trajectory_turns_become_skill_events() { + let root = std::env::temp_dir().join(format!( + "lazyboy-traj-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&root).unwrap(); + write_turn( + &root, + "turn-00001", + &json!({ + "tool": "browser_click", + "timestamp": "2026-09-07T00:00:01Z", + "arguments": { "ref": "p1:1", "role": "button", "name": "Smoke Click" } + }), + Some(&json!({ "title": "LazyBoy Cua Smoke" })), + ); + write_turn( + &root, + "turn-00002", + &json!({ + "tool": "type_text", + "timestamp": "2026-09-07T00:00:02Z", + "arguments": { "text": "hunter2", "name": "Password", "role": "textbox" } + }), + None, + ); + write_turn( + &root, + "turn-00003", + &json!({ + "tool": "browser_navigate", + "timestamp": "2026-09-07T00:00:03Z", + "arguments": { "url": "https://example.com" } + }), + None, + ); + let events = events_from_dir(&root); + let _ = fs::remove_dir_all(&root); + assert_eq!(events.len(), 3); + assert_eq!(events[0]["t"], "click"); + assert_eq!(events[0]["el"]["name"], "Smoke Click"); + assert_eq!(events[0]["window"], "LazyBoy Cua Smoke"); + assert_eq!(events[1]["t"], "input"); + assert_eq!(events[1]["value"], "[已遮罩]"); + assert_eq!(events[2]["t"], "navigate"); + assert_eq!(events[2]["url"], "https://example.com"); + } + + #[test] + fn secret_labels_are_detected() { + assert!(looks_secret("Password")); + assert!(looks_secret("確認密碼")); + assert!(!looks_secret("Search")); + } +} diff --git a/crates/control/src/cua/translate.rs b/crates/control/src/cua/translate.rs new file mode 100644 index 0000000..936676d --- /dev/null +++ b/crates/control/src/cua/translate.rs @@ -0,0 +1,219 @@ +use lazyboy_contracts::{ComputerAction, PointerButton, PointerType, ScrollDirection}; +use serde_json::{Value, json}; + +use crate::controller::ControlError; +use crate::{launch_argv_on, open_argv_on}; + +#[derive(Debug, Clone, PartialEq)] +pub enum TranslatedAction { + Cua { tool: &'static str, payload: Value }, + Sleep { ms: u64 }, + LegacyArgv { argv: Vec, detached: bool }, + FocusTitle { title: String }, +} + +pub fn translate_action( + action: &ComputerAction, + display: &str, + profile: Option<&str>, +) -> Result { + match action { + ComputerAction::Wait { ms } => Ok(TranslatedAction::Sleep { ms: u64::from(*ms) }), + ComputerAction::Open { path } => Ok(TranslatedAction::LegacyArgv { + argv: open_argv_on(display, profile, path), + detached: true, + }), + ComputerAction::Launch { application, uri } => { + let argv = launch_argv_on(display, profile, application, uri.as_deref()) + .ok_or(ControlError::Unsupported)?; + Ok(TranslatedAction::LegacyArgv { + argv, + detached: true, + }) + } + ComputerAction::Ref { .. } => Err(ControlError::Unsupported), + ComputerAction::Focus { title } => Ok(TranslatedAction::FocusTitle { + title: title.clone(), + }), + ComputerAction::Pointer { + x, + y, + pointer_type, + button, + } => Ok(translate_pointer(*x, *y, pointer_type, *button)), + ComputerAction::Clipboard { text } => { + tracing::info!(backend = "cua", tool = "type_text", length = text.len()); + Ok(TranslatedAction::Cua { + tool: "type_text", + payload: json!({ + "text": text, + "scope": "desktop", + "target": { "kind": "desktop", "display_id": "primary" }, + }), + }) + } + ComputerAction::Key { key, modifiers } => Ok(translate_key(key, modifiers.as_deref())), + ComputerAction::Scroll { direction, amount } => Ok(TranslatedAction::Cua { + tool: "scroll", + payload: json!({ + "direction": match direction { + ScrollDirection::Up => "up", + ScrollDirection::Down => "down", + }, + "amount": amount.unwrap_or(12), + "by": "line", + "scope": "desktop", + "target": { "kind": "desktop", "display_id": "primary" }, + }), + }), + } +} + +fn translate_pointer( + x: u32, + y: u32, + pointer_type: &PointerType, + button: Option, +) -> TranslatedAction { + let button = match button.unwrap_or(PointerButton::Left) { + PointerButton::Left => "left", + PointerButton::Middle => "middle", + PointerButton::Right => "right", + }; + match pointer_type { + PointerType::Move => TranslatedAction::Cua { + tool: "move_cursor", + payload: json!({ + "x": x, + "y": y, + "scope": "desktop", + "target": { "kind": "desktop", "display_id": "primary" }, + }), + }, + PointerType::Click => TranslatedAction::Cua { + tool: "click", + payload: json!({ + "x": x, + "y": y, + "button": button, + "scope": "desktop", + "target": { "kind": "desktop", "display_id": "primary" }, + }), + }, + PointerType::Down => TranslatedAction::Cua { + tool: "mouse_button_down", + payload: json!({ "x": x, "y": y, "button": button }), + }, + PointerType::Up => TranslatedAction::Cua { + tool: "mouse_button_up", + payload: json!({ "x": x, "y": y, "button": button }), + }, + } +} + +fn translate_key(key: &str, modifiers: Option<&[String]>) -> TranslatedAction { + let key = map_key(key); + match modifiers { + Some(items) if !items.is_empty() => { + let mut keys: Vec = items.iter().map(|item| map_key(item)).collect(); + keys.push(key); + TranslatedAction::Cua { + tool: "hotkey", + payload: json!({ + "keys": keys, + "scope": "desktop", + "target": { "kind": "desktop", "display_id": "primary" }, + }), + } + } + _ => TranslatedAction::Cua { + tool: "press_key", + payload: json!({ + "key": key, + "scope": "desktop", + "target": { "kind": "desktop", "display_id": "primary" }, + }), + }, + } +} + +fn map_key(key: &str) -> String { + match key.to_ascii_lowercase().as_str() { + "enter" | "return" => "return".into(), + "esc" | "escape" => "escape".into(), + "cmd" | "command" | "super" | "meta" | "win" => "ctrl".into(), + "control" | "ctl" => "ctrl".into(), + "option" => "alt".into(), + other => other.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use lazyboy_contracts::RefVerb; + + #[test] + fn pixel_click_is_desktop_cua_click() { + let action = ComputerAction::Pointer { + x: 40, + y: 80, + pointer_type: PointerType::Click, + button: Some(PointerButton::Left), + }; + let TranslatedAction::Cua { tool, payload } = + translate_action(&action, ":1", None).unwrap() + else { + panic!("expected cua click"); + }; + assert_eq!(tool, "click"); + assert_eq!(payload["x"], 40); + assert_eq!(payload["y"], 80); + assert_eq!(payload["scope"], "desktop"); + assert_eq!(payload["target"]["display_id"], "primary"); + } + + #[test] + fn typed_text_is_redacted_from_tool_name_only() { + let action = ComputerAction::Clipboard { + text: "secret-password".into(), + }; + let TranslatedAction::Cua { tool, payload } = + translate_action(&action, ":1", None).unwrap() + else { + panic!("expected type_text"); + }; + assert_eq!(tool, "type_text"); + assert_eq!(payload["text"], "secret-password"); + assert_eq!(payload["scope"], "desktop"); + } + + #[test] + fn chord_uses_hotkey_and_maps_cmd_to_ctrl() { + let action = ComputerAction::Key { + key: "c".into(), + modifiers: Some(vec!["cmd".into()]), + }; + let TranslatedAction::Cua { tool, payload } = + translate_action(&action, ":1", None).unwrap() + else { + panic!("expected hotkey"); + }; + assert_eq!(tool, "hotkey"); + assert_eq!(payload["keys"], json!(["ctrl", "c"])); + } + + #[test] + fn semantic_ref_is_not_translated_here() { + let action = ComputerAction::Ref { + verb: RefVerb::Click, + target: "#go".into(), + ref_kind: "dom".into(), + text: None, + }; + assert!(matches!( + translate_action(&action, ":1", None), + Err(ControlError::Unsupported) + )); + } +} diff --git a/crates/control/src/legacy.rs b/crates/control/src/legacy.rs new file mode 100644 index 0000000..b74d20e --- /dev/null +++ b/crates/control/src/legacy.rs @@ -0,0 +1,300 @@ +use async_trait::async_trait; +use lazyboy_contracts::{ComputerAction, ComputerObservation, RefVerb}; +use tokio::time::{Duration, sleep}; + +use crate::controller::{ + ComputerController, ComputerDriver, ControlContext, ControlError, ControllerHealth, +}; +use crate::process::{ + capture_stdout, run_output, run_stdout_text, run_stdout_text_timeout, spawn_detached, +}; +use crate::{ + ActionRequest, ActionResult, BrowserRequest, CdpPage, RecordingRequest, RecordingResult, + RecordingSession, a11y_command_on, action_pause_ms, cdp_command_on, cdp_record_command_on, + cdp_record_stop_command, devtools_port, launch_argv_on, observation_from_png, + observation_with_elements, open_argv_on, parse_a11y_page, parse_cdp_page, parse_pointer_state, + parse_ui_elements, pointer_state_command_on, screenshot_command_on, teach_recorder_output, + teach_trajectory_dir, window_list_command_on, xdotool_argv_on, +}; + +#[derive(Debug, Default, Clone, Copy)] +pub struct LegacyController; + +#[async_trait] +impl ComputerController for LegacyController { + fn backend(&self) -> ComputerDriver { + ComputerDriver::Legacy + } + + async fn health(&self, ctx: &ControlContext) -> Result { + Ok(ControllerHealth { + backend: ComputerDriver::Legacy.as_str().to_string(), + version: None, + healthy: true, + degraded: false, + details: vec![format!("display {}", ctx.display)], + }) + } + + async fn observe(&self, ctx: &ControlContext) -> Result { + observe_display(&ctx.display).await + } + + async fn act( + &self, + request: &ActionRequest, + ctx: &ControlContext, + ) -> Result { + let display = ctx.display.as_str(); + let profile = ctx.profile_path.as_deref(); + let mut completed = 0usize; + for action in &request.actions { + apply_action(display, profile, action).await?; + let pause = action_pause_ms(action); + if pause > 0 { + sleep(Duration::from_millis(pause)).await; + } + completed += 1; + } + if request.settle_ms > 0 { + sleep(Duration::from_millis(u64::from(request.settle_ms))).await; + } + let observation = if request.observe { + Some(observe_display(display).await?) + } else { + None + }; + Ok(ActionResult { + completed, + observation, + }) + } + + async fn browser( + &self, + request: &BrowserRequest, + ctx: &ControlContext, + ) -> Result { + legacy_browser(request, ctx).await + } + + async fn start_recording( + &self, + request: &RecordingRequest, + ctx: &ControlContext, + ) -> Result { + start_cdp_recorder(&ctx.display, ctx.profile_path.as_deref(), &request.skill_id).await?; + Ok(RecordingSession { + skill_id: request.skill_id.clone(), + output_dir: teach_trajectory_dir(&request.skill_id), + }) + } + + async fn stop_recording( + &self, + request: &RecordingRequest, + _ctx: &ControlContext, + ) -> Result<(), ControlError> { + stop_cdp_recorder(&request.skill_id).await + } + + async fn collect_recording( + &self, + request: &RecordingRequest, + _ctx: &ControlContext, + ) -> Result { + Ok(RecordingResult { + events: collect_cdp_events(&request.skill_id).await, + }) + } +} + +async fn legacy_browser( + request: &BrowserRequest, + ctx: &ControlContext, +) -> Result { + let mut body = serde_json::json!({ + "action": request.action, + "ensure": request.ensure, + "display": ctx.display, + "port": devtools_port(&ctx.display), + }); + if let Some(profile) = &ctx.profile_path { + body["profile"] = serde_json::json!(profile); + } + if let Some(url) = &request.url { + body["url"] = serde_json::json!(url); + } + if let Some(text) = &request.text { + body["text"] = serde_json::json!(text); + } + if let Some(key) = &request.key { + body["key"] = serde_json::json!(key); + } + if let Some(ms) = request.ms { + body["ms"] = serde_json::json!(ms); + } + if let Some(selector) = &request.selector { + body["selector"] = serde_json::json!(selector); + } + if let Some(wait_ms) = request.wait_ms { + body["waitMs"] = serde_json::json!(wait_ms); + } + let timeout_ms = if request.action == "click" { + request.wait_ms.unwrap_or(45_000).min(120_000) + 25_000 + } else { + 20_000 + }; + let argv = cdp_command_on(&ctx.display, ctx.profile_path.as_deref(), &body); + let raw = run_stdout_text_timeout(&argv, timeout_ms) + .await + .map_err(ControlError::internal)?; + Ok(parse_cdp_page(&raw)) +} + +pub async fn observe_display(display: &str) -> Result { + let png = capture_stdout(&screenshot_command_on(display)) + .await + .map_err(|_| ControlError::Internal("screenshot failed".into()))?; + let ((cursor, window), elements) = + tokio::join!(run_pointer_state(display), run_window_list(display)); + Ok(observation_with_elements( + observation_from_png(png, 1280, 800, cursor, window), + elements, + )) +} + +async fn run_window_list(display: &str) -> Vec { + let Ok(raw) = run_stdout_text(&window_list_command_on(display)).await else { + return Vec::new(); + }; + parse_ui_elements(&raw) +} + +async fn run_pointer_state( + display: &str, +) -> ( + Option, + Option, +) { + let Ok(raw) = run_stdout_text(&pointer_state_command_on(display)).await else { + return (None, None); + }; + parse_pointer_state(&raw) +} + +pub async fn apply_action( + display: &str, + profile: Option<&str>, + action: &ComputerAction, +) -> Result<(), ControlError> { + match action { + ComputerAction::Wait { ms } => { + sleep(Duration::from_millis(u64::from(*ms))).await; + Ok(()) + } + ComputerAction::Open { path } => spawn_detached(&open_argv_on(display, profile, path)) + .await + .map_err(ControlError::internal), + ComputerAction::Focus { .. } => run_xdotool(display, action).await, + ComputerAction::Launch { application, uri } => { + let argv = launch_argv_on(display, profile, application, uri.as_deref()) + .ok_or(ControlError::Unsupported)?; + spawn_detached(&argv).await.map_err(ControlError::internal) + } + ComputerAction::Ref { + verb, + target, + ref_kind, + text, + } => apply_ref(display, profile, *verb, target, ref_kind, text.as_deref()).await, + other => run_xdotool(display, other).await, + } +} + +async fn run_xdotool(display: &str, action: &ComputerAction) -> Result<(), ControlError> { + let argv = xdotool_argv_on(display, action).ok_or(ControlError::Unsupported)?; + let output = run_output(&argv).await.map_err(ControlError::internal)?; + if output.status.success() { + Ok(()) + } else { + Err(ControlError::internal(String::from_utf8_lossy( + &output.stderr, + ))) + } +} + +async fn apply_ref( + display: &str, + profile: Option<&str>, + verb: RefVerb, + target: &str, + kind: &str, + text: Option<&str>, +) -> Result<(), ControlError> { + let action = match verb { + RefVerb::Click => "click", + RefVerb::SetValue => "type", + RefVerb::Focus => "focus", + }; + let mut request = serde_json::json!({ + "action": action, + "selector": target, + "display": display, + "ensure": false, + }); + if let Some(text) = text { + request["text"] = serde_json::json!(text); + } + let argv = if kind == "dom" { + cdp_command_on(display, profile, &request) + } else { + a11y_command_on(display, &request) + }; + let raw = run_stdout_text(&argv) + .await + .map_err(ControlError::internal)?; + let ok = if kind == "dom" { + parse_cdp_page(&raw).ok + } else { + parse_a11y_page(&raw).ok + }; + if ok { + Ok(()) + } else { + Err(ControlError::internal(raw)) + } +} + +pub(crate) async fn start_cdp_recorder( + display: &str, + profile: Option<&str>, + skill_id: &str, +) -> Result<(), ControlError> { + if skill_id.trim().is_empty() { + return Err(ControlError::InvalidAction( + "recording needs a skill id".into(), + )); + } + let argv = cdp_record_command_on(display, profile, skill_id); + run_output(&argv).await.map_err(ControlError::internal)?; + Ok(()) +} + +pub(crate) async fn stop_cdp_recorder(skill_id: &str) -> Result<(), ControlError> { + let argv = cdp_record_stop_command(skill_id); + let _ = run_output(&argv).await; + Ok(()) +} + +pub(crate) async fn collect_cdp_events(skill_id: &str) -> Vec { + let path = teach_recorder_output(skill_id); + let Ok(text) = tokio::fs::read_to_string(&path).await else { + return Vec::new(); + }; + let _ = tokio::fs::remove_file(&path).await; + text.lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|event| event.get("t").and_then(serde_json::Value::as_str) != Some("recorder")) + .collect() +} diff --git a/crates/control/src/lib.rs b/crates/control/src/lib.rs index 5108723..0790422 100644 --- a/crates/control/src/lib.rs +++ b/crates/control/src/lib.rs @@ -1,10 +1,14 @@ mod a11y; mod actions; mod cdp; +mod controller; +mod cua; mod lease; +mod legacy; mod observe; mod overlay; mod path; +mod process; mod sandbox; mod screen; mod takeover; @@ -13,7 +17,10 @@ mod x11; pub use a11y::*; pub use actions::*; pub use cdp::*; +pub use controller::*; +pub use cua::{CuaClient, CuaController, TranslatedAction, translate_action}; pub use lease::*; +pub use legacy::LegacyController; pub use observe::*; pub use overlay::*; pub use path::*; diff --git a/crates/control/src/observe.rs b/crates/control/src/observe.rs index d3bb097..c958a47 100644 --- a/crates/control/src/observe.rs +++ b/crates/control/src/observe.rs @@ -1,5 +1,7 @@ +use base64::Engine; use chrono::Utc; use lazyboy_contracts::{ActiveWindow, ComputerObservation, CursorPosition, UiElement}; +use serde_json::{Value, json}; use sha2::{Digest, Sha256}; pub fn observation_from_png( @@ -22,6 +24,24 @@ pub fn observation_from_png( } } +pub fn observation_to_control_json(observation: &ComputerObservation) -> Value { + let mut body = json!({ + "png_base64": base64::engine::general_purpose::STANDARD.encode(&observation.image), + }); + if let Some(cursor) = &observation.cursor { + body["cursor"] = json!({ "x": cursor.x, "y": cursor.y }); + } + if let Some(window) = &observation.active_window { + body["activeWindow"] = json!({ "id": window.id, "title": window.title }); + } + if !observation.elements.is_empty() + && let Ok(value) = serde_json::to_value(&observation.elements) + { + body["elements"] = value; + } + body +} + pub fn observation_with_elements( mut observation: ComputerObservation, elements: Vec, @@ -62,7 +82,11 @@ const SIGNATURE_H: u32 = 18; pub fn frame_signature(image: &[u8]) -> Option> { let dynamic = image::load_from_memory(image).ok()?; let thumb = dynamic - .resize_exact(SIGNATURE_W, SIGNATURE_H, image::imageops::FilterType::Triangle) + .resize_exact( + SIGNATURE_W, + SIGNATURE_H, + image::imageops::FilterType::Triangle, + ) .to_luma8(); Some(thumb.into_raw()) } @@ -108,12 +132,20 @@ mod tests { let base = frame_signature(&png(|_, _| Rgb([240, 240, 240]))).unwrap(); // A panel clock flipping digits touches a couple of pixels only. let clock = frame_signature(&png(|x, y| { - if x < 6 && y < 6 { Rgb([0, 0, 0]) } else { Rgb([240, 240, 240]) } + if x < 6 && y < 6 { + Rgb([0, 0, 0]) + } else { + Rgb([240, 240, 240]) + } })) .unwrap(); // A dialog covering a quarter of the screen. let dialog = frame_signature(&png(|x, y| { - if x < 160 && y < 90 { Rgb([20, 20, 20]) } else { Rgb([240, 240, 240]) } + if x < 160 && y < 90 { + Rgb([20, 20, 20]) + } else { + Rgb([240, 240, 240]) + } })) .unwrap(); assert!(signatures_similar(&base, &clock)); diff --git a/crates/control/src/process.rs b/crates/control/src/process.rs new file mode 100644 index 0000000..0bb6a3c --- /dev/null +++ b/crates/control/src/process.rs @@ -0,0 +1,71 @@ +use std::process::Stdio; +use std::time::Duration; + +use tokio::io::AsyncReadExt; +use tokio::process::Command; + +pub async fn run_output(argv: &[String]) -> Result { + if argv.is_empty() { + return Err("empty command".into()); + } + Command::new(&argv[0]) + .args(&argv[1..]) + .output() + .await + .map_err(|error| error.to_string()) +} + +pub async fn run_stdout_text(argv: &[String]) -> Result { + let output = run_output(argv).await?; + Ok(if output.stdout.is_empty() { + String::from_utf8_lossy(&output.stderr).into_owned() + } else { + String::from_utf8_lossy(&output.stdout).into_owned() + }) +} + +pub async fn run_stdout_text_timeout(argv: &[String], timeout_ms: u64) -> Result { + tokio::time::timeout( + Duration::from_millis(timeout_ms.max(100)), + run_stdout_text(argv), + ) + .await + .map_err(|_| "timed out".to_string())? +} + +pub async fn capture_stdout(argv: &[String]) -> Result, String> { + if argv.is_empty() { + return Err("empty command".into()); + } + let mut child = Command::new(&argv[0]) + .args(&argv[1..]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| error.to_string())?; + let mut stdout = Vec::new(); + if let Some(mut pipe) = child.stdout.take() { + pipe.read_to_end(&mut stdout) + .await + .map_err(|error| error.to_string())?; + } + let status = child.wait().await.map_err(|error| error.to_string())?; + if !status.success() || stdout.is_empty() { + return Err("command failed".into()); + } + Ok(stdout) +} + +pub async fn spawn_detached(argv: &[String]) -> Result<(), String> { + if argv.is_empty() { + return Err("empty command".into()); + } + Command::new(&argv[0]) + .args(&argv[1..]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| error.to_string())?; + Ok(()) +} diff --git a/crates/control/src/sandbox.rs b/crates/control/src/sandbox.rs index 64aee79..be91756 100644 --- a/crates/control/src/sandbox.rs +++ b/crates/control/src/sandbox.rs @@ -77,6 +77,53 @@ impl ActionRequest { } } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct BrowserRequest { + pub action: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub key: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub wait_ms: Option, + #[serde(default)] + pub ensure: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub display: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub profile_path: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct RecordingRequest { + pub skill_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub display: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub profile_path: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct RecordingSession { + pub skill_id: String, + pub output_dir: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct RecordingResult { + pub events: Vec, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct EnsureScreenRequest { pub slot: u32, @@ -208,6 +255,46 @@ pub trait SandboxProvider: Send + Sync { context: &AdapterContext, ) -> Result; + async fn browser( + &self, + computer: &ComputerRef, + request: BrowserRequest, + context: &AdapterContext, + ) -> Result { + let _ = (computer, request, context); + Err(SandboxError::message("browser is unavailable")) + } + + async fn start_recording( + &self, + computer: &ComputerRef, + request: RecordingRequest, + context: &AdapterContext, + ) -> Result { + let _ = (computer, request, context); + Err(SandboxError::message("recording is unavailable")) + } + + async fn stop_recording( + &self, + computer: &ComputerRef, + request: RecordingRequest, + context: &AdapterContext, + ) -> Result<(), SandboxError> { + let _ = (computer, request, context); + Err(SandboxError::message("recording is unavailable")) + } + + async fn collect_recording( + &self, + computer: &ComputerRef, + request: RecordingRequest, + context: &AdapterContext, + ) -> Result { + let _ = (computer, request, context); + Err(SandboxError::message("recording is unavailable")) + } + async fn connect_screen( &self, computer: &ComputerRef, @@ -249,3 +336,35 @@ pub trait SandboxProvider: Send + Sync { context: &AdapterContext, ) -> Result<(), SandboxError>; } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn browser_request_reads_camel_case_wait_ms() { + let request: BrowserRequest = serde_json::from_value(json!({ + "action": "click", + "waitMs": 12000, + "ensure": true, + "selector": "p1:1" + })) + .unwrap(); + assert_eq!(request.action, "click"); + assert_eq!(request.wait_ms, Some(12_000)); + assert!(request.ensure); + assert_eq!(request.selector.as_deref(), Some("p1:1")); + } + + #[test] + fn recording_request_reads_camel_case_skill_id() { + let request: RecordingRequest = serde_json::from_value(json!({ + "skillId": "abc-1", + "display": ":2" + })) + .unwrap(); + assert_eq!(request.skill_id, "abc-1"); + assert_eq!(request.display.as_deref(), Some(":2")); + } +} diff --git a/crates/controld/Cargo.toml b/crates/controld/Cargo.toml index 06ddbe8..0ed926b 100644 --- a/crates/controld/Cargo.toml +++ b/crates/controld/Cargo.toml @@ -7,14 +7,12 @@ license.workspace = true publish.workspace = true [dependencies] -lazyboy-contracts.workspace = true lazyboy-control.workspace = true axum.workspace = true tokio.workspace = true serde_json.workspace = true tracing.workspace = true tracing-subscriber.workspace = true -base64.workspace = true [lints] workspace = true diff --git a/crates/controld/src/main.rs b/crates/controld/src/main.rs index 5339be1..537b65f 100644 --- a/crates/controld/src/main.rs +++ b/crates/controld/src/main.rs @@ -1,24 +1,19 @@ -use std::process::Stdio; -use std::time::Duration; +use std::sync::Arc; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::routing::{get, post}; use axum::{Json, Router}; -use lazyboy_contracts::{ComputerAction, RefVerb}; use lazyboy_control::{ - ActionRequest, PRIMARY_DISPLAY, a11y_command_on, action_pause_ms, cdp_command_on, - launch_argv_on, normalize_display, open_argv_on, parse_a11y_page, parse_cdp_page, - parse_pointer_state, parse_ui_elements, pointer_state_command_on, screenshot_command_on, - window_list_command_on, xdotool_argv_on, + ActionRequest, BrowserRequest, ComputerController, ComputerDriver, ControlContext, + ControlError, PRIMARY_DISPLAY, RecordingRequest, normalize_display, + observation_to_control_json, }; -use tokio::io::AsyncReadExt; -use tokio::process::Command; -use tokio::time::sleep; #[derive(Clone)] struct App { token: String, + controller: Arc, } #[tokio::main] @@ -27,11 +22,20 @@ async fn main() { .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .init(); let token = std::env::var("LAZYBOY_CONTROL_TOKEN").unwrap_or_default(); + let driver = ComputerDriver::from_env(); + tracing::info!(backend = driver.as_str(), "computer controller"); let app = Router::new() .route("/health", get(|| async { "ok" })) .route("/observe", post(observe)) .route("/act", post(act)) - .with_state(App { token }); + .route("/browser", post(browser)) + .route("/recording/start", post(recording_start)) + .route("/recording/stop", post(recording_stop)) + .route("/recording/collect", post(recording_collect)) + .with_state(App { + token, + controller: driver.controller(), + }); let listener = tokio::net::TcpListener::bind("127.0.0.1:7070") .await .expect("bind control port"); @@ -72,6 +76,15 @@ fn profile_of(headers: &HeaderMap, fallback: Option<&str>) -> Option { .map(str::to_string) } +fn status_for(error: &ControlError) -> StatusCode { + if error.is_client_error() { + StatusCode::BAD_REQUEST + } else { + tracing::error!(error = %error, "control failed"); + StatusCode::INTERNAL_SERVER_ERROR + } +} + async fn observe( State(app): State, headers: HeaderMap, @@ -79,11 +92,11 @@ async fn observe( if !authorized(&headers, &app.token) { return Err(StatusCode::UNAUTHORIZED); } - let display = display_of(&headers, None); - let png = run_capture(&display) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - Ok(Json(observation_json(&display, png).await)) + let ctx = ControlContext::new(display_of(&headers, None), None); + match app.controller.observe(&ctx).await { + Ok(observation) => Ok(Json(observation_to_control_json(&observation))), + Err(error) => Err(status_for(&error)), + } } async fn act( @@ -94,205 +107,105 @@ async fn act( if !authorized(&headers, &app.token) { return Err(StatusCode::UNAUTHORIZED); } - let display = display_of(&headers, request.display.as_deref()); - let profile = profile_of(&headers, request.profile_path.as_deref()); - let mut completed = 0usize; - for action in &request.actions { - apply_action(&display, profile.as_deref(), action) - .await - .map_err(|_| StatusCode::BAD_REQUEST)?; - let pause = action_pause_ms(action); - if pause > 0 { - sleep(Duration::from_millis(pause)).await; - } - completed += 1; - } - if request.settle_ms > 0 { - sleep(Duration::from_millis(request.settle_ms as u64)).await; - } - let mut body = serde_json::json!({ "completed": completed }); - if request.observe { - let png = run_capture(&display) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - if let serde_json::Value::Object(map) = observation_json(&display, png).await { - body.as_object_mut().unwrap().extend(map); - } - } - Ok(Json(body)) -} - -async fn apply_action( - display: &str, - profile: Option<&str>, - action: &ComputerAction, -) -> Result<(), String> { - match action { - ComputerAction::Wait { ms } => { - sleep(Duration::from_millis(*ms as u64)).await; - Ok(()) - } - ComputerAction::Open { path } => { - spawn_detached(&open_argv_on(display, profile, path)).await - } - ComputerAction::Focus { .. } => { - let argv = - xdotool_argv_on(display, action).ok_or_else(|| "unsupported action".to_string())?; - let output = Command::new(&argv[0]) - .args(&argv[1..]) - .output() - .await - .map_err(|error| error.to_string())?; - if output.status.success() { - Ok(()) - } else { - Err(String::from_utf8_lossy(&output.stderr).into_owned()) - } - } - ComputerAction::Launch { application, uri } => { - let argv = launch_argv_on(display, profile, application, uri.as_deref()) - .ok_or_else(|| "unknown application".to_string())?; - spawn_detached(&argv).await - } - ComputerAction::Ref { - verb, - target, - ref_kind, - text, - } => apply_ref(display, profile, *verb, target, ref_kind, text.as_deref()).await, - other => { - let argv = - xdotool_argv_on(display, other).ok_or_else(|| "unsupported action".to_string())?; - let output = Command::new(&argv[0]) - .args(&argv[1..]) - .output() - .await - .map_err(|error| error.to_string())?; - if output.status.success() { - Ok(()) - } else { - Err(String::from_utf8_lossy(&output.stderr).into_owned()) + let ctx = ControlContext::new( + display_of(&headers, request.display.as_deref()), + profile_of(&headers, request.profile_path.as_deref()), + ); + match app.controller.act(&request, &ctx).await { + Ok(result) => { + let mut body = serde_json::json!({ "completed": result.completed }); + if let Some(observation) = result.observation + && let serde_json::Value::Object(map) = observation_to_control_json(&observation) + { + body.as_object_mut().expect("object").extend(map); } + Ok(Json(body)) } + Err(error) => Err(status_for(&error)), } } -async fn apply_ref( - display: &str, - profile: Option<&str>, - verb: RefVerb, - target: &str, - kind: &str, - text: Option<&str>, -) -> Result<(), String> { - let action = match verb { - RefVerb::Click => "click", - RefVerb::SetValue => "type", - RefVerb::Focus => "focus", - }; - let mut request = serde_json::json!({ - "action": action, - "selector": target, - "display": display, - "ensure": false, - }); - if let Some(text) = text { - request["text"] = serde_json::json!(text); +async fn browser( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> Result, StatusCode> { + if !authorized(&headers, &app.token) { + return Err(StatusCode::UNAUTHORIZED); } - let argv = if kind == "dom" { - cdp_command_on(display, profile, &request) - } else { - a11y_command_on(display, &request) - }; - let output = Command::new(&argv[0]) - .args(&argv[1..]) - .output() + let ctx = ControlContext::new( + display_of(&headers, request.display.as_deref()), + profile_of(&headers, request.profile_path.as_deref()), + ); + match app.controller.browser(&request, &ctx).await { + Ok(page) => Ok(Json( + serde_json::to_value(&page).unwrap_or_else(|_| serde_json::json!({"ok": false})), + )), + Err(error) => Err(status_for(&error)), + } +} + +fn recording_ctx(headers: &HeaderMap, request: &RecordingRequest) -> ControlContext { + ControlContext::new( + display_of(headers, request.display.as_deref()), + profile_of(headers, request.profile_path.as_deref()), + ) +} + +async fn recording_start( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> Result, StatusCode> { + if !authorized(&headers, &app.token) { + return Err(StatusCode::UNAUTHORIZED); + } + match app + .controller + .start_recording(&request, &recording_ctx(&headers, &request)) .await - .map_err(|error| error.to_string())?; - let raw = if output.stdout.is_empty() { - String::from_utf8_lossy(&output.stderr).into_owned() - } else { - String::from_utf8_lossy(&output.stdout).into_owned() - }; - let ok = if kind == "dom" { - parse_cdp_page(&raw).ok - } else { - parse_a11y_page(&raw).ok - }; - if ok { Ok(()) } else { Err(raw) } -} - -async fn spawn_detached(argv: &[String]) -> Result<(), String> { - Command::new(&argv[0]) - .args(&argv[1..]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .map_err(|error| error.to_string())?; - Ok(()) -} - -async fn observation_json(display: &str, png: Vec) -> serde_json::Value { - use base64::Engine; - let mut body = serde_json::json!({ - "png_base64": base64::engine::general_purpose::STANDARD.encode(png) - }); - let ((cursor, window), elements) = - tokio::join!(run_pointer_state(display), run_window_list(display)); - if let Some(cursor) = cursor { - body["cursor"] = serde_json::json!({ "x": cursor.x, "y": cursor.y }); + { + Ok(session) => Ok(Json( + serde_json::to_value(&session).unwrap_or_else(|_| serde_json::json!({"ok": false})), + )), + Err(error) => Err(status_for(&error)), } - if let Some(window) = window { - body["activeWindow"] = serde_json::json!({ "id": window.id, "title": window.title }); - } - - if !elements.is_empty() { - body["elements"] = serde_json::to_value(elements).unwrap_or(serde_json::json!([])); - } - body } -async fn run_window_list(display: &str) -> Vec { - let argv = window_list_command_on(display); - let output = Command::new(&argv[0]).args(&argv[1..]).output().await.ok(); - let Some(output) = output else { - return Vec::new(); - }; - parse_ui_elements(&String::from_utf8_lossy(&output.stdout)) +async fn recording_stop( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> Result, StatusCode> { + if !authorized(&headers, &app.token) { + return Err(StatusCode::UNAUTHORIZED); + } + match app + .controller + .stop_recording(&request, &recording_ctx(&headers, &request)) + .await + { + Ok(()) => Ok(Json(serde_json::json!({ "ok": true }))), + Err(error) => Err(status_for(&error)), + } } -async fn run_pointer_state( - display: &str, -) -> ( - Option, - Option, -) { - let argv = pointer_state_command_on(display); - let output = Command::new(&argv[0]).args(&argv[1..]).output().await.ok(); - let Some(output) = output else { - return (None, None); - }; - parse_pointer_state(&String::from_utf8_lossy(&output.stdout)) -} - -async fn run_capture(display: &str) -> Result, String> { - let argv = screenshot_command_on(display); - let mut child = Command::new(&argv[0]) - .args(&argv[1..]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|error| error.to_string())?; - let mut stdout = Vec::new(); - if let Some(mut pipe) = child.stdout.take() { - pipe.read_to_end(&mut stdout) - .await - .map_err(|error| error.to_string())?; +async fn recording_collect( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> Result, StatusCode> { + if !authorized(&headers, &app.token) { + return Err(StatusCode::UNAUTHORIZED); } - let status = child.wait().await.map_err(|error| error.to_string())?; - if !status.success() || stdout.is_empty() { - return Err("screenshot failed".into()); + match app + .controller + .collect_recording(&request, &recording_ctx(&headers, &request)) + .await + { + Ok(result) => Ok(Json( + serde_json::to_value(&result).unwrap_or_else(|_| serde_json::json!({"events": []})), + )), + Err(error) => Err(status_for(&error)), } - Ok(stdout) } diff --git a/crates/harness/src/execution.rs b/crates/harness/src/execution.rs index dd25edc..0dd8f6e 100644 --- a/crates/harness/src/execution.rs +++ b/crates/harness/src/execution.rs @@ -115,7 +115,10 @@ mod tests { #[test] fn a_prose_answer_is_not_a_stop() { - assert_eq!(stop_reason(ExecutionMode::Bounded(40), 3, "done", false, false), None); + assert_eq!( + stop_reason(ExecutionMode::Bounded(40), 3, "done", false, false), + None + ); } #[test] @@ -132,7 +135,10 @@ mod tests { stop_reason(mode, 3, "需要密码\n[NEEDS_INPUT]", true, false), Some(StopReason::MidTaskText) ); - assert_eq!(stop_reason(mode, 3, "我先跳到下一步", true, true), Some(StopReason::MidTaskText)); + assert_eq!( + stop_reason(mode, 3, "我先跳到下一步", true, true), + Some(StopReason::MidTaskText) + ); assert_eq!(stop_reason(mode, 3, "看起来好了", true, false), None); } @@ -143,12 +149,21 @@ mod tests { Some(StopReason::BudgetExhausted) ); assert_eq!( - stop_reason(ExecutionMode::Bounded(40), 40, "需要密码\n[NEEDS_INPUT]", true, false), + stop_reason( + ExecutionMode::Bounded(40), + 40, + "需要密码\n[NEEDS_INPUT]", + true, + false + ), Some(StopReason::BudgetExhausted) ); // A goal run that left the loop on its own terms reported its outcome; // the after-loop guard must not invent a second stop for it. - assert_eq!(stop_reason(ExecutionMode::Goal, 4000, "", true, false), None); + assert_eq!( + stop_reason(ExecutionMode::Goal, 4000, "", true, false), + None + ); assert_eq!( stop_reason(ExecutionMode::Goal, 4000, "我先跳过这一步", true, true), Some(StopReason::MidTaskText) @@ -178,8 +193,17 @@ mod tests { fn progress_and_quoted_markers_do_not_complete_a_goal() { assert_eq!(goal_outcome("[GOAL_COMPLETE]"), GoalOutcome::Continue); assert_eq!(goal_outcome("\n[GOAL_BLOCKED]"), GoalOutcome::Continue); - assert_eq!(goal_outcome("Next I will use [GOAL_COMPLETE]."), GoalOutcome::Continue); - assert_eq!(goal_outcome("Verified output.\n[GOAL_COMPLETE]"), GoalOutcome::Complete); - assert_eq!(goal_outcome("Please supply the date.\n[GOAL_BLOCKED]"), GoalOutcome::NeedsInput); + assert_eq!( + goal_outcome("Next I will use [GOAL_COMPLETE]."), + GoalOutcome::Continue + ); + assert_eq!( + goal_outcome("Verified output.\n[GOAL_COMPLETE]"), + GoalOutcome::Complete + ); + assert_eq!( + goal_outcome("Please supply the date.\n[GOAL_BLOCKED]"), + GoalOutcome::NeedsInput + ); } } diff --git a/crates/harness/src/lib.rs b/crates/harness/src/lib.rs index b1061f3..6300a8f 100644 --- a/crates/harness/src/lib.rs +++ b/crates/harness/src/lib.rs @@ -1,6 +1,6 @@ -mod resolve; pub mod execution; pub mod policy; +mod resolve; mod voice; pub use resolve::*; diff --git a/crates/harness/src/voice.rs b/crates/harness/src/voice.rs index 7d2c4b3..86419f5 100644 --- a/crates/harness/src/voice.rs +++ b/crates/harness/src/voice.rs @@ -116,21 +116,36 @@ pub enum VoiceEvent { AudioPcm(Vec), SpeechStarted, SpeechStopped, - InputTranscript { text: String, final_: bool }, - OutputTranscript { text: String, final_: bool }, + InputTranscript { + text: String, + final_: bool, + }, + OutputTranscript { + text: String, + final_: bool, + }, FunctionCall { call_id: String, name: String, arguments: String, }, - FunctionCallOutput { call_id: String, output: String }, - SpeakNow { text: String }, - InjectContext { text: String }, + FunctionCallOutput { + call_id: String, + output: String, + }, + SpeakNow { + text: String, + }, + InjectContext { + text: String, + }, ResponseCreate, ResponseStarted, ResponseFinished, CancelResponse, - Error { message: String }, + Error { + message: String, + }, } #[derive(Debug, Clone)] @@ -190,7 +205,9 @@ fn native_roots() -> Result { let certs = rustls_native_certs::load_native_certs(); roots.add_parsable_certificates(certs.certs); if roots.is_empty() { - return Err(VoiceError::Message("No trusted TLS certificates available".into())); + return Err(VoiceError::Message( + "No trusted TLS certificates available".into(), + )); } Ok(roots) } @@ -222,18 +239,21 @@ impl VoiceRealtime for HostedVoice { ); // Choose explicitly: the dependency graph enables both ring and aws-lc-rs. // Rustls's automatic provider selection panics in that configuration. - let tls = rustls::ClientConfig::builder_with_provider( - Arc::new(rustls::crypto::ring::default_provider()), - ) + let tls = rustls::ClientConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) .with_safe_default_protocol_versions() .map_err(|error| VoiceError::Message(error.to_string()))? .with_root_certificates(native_roots()?) .with_no_client_auth(); let (stream, _) = tokio_tungstenite::connect_async_tls_with_config( - http_request, None, false, Some(tokio_tungstenite::Connector::Rustls(Arc::new(tls))), + http_request, + None, + false, + Some(tokio_tungstenite::Connector::Rustls(Arc::new(tls))), ) - .await - .map_err(|error| VoiceError::Message(error.to_string()))?; + .await + .map_err(|error| VoiceError::Message(error.to_string()))?; let (write, read) = stream.split(); let socket = HostedSocket { provider: self.provider, @@ -241,7 +261,9 @@ impl VoiceRealtime for HostedVoice { read, }; socket - .send_raw(Message::Text(session_update_json(self.provider, &request).into())) + .send_raw(Message::Text( + session_update_json(self.provider, &request).into(), + )) .await?; for (role, text) in &request.history { if text.trim().is_empty() { @@ -266,10 +288,13 @@ impl VoiceRealtime for HostedVoice { } } -type HostedWrite = - futures_util::stream::SplitSink>, Message>; -type HostedRead = - futures_util::stream::SplitStream>>; +type HostedWrite = futures_util::stream::SplitSink< + tokio_tungstenite::WebSocketStream>, + Message, +>; +type HostedRead = futures_util::stream::SplitStream< + tokio_tungstenite::WebSocketStream>, +>; struct HostedSocket { provider: VoiceProvider, @@ -447,10 +472,7 @@ pub fn parse_provider_event(text: &str) -> Option { .and_then(Value::as_str) .or_else(|| event.get("text").and_then(Value::as_str))? .to_string(); - Some(VoiceEvent::InputTranscript { - text, - final_: true, - }) + Some(VoiceEvent::InputTranscript { text, final_: true }) } "response.output_audio.delta" | "response.audio.delta" => { let encoded = event.get("delta").and_then(Value::as_str)?; @@ -473,10 +495,7 @@ pub fn parse_provider_event(text: &str) -> Option { .or_else(|| event.get("text").and_then(Value::as_str)) .unwrap_or("") .to_string(); - Some(VoiceEvent::OutputTranscript { - text, - final_: true, - }) + Some(VoiceEvent::OutputTranscript { text, final_: true }) } "response.function_call_arguments.done" => { let call_id = event.get("call_id")?.as_str()?.to_string(); @@ -574,10 +593,7 @@ impl VoiceSocket for ScriptedSocket { } VoiceEvent::SpeakNow { text } | VoiceEvent::InjectContext { text } => { let tx = self.incoming.lock().await; - let _ = tx.send(VoiceEvent::OutputTranscript { - text, - final_: true, - }); + let _ = tx.send(VoiceEvent::OutputTranscript { text, final_: true }); let _ = tx.send(VoiceEvent::AudioPcm(Self::beep())); } VoiceEvent::ResponseCreate => {} @@ -597,31 +613,49 @@ mod tests { #[test] fn voice_tls_config_uses_an_explicit_provider() { - let config = rustls::ClientConfig::builder_with_provider( - Arc::new(rustls::crypto::ring::default_provider()), - ).with_safe_default_protocol_versions().unwrap() - .with_root_certificates(native_roots().unwrap()).with_no_client_auth(); + let config = rustls::ClientConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_safe_default_protocol_versions() + .unwrap() + .with_root_certificates(native_roots().unwrap()) + .with_no_client_auth(); assert!(!config.crypto_provider().cipher_suites.is_empty()); } #[test] fn openai_audio_uses_json_and_nested_session_settings() { let request = VoiceConnectRequest { - api_key: String::new(), model_id: "gpt-realtime".into(), - voice_id: "marin".into(), instructions: "test".into(), - tools: vec![], history: vec![], + api_key: String::new(), + model_id: "gpt-realtime".into(), + voice_id: "marin".into(), + instructions: "test".into(), + tools: vec![], + history: vec![], }; - let session: Value = serde_json::from_str(&session_update_json(VoiceProvider::Openai, &request)).unwrap(); + let session: Value = + serde_json::from_str(&session_update_json(VoiceProvider::Openai, &request)).unwrap(); assert!(session["session"].get("voice").is_none()); assert!(session["session"].get("turn_detection").is_none()); - assert_eq!(session.pointer("/session/audio/input/format/rate"), Some(&json!(24000))); - assert_eq!(session.pointer("/session/audio/input/turn_detection/type"), Some(&json!("server_vad"))); + assert_eq!( + session.pointer("/session/audio/input/format/rate"), + Some(&json!(24000)) + ); + assert_eq!( + session.pointer("/session/audio/input/turn_detection/type"), + Some(&json!("server_vad")) + ); let event = VoiceEvent::AudioPcm(vec![0, 1, 2, 3]); - let Some(Message::Text(text)) = encode_provider_event(VoiceProvider::Openai, &event) else { panic!("expected JSON audio") }; + let Some(Message::Text(text)) = encode_provider_event(VoiceProvider::Openai, &event) else { + panic!("expected JSON audio") + }; let encoded: Value = serde_json::from_str(&text).unwrap(); assert_eq!(encoded["type"], "input_audio_buffer.append"); assert_eq!(encoded["audio"], "AAECAw=="); - assert!(matches!(encode_provider_event(VoiceProvider::Xai, &event), Some(Message::Binary(_)))); + assert!(matches!( + encode_provider_event(VoiceProvider::Xai, &event), + Some(Message::Binary(_)) + )); } #[test] @@ -656,9 +690,7 @@ mod tests { #[test] fn parses_xai_and_openai_event_aliases() { - let started = parse_provider_event( - r#"{"type":"input_audio_buffer.speech_started"}"#, - ); + let started = parse_provider_event(r#"{"type":"input_audio_buffer.speech_started"}"#); assert!(matches!(started, Some(VoiceEvent::SpeechStarted))); let transcript = parse_provider_event( @@ -672,9 +704,7 @@ mod tests { other => panic!("{other:?}"), } - let old_audio = parse_provider_event( - r#"{"type":"response.audio.delta","delta":"AQID"}"#, - ); + let old_audio = parse_provider_event(r#"{"type":"response.audio.delta","delta":"AQID"}"#); assert!(matches!(old_audio, Some(VoiceEvent::AudioPcm(_)))); let tool = parse_provider_event( diff --git a/crates/sandbox/src/docker.rs b/crates/sandbox/src/docker.rs index 773c62f..88f7263 100644 --- a/crates/sandbox/src/docker.rs +++ b/crates/sandbox/src/docker.rs @@ -3,8 +3,9 @@ use base64::Engine; use lazyboy_contracts::ComputerCapabilities; use lazyboy_contracts::{ActiveWindow, ComputerObservation, CursorPosition, SandboxKind}; use lazyboy_control::{ - ActionRequest, ActionResult, AdapterContext, CommandRequest, CommandResult, ComputerRef, - EnsureScreenRequest, EnsureScreenResult, FileEntry, ProvisionRequest, SandboxError, + ActionRequest, ActionResult, AdapterContext, BrowserRequest, CdpPage, CommandRequest, + CommandResult, ComputerRef, EnsureScreenRequest, EnsureScreenResult, FileEntry, + ProvisionRequest, RecordingRequest, RecordingResult, RecordingSession, SandboxError, SandboxProvider, ScreenSession, observation_from_png, observation_with_elements, parse_ui_elements, }; @@ -251,6 +252,93 @@ impl SandboxProvider for DockerSandbox { }) } + async fn browser( + &self, + computer: &ComputerRef, + request: BrowserRequest, + context: &AdapterContext, + ) -> Result { + let response = self + .client + .post(self.url(&format!("/computers/{}/browser", computer.id))) + .headers(self.headers(context)) + .json(&request) + .send() + .await + .map_err(|error| SandboxError::message(error.to_string()))?; + let body: Value = response + .json() + .await + .map_err(|error| SandboxError::message(error.to_string()))?; + serde_json::from_value(body).map_err(|error| SandboxError::message(error.to_string())) + } + + async fn start_recording( + &self, + computer: &ComputerRef, + request: RecordingRequest, + context: &AdapterContext, + ) -> Result { + let response = self + .client + .post(self.url(&format!("/computers/{}/recording/start", computer.id))) + .headers(self.headers(context)) + .json(&request) + .send() + .await + .map_err(|error| SandboxError::message(error.to_string()))?; + let body: Value = response + .json() + .await + .map_err(|error| SandboxError::message(error.to_string()))?; + serde_json::from_value(body).map_err(|error| SandboxError::message(error.to_string())) + } + + async fn stop_recording( + &self, + computer: &ComputerRef, + request: RecordingRequest, + context: &AdapterContext, + ) -> Result<(), SandboxError> { + let response = self + .client + .post(self.url(&format!("/computers/{}/recording/stop", computer.id))) + .headers(self.headers(context)) + .json(&request) + .send() + .await + .map_err(|error| SandboxError::message(error.to_string()))?; + if response.status().is_success() { + Ok(()) + } else { + Err(SandboxError::message(format!( + "stop recording failed: {}", + response.status() + ))) + } + } + + async fn collect_recording( + &self, + computer: &ComputerRef, + request: RecordingRequest, + context: &AdapterContext, + ) -> Result { + let response = self + .client + .post(self.url(&format!("/computers/{}/recording/collect", computer.id))) + .headers(self.headers(context)) + .json(&request) + .send() + .await + .map_err(|error| SandboxError::message(error.to_string()))?; + let body: Value = response + .json() + .await + .map_err(|error| SandboxError::message(error.to_string()))?; + serde_json::from_value(body).map_err(|error| SandboxError::message(error.to_string())) + } + async fn connect_screen( &self, computer: &ComputerRef, diff --git a/crates/sandbox/src/fake.rs b/crates/sandbox/src/fake.rs index 627df9c..592a34b 100644 --- a/crates/sandbox/src/fake.rs +++ b/crates/sandbox/src/fake.rs @@ -4,9 +4,9 @@ use std::sync::Mutex; use async_trait::async_trait; use lazyboy_contracts::{ComputerObservation, SandboxKind}; use lazyboy_control::{ - ActionRequest, ActionResult, AdapterContext, CommandRequest, CommandResult, ComputerRef, - FileEntry, ProvisionRequest, SandboxError, SandboxProvider, ScreenSession, - observation_from_png, + ActionRequest, ActionResult, AdapterContext, BrowserRequest, CdpPage, CommandRequest, + CommandResult, ComputerRef, FileEntry, ProvisionRequest, RecordingRequest, RecordingResult, + RecordingSession, SandboxError, SandboxProvider, ScreenSession, observation_from_png, }; const EMPTY_PNG: &[u8] = &[ @@ -103,6 +103,50 @@ impl SandboxProvider for FakeSandbox { }) } + async fn browser( + &self, + _computer: &ComputerRef, + _request: BrowserRequest, + _context: &AdapterContext, + ) -> Result { + Ok(CdpPage { + ok: true, + url: "about:blank".into(), + title: "fake".into(), + ..CdpPage::default() + }) + } + + async fn start_recording( + &self, + _computer: &ComputerRef, + request: RecordingRequest, + _context: &AdapterContext, + ) -> Result { + Ok(RecordingSession { + skill_id: request.skill_id, + output_dir: "/tmp/lazyboy/teach-fake".into(), + }) + } + + async fn stop_recording( + &self, + _computer: &ComputerRef, + _request: RecordingRequest, + _context: &AdapterContext, + ) -> Result<(), SandboxError> { + Ok(()) + } + + async fn collect_recording( + &self, + _computer: &ComputerRef, + _request: RecordingRequest, + _context: &AdapterContext, + ) -> Result { + Ok(RecordingResult { events: Vec::new() }) + } + async fn connect_screen( &self, computer: &ComputerRef, diff --git a/crates/supervisor/src/docker.rs b/crates/supervisor/src/docker.rs index 692d647..7820a8b 100644 --- a/crates/supervisor/src/docker.rs +++ b/crates/supervisor/src/docker.rs @@ -13,9 +13,10 @@ use bollard::models::{EndpointSettings, HostConfig, HostConfigLogConfig, PortBin use bollard::network::{ConnectNetworkOptions, CreateNetworkOptions}; use futures_util::StreamExt; use lazyboy_control::{ - ActionRequest, CommandRequest, CommandResult, EnsureScreenRequest, EnsureScreenResult, HOME, - ScreenTarget, TEAM_SCREEN_LIMIT, normalize_display, normalize_workspace_path, - pointer_state_command_on, screen_layout, screenshot_command_on, window_list_command_on, + ActionRequest, BrowserRequest, CommandRequest, CommandResult, EnsureScreenRequest, + EnsureScreenResult, HOME, RecordingRequest, ScreenTarget, TEAM_SCREEN_LIMIT, normalize_display, + normalize_workspace_path, pointer_state_command_on, screen_layout, screenshot_command_on, + window_list_command_on, }; use tokio::time::{Duration, sleep}; @@ -187,7 +188,15 @@ impl DockerHost { ), format!( "LAZYBOY_COMPUTER_SUDO={}", - if computer_sudo_enabled() { "true" } else { "false" } + if computer_sudo_enabled() { + "true" + } else { + "false" + } + ), + format!( + "LAZYBOY_COMPUTER_DRIVER={}", + lazyboy_control::ComputerDriver::from_env().as_str() ), ]), labels: Some(labels), @@ -384,6 +393,25 @@ impl DockerHost { self.control_act(id, &request, &target).await } + pub async fn browser( + &self, + id: &str, + request: BrowserRequest, + target: &ScreenTarget, + ) -> Result { + self.control_browser(id, &request, target).await + } + + pub async fn recording( + &self, + id: &str, + action: &str, + request: RecordingRequest, + target: &ScreenTarget, + ) -> Result { + self.control_recording(id, action, &request, target).await + } + pub async fn screen_url(&self, id: &str, interactive: bool) -> Result { self.screen_url_for(id, 0, interactive).await } @@ -421,11 +449,7 @@ impl DockerHost { .await .map_err(|error| error.to_string())?; if info.state.as_ref().and_then(|state| state.running) == Some(true) { - let name = info - .name - .unwrap_or_default() - .trim_matches('/') - .to_string(); + let name = info.name.unwrap_or_default().trim_matches('/').to_string(); if name.is_empty() { return Err("computer container has no name".into()); } @@ -994,6 +1018,87 @@ PY"#, } serde_json::from_str(&result.stdout).map_err(|error| error.to_string()) } + + async fn control_browser( + &self, + id: &str, + request: &BrowserRequest, + target: &ScreenTarget, + ) -> Result { + let payload = serde_json::to_string(request).map_err(|error| error.to_string())?; + let token = self.container_control_token(id).await?; + let timeout = if request.action == "click" { + "140" + } else { + "30" + }; + let mut argv = vec![ + "curl".into(), + "-fsS".into(), + "--max-time".into(), + timeout.into(), + "-H".into(), + format!("Authorization: Bearer {token}"), + "-H".into(), + format!("x-lazyboy-display: {}", target.display), + "-H".into(), + "content-type: application/json".into(), + ]; + if let Some(profile) = &target.profile_path { + argv.extend(["-H".into(), format!("x-lazyboy-profile: {profile}")]); + } + argv.extend([ + "--data-binary".into(), + "@-".into(), + "http://127.0.0.1:7070/browser".into(), + ]); + let result = self + .exec_raw_cmd(id, &argv, None, target, Some(payload)) + .await?; + if result.code != 0 { + return Err(result.stderr); + } + serde_json::from_str(&result.stdout).map_err(|error| error.to_string()) + } + + async fn control_recording( + &self, + id: &str, + action: &str, + request: &RecordingRequest, + target: &ScreenTarget, + ) -> Result { + let payload = serde_json::to_string(request).map_err(|error| error.to_string())?; + let token = self.container_control_token(id).await?; + let timeout = if action == "collect" { "30" } else { "20" }; + let mut argv = vec![ + "curl".into(), + "-fsS".into(), + "--max-time".into(), + timeout.into(), + "-H".into(), + format!("Authorization: Bearer {token}"), + "-H".into(), + format!("x-lazyboy-display: {}", target.display), + "-H".into(), + "content-type: application/json".into(), + ]; + if let Some(profile) = &target.profile_path { + argv.extend(["-H".into(), format!("x-lazyboy-profile: {profile}")]); + } + argv.extend([ + "--data-binary".into(), + "@-".into(), + format!("http://127.0.0.1:7070/recording/{action}"), + ]); + let result = self + .exec_raw_cmd(id, &argv, None, target, Some(payload)) + .await?; + if result.code != 0 { + return Err(result.stderr); + } + serde_json::from_str(&result.stdout).map_err(|error| error.to_string()) + } } fn image_ids_match(wanted: &str, have: &str) -> bool { @@ -1041,7 +1146,10 @@ fn computer_pids_limit() -> i64 { } fn computer_sudo_enabled() -> bool { - matches!(std::env::var("LAZYBOY_COMPUTER_SUDO").as_deref(), Ok("1" | "true" | "yes")) + matches!( + std::env::var("LAZYBOY_COMPUTER_SUDO").as_deref(), + Ok("1" | "true" | "yes") + ) } /// LXCFS supplies cgroup-aware /proc views so tools such as htop and free diff --git a/crates/supervisor/src/main.rs b/crates/supervisor/src/main.rs index a23f344..001b54e 100644 --- a/crates/supervisor/src/main.rs +++ b/crates/supervisor/src/main.rs @@ -8,8 +8,8 @@ use axum::routing::{delete, get, post}; use axum::{Json, Router}; use docker::DockerHost; use lazyboy_control::{ - ActionRequest, CommandRequest, EnsureScreenRequest, HOME, ScreenTarget, - normalize_workspace_path, + ActionRequest, BrowserRequest, CommandRequest, EnsureScreenRequest, HOME, RecordingRequest, + ScreenTarget, normalize_workspace_path, }; use serde::{Deserialize, Serialize}; use tracing_subscriber::EnvFilter; @@ -77,6 +77,10 @@ async fn main() { .route("/computers/{id}/exec", post(exec)) .route("/computers/{id}/observe", post(observe)) .route("/computers/{id}/act", post(act)) + .route("/computers/{id}/browser", post(browser)) + .route("/computers/{id}/recording/start", post(recording_start)) + .route("/computers/{id}/recording/stop", post(recording_stop)) + .route("/computers/{id}/recording/collect", post(recording_collect)) .route("/computers/{id}/screens", post(ensure_screen)) .route("/computers/{id}/screen-mode", post(screen_mode)) .route("/computers/{id}/files", get(list_files).post(write_file)) @@ -205,6 +209,99 @@ async fn act( Ok(Json(result)) } +async fn browser( + State(app): State, + headers: HeaderMap, + Path(id): Path, + Json(body): Json, +) -> Result, StatusCode> { + require_token(&headers, &app.token)?; + let mut body = body; + let target = screen_target(&headers); + if body.display.is_none() { + body.display = Some(target.display.clone()); + } + if body.profile_path.is_none() { + body.profile_path = target.profile_path.clone(); + } + let result = app + .docker + .browser(&id, body, &target) + .await + .map_err(|error| { + tracing::error!("browser: {error}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + Ok(Json(result)) +} + +fn with_screen_target(mut body: RecordingRequest, target: &ScreenTarget) -> RecordingRequest { + if body.display.is_none() { + body.display = Some(target.display.clone()); + } + if body.profile_path.is_none() { + body.profile_path = target.profile_path.clone(); + } + body +} + +async fn recording_start( + State(app): State, + headers: HeaderMap, + Path(id): Path, + Json(body): Json, +) -> Result, StatusCode> { + require_token(&headers, &app.token)?; + let target = screen_target(&headers); + let result = app + .docker + .recording(&id, "start", with_screen_target(body, &target), &target) + .await + .map_err(|error| { + tracing::error!("recording start: {error}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + Ok(Json(result)) +} + +async fn recording_stop( + State(app): State, + headers: HeaderMap, + Path(id): Path, + Json(body): Json, +) -> Result, StatusCode> { + require_token(&headers, &app.token)?; + let target = screen_target(&headers); + let result = app + .docker + .recording(&id, "stop", with_screen_target(body, &target), &target) + .await + .map_err(|error| { + tracing::error!("recording stop: {error}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + Ok(Json(result)) +} + +async fn recording_collect( + State(app): State, + headers: HeaderMap, + Path(id): Path, + Json(body): Json, +) -> Result, StatusCode> { + require_token(&headers, &app.token)?; + let target = screen_target(&headers); + let result = app + .docker + .recording(&id, "collect", with_screen_target(body, &target), &target) + .await + .map_err(|error| { + tracing::error!("recording collect: {error}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + Ok(Json(result)) +} + #[derive(Deserialize)] struct ScreenModeBody { interactive: bool, diff --git a/docker-compose.yml b/docker-compose.yml index 8565321..c8f9faa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,6 +51,7 @@ services: LAZYBOY_COMPUTER_MEMORY_MB: ${LAZYBOY_COMPUTER_MEMORY_MB:-2048} LAZYBOY_COMPUTER_PIDS: ${LAZYBOY_COMPUTER_PIDS:-2048} LAZYBOY_COMPUTER_SUDO: ${LAZYBOY_COMPUTER_SUDO:-false} + LAZYBOY_COMPUTER_DRIVER: ${LAZYBOY_COMPUTER_DRIVER:-cua} LAZYBOY_LXCFS_ROOT: /var/lib/lxcfs SUPERVISOR_BIND: 0.0.0.0:7091 DATA_DIR: /data diff --git a/docs/cua-compatibility.md b/docs/cua-compatibility.md new file mode 100644 index 0000000..5865cd9 --- /dev/null +++ b/docs/cua-compatibility.md @@ -0,0 +1,87 @@ +# Cua Driver compatibility (LazyBoy desktop) + +This report answers one question, from a real `make cua-smoke` run on 2026-09-07: + +> Can Cua Driver reliably control the existing LazyBoy XFCE + Xvfb desktop container? + +**Yes.** Five core actions succeeded 10 consecutive times. Production `computer_observe` / `computer_act` / `browser` still use the legacy controld path; this only proves Cua as a driver behind those abstractions. + +## Environment + +- Image: `lazyboy/computer:local` (`image/computer/Dockerfile`) +- Distro: Debian bookworm, x86_64 +- Display: Xvfb `DISPLAY=:1` at 1280×800, XFCE (`xfwm4` compositor off, `xfce4-panel`, `xfdesktop`) +- Accessibility: AT-SPI 2 per screen (`at-spi-bus-launcher` + `at-spi2-registryd`) +- Browser: Debian `chromium` via `lazyboy-browser` (persistent profile, `--remote-debugging-port=9221+display`, `--force-renderer-accessibility`, `--lang=zh-TW`) +- Cua Driver: **0.23.2** (`cua-driver-rs-v0.23.2` linux-x86_64-binary, SHA256 `01bf8339ec129cc00f4b4b2c6056ef1a7c5b52df39ff83ad17c9b16818aec500`) +- Install path: `/usr/local/lib/cua-driver` + `/usr/local/bin/cua-driver` (not under the persisted `/home/lazyboy` bind) +- Daemon: `cua-driver serve --grant existing-profile --socket /tmp/lazyboy/cua.sock --no-overlay` on the primary display only +- Telemetry: disabled +- How to reproduce: `make cua-smoke` (artifacts in `/tmp/lazyboy-cua-smoke-last/`) + +## Doctor + +`cua-driver doctor --json` exit 0, `ok: true`. + +| Probe | Status | Note | +| --- | --- | --- | +| binary | ok | `cua-driver 0.23.2 (x86_64-linux)` | +| install dir | ok | `/usr/local/lib/cua-driver/cua-driver` | +| telemetry | ok | disabled | +| display server | ok | X11 `DISPLAY=:1` | +| X11 connection | ok | connected, visible top-level windows | +| AT-SPI | **warn** | CLI `doctor` (docker exec) does not always see the XFCE session bus. The **daemon** started from `lazyboy-screen` does: native `get_window_state` + AT-SPI click/type worked 10/10. | + +`cua-driver status --socket /tmp/lazyboy/cua.sock`: daemon running, permission mode `standard`. Unix socket rejects uid 0; smoke and future controld calls must run as uid 1000 (`lazyboy`). + +## Results (10 consecutive iterations) + +Independent application state, not Cua `"ok"`: + +| Check | Result | +| --- | --- | +| `cua-driver --version` | `cua-driver 0.23.2` | +| screenshot (`get_desktop_state`) | 10/10 PNG of the XFCE desktop | +| window / accessibility observation | 10/10 `list_windows` + GTK `get_window_state` | +| native click | 10/10 GTK `Smoke Click` wrote `/tmp/lazyboy/cua-smoke-clicked` | +| native type | 10/10 GTK entry + `Smoke Save` wrote `hello-cua` | +| Chromium attach (existing window/profile) | 10/10 `browser_prepare` `attached_existing_profile` | +| browser semantic click / type | 10/10 local `http://127.0.0.1:8765/cua-smoke.html`; DOM became `clicked-ok` then `typed:hello-cua` | +| noVNC `:6080` still up | 10/10 | +| leftover Cua processes | none (only `cua-driver serve`) | + +Same Chromium **pid 334** / **window_id 29360131** across all ten iterations. `browser_prepare` side effects were all false: no isolated profile, no copy, no restart, no extra remote-debugging toggle (LazyBoy already exposes loopback CDP). + +Element refs are snapshot-scoped (`p1:1`, `p4:1`, … `p28:1`). Reusing an old ref would be wrong; the smoke re-snapshots every action. + +## Relevant tools (0.23.2 `list-tools`) + +Observation / native input: `get_desktop_state`, `list_windows`, `get_accessibility_tree`, `get_window_state`, `click`, `type_text`, `press_key`, `hotkey`, `scroll`, `drag`, `get_cursor_position`, `get_screen_size`. + +Browser (attach only): `browser_prepare` (`strategy.kind=existing_profile`, `allow_launch=false`), `get_browser_state` (`semantic_v2`), `browser_navigate` (http/https/about only), `browser_click`, `browser_type`. + +Not used here: recording, isolated `launch_app` browsers, Wayland helpers. These names must not be exposed to the LLM. + +## Integration notes for the next PR + +- Call Cua as uid 1000 via `cua-driver call --socket /tmp/lazyboy/cua.sock`. Root is rejected (`reject Unix peer uid 0 for runtime owned by uid 1000`). +- `get_browser_state` on a live LazyBoy Chromium first returns `browser_consent_required` / `consumer_profile_endpoint_requires_grant`. Then `browser_prepare` with `existing_profile` attaches. Serve must keep `--grant existing-profile`. Never `allow_launch`. +- Linux Chromium trusted CDP pointer is unavailable; smoke used `browser_click` `input_route=dom_event` and verified the DOM. Production adapter should prefer that route on this platform and treat `browser_input_trust_unavailable` as classified, not a silent xdotool fallback. +- Extra Team screens are extra Xvfb `DISPLAY`s. This POC only runs a daemon on `:1`. Later: one socket per slot. +- `browser_navigate` refuses `file://`; local fixtures need `http://127.0.0.1`. +- `get_window_state` on Linux is `additionalProperties: false` — do not send macOS-only fields such as `include_accessibility_tree`. + +## Known limits + +- Doctor AT-SPI warn from a non-desktop D-Bus is not a daemon failure. +- Overlay warnings (`X11 channel rejected command`) appeared in the daemon log with `--no-overlay`; they did not block actions. +- Debian Chromium + zh-TW UI: existing-profile attach worked because CDP was already open, so Cua did not need the English setup-checkbox path. +- Multi-screen, pause/resume, takeover, and skill recording were **not** in this POC; later PRs added controller routing, browser attach, takeover re-observe, and dual-source skill recording. + +## Conclusion + +Cua Driver 0.23.2 **can** control the existing LazyBoy XFCE + Xvfb container: screenshot, window/AT-SPI observation, native click/type, and Chromium semantic click/type, 10/10, without replacing the browser profile or breaking noVNC. + +`ComputerController` is in place. Production now defaults to `cua`. Set `LAZYBOY_COMPUTER_DRIVER=legacy` on the supervisor (passed into each desktop container) to roll back to CDP/AT-SPI/xdotool. Recreate desktop containers after changing the flag. + +With `cua`: `POST /observe`, `POST /act`, and `POST /browser` go through Cua Driver. The Agent-facing `browser` schema is unchanged (`snapshot` / `click` / `type` / `press` / `navigate` / `wait`); Cua attaches with `existing_profile` and maps `semantic_v2` refs (`pN:M`) onto the existing element list. After human takeover ends, the run forces a fresh `computer_observe` and drops pre-handoff ids/refs. Skill teaching starts Cua `start_recording` (no video) plus the existing CDP DOM recorder so a human noVNC demo still yields semantic click/type/navigate events; Cua trajectory turns are ingested as extra evidence and password-labelled typing is masked. `use_saved_login` still fills via CDP stdin so passwords never appear on argv. `cdp.py` / AT-SPI remain for login fill, human browser recording, and the `legacy` rollback. diff --git a/docs/development.md b/docs/development.md index b541504..71a7644 100644 --- a/docs/development.md +++ b/docs/development.md @@ -38,6 +38,13 @@ python3 tests/control.test.py python3 tests/log-rotation.test.py python3 tests/shell-session.test.py # 持久終端機腳本，只需要 tmux +# Cua Driver 能否控制現有 XFCE + Xvfb 桌面（會建 computer image） +make cua-smoke +# 結果摘要見 docs/cua-compatibility.md +# 生產路徑預設是 cua。要回退 CDP/xdotool： +# LAZYBOY_COMPUTER_DRIVER=legacy +# 寫進 .env 後重建 supervisor 與桌面容器。Agent 工具 schema 不變。 + # Python 整合測試用 docker compose exec 連進 Postgres，自己建一次性資料庫後清掉 python3 tests/retention.test.py python3 tests/run-resume.test.py diff --git a/docs/operations.md b/docs/operations.md index 9b215e9..8ea9c6f 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -57,6 +57,7 @@ | `LAZYBOY_COMPUTER_MEMORY_MB` | 每台電腦記憶體 | `2048` | | `LAZYBOY_COMPUTER_PIDS` | 每台電腦 PID 上限 | `2048` | | `LAZYBOY_COMPUTER_SUDO` | 容器內免密碼 sudo；重建桌面容器後生效 | `false` | +| `LAZYBOY_COMPUTER_DRIVER` | 桌面控制後端：`cua`（預設）或 `legacy`（observe / act / browser / 示範錄製）；重建桌面容器後生效 | `cua` | | `LAZYBOY_MEMORY_ENABLED` | 長期記憶 | `true` | 完整清單與保留政策請見 [`.env.example`](../.env.example)。 diff --git a/image/computer/Dockerfile b/image/computer/Dockerfile index 90dcc06..b7ffdba 100644 --- a/image/computer/Dockerfile +++ b/image/computer/Dockerfile @@ -65,6 +65,9 @@ RUN printf '%s\n' \ libatk-adaptor \ python3-gi \ gir1.2-atspi-2.0 \ + gir1.2-gtk-3.0 \ + libxi6 \ + libxkbcommon0 \ zsh \ && echo "zh_TW.UTF-8 UTF-8" >> /etc/locale.gen \ && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \ @@ -136,8 +139,27 @@ COPY --chmod=644 image/computer/xfce/helper-terminal.desktop /usr/share/xfce4/he COPY --chmod=644 image/computer/xfce/helper-browser.desktop /usr/share/xfce4/helpers/lazyboy-browser.desktop COPY --chmod=755 image/computer/chromium /usr/local/bin/chromium RUN sed -i 's|^Exec=/usr/bin/chromium|Exec=/usr/local/bin/lazyboy-browser|' /usr/share/applications/chromium.desktop || true + +# Pin Cua Driver outside the persisted /home/lazyboy bind-mount. +# SHA256 is the linux-x86_64-binary tarball from the matching GitHub release. +ARG CUA_DRIVER_RS_VERSION=0.23.2 +ARG CUA_DRIVER_RS_SHA256=01bf8339ec129cc00f4b4b2c6056ef1a7c5b52df39ff83ad17c9b16818aec500 +RUN curl -fsSL -o /tmp/cua-driver.tar.gz \ + "https://github.com/trycua/cua/releases/download/cua-driver-rs-v${CUA_DRIVER_RS_VERSION}/cua-driver-rs-${CUA_DRIVER_RS_VERSION}-linux-x86_64-binary.tar.gz" \ + && echo "${CUA_DRIVER_RS_SHA256} /tmp/cua-driver.tar.gz" | sha256sum -c \ + && mkdir -p /usr/local/lib/cua-driver \ + && tar -xzf /tmp/cua-driver.tar.gz -C /usr/local/lib/cua-driver \ + && chmod 755 /usr/local/lib/cua-driver/cua-driver \ + && ln -sf /usr/local/lib/cua-driver/cua-driver /usr/local/bin/cua-driver \ + && rm -f /tmp/cua-driver.tar.gz \ + && cua-driver --version + COPY --chmod=755 image/computer/start.sh /usr/local/bin/lazyboy-computer COPY --chmod=755 image/computer/entrypoint.sh /usr/local/bin/lazyboy-entrypoint +COPY --chmod=644 image/computer/cua-smoke.html /usr/share/lazyboy/cua-smoke.html +COPY --chmod=755 image/computer/cua-smoke-gtk.py /usr/local/bin/lazyboy-cua-smoke-gtk +COPY --chmod=755 scripts/cua-smoke-inner.py /usr/local/bin/lazyboy-cua-smoke +COPY --chmod=755 scripts/cua-smoke-test.sh /usr/local/bin/lazyboy-cua-smoke-host USER root ENV HOME=/home/lazyboy DISPLAY=:1 SHELL=/bin/zsh TERM=xterm-256color \ diff --git a/image/computer/cua-smoke-gtk.py b/image/computer/cua-smoke-gtk.py new file mode 100755 index 0000000..6e4664e --- /dev/null +++ b/image/computer/cua-smoke-gtk.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Tiny GTK window used by the Cua smoke test to verify native click/type.""" + +from pathlib import Path + +import gi + +gi.require_version("Gtk", "3.0") +from gi.repository import Gtk + +ROOT = Path("/tmp/lazyboy") +CLICKED = ROOT / "cua-smoke-clicked" +TYPED = ROOT / "cua-smoke-typed" + + +class SmokeWindow(Gtk.Window): + def __init__(self) -> None: + super().__init__(title="LazyBoy Cua Smoke") + self.set_default_size(480, 240) + box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=8) + box.set_margin_top(16) + box.set_margin_bottom(16) + box.set_margin_start(16) + box.set_margin_end(16) + self.add(box) + + self.status = Gtk.Label(label="ready") + self.entry = Gtk.Entry() + self.entry.set_placeholder_text("type here") + click = Gtk.Button(label="Smoke Click") + save = Gtk.Button(label="Smoke Save") + click.connect("clicked", self.on_click) + save.connect("clicked", self.on_save) + box.pack_start(self.status, False, False, 0) + box.pack_start(click, False, False, 0) + box.pack_start(self.entry, False, False, 0) + box.pack_start(save, False, False, 0) + + click.get_accessible().set_name("Smoke Click") + self.entry.get_accessible().set_name("Smoke Entry") + save.get_accessible().set_name("Smoke Save") + self.status.get_accessible().set_name("Smoke Status") + self.connect("destroy", Gtk.main_quit) + + def on_click(self, _button: Gtk.Button) -> None: + ROOT.mkdir(parents=True, exist_ok=True) + CLICKED.write_text("clicked\n", encoding="utf-8") + self.status.set_text("clicked-ok") + + def on_save(self, _button: Gtk.Button) -> None: + ROOT.mkdir(parents=True, exist_ok=True) + TYPED.write_text(self.entry.get_text(), encoding="utf-8") + self.status.set_text("typed-ok") + + +def main() -> None: + ROOT.mkdir(parents=True, exist_ok=True) + window = SmokeWindow() + window.show_all() + Gtk.main() + + +if __name__ == "__main__": + main() diff --git a/image/computer/cua-smoke.html b/image/computer/cua-smoke.html new file mode 100644 index 0000000..d8ac468 --- /dev/null +++ b/image/computer/cua-smoke.html @@ -0,0 +1,31 @@ + + + + + LazyBoy Cua Smoke + + + +

LazyBoy Cua Smoke

+

ready

+

+ +

+

+ + +

+ + + diff --git a/image/computer/lazyboy-browser b/image/computer/lazyboy-browser index 58c7a82..9bf0b5c 100755 --- a/image/computer/lazyboy-browser +++ b/image/computer/lazyboy-browser @@ -54,6 +54,7 @@ exec /usr/bin/chromium \ --disable-session-crashed-bubble \ --hide-crash-restore-bubble \ --disable-infobars \ + --force-renderer-accessibility \ --password-store=basic \ --lang=zh-TW \ --accept-lang=zh-TW,zh,en-US,en \ diff --git a/image/computer/lazyboy-screen b/image/computer/lazyboy-screen index 062b926..5f29a13 100755 --- a/image/computer/lazyboy-screen +++ b/image/computer/lazyboy-screen @@ -211,6 +211,48 @@ start_vnc() { wait_port "$view_port" } +start_cua_driver() { + local display="$1" + local log="$2" + local number="${display#:}" + if ! command -v cua-driver >/dev/null 2>&1; then + return 0 + fi + if alive_pidfile "${log}-cua.pid"; then + return 0 + fi + export CUA_DRIVER_RS_HOME="${CUA_DRIVER_RS_HOME:-$ROOT/cua-home}" + mkdir -p "$CUA_DRIVER_RS_HOME" + cua-driver telemetry disable >>"${log}-cua.log" 2>&1 || true + local sock="$ROOT/cua-${number}.sock" + if [[ "$number" == "1" ]]; then + sock="$ROOT/cua.sock" + fi + rm -f "$sock" + DISPLAY="$display" cua-driver serve \ + --grant existing-profile \ + --socket "$sock" \ + --pid-file "${log}-cua.pid" \ + --no-overlay \ + >>"${log}-cua.log" 2>&1 & + local bg=$! + local n + for n in $(seq 1 50); do + if [[ -S "$sock" ]]; then + return 0 + fi + if ! kill -0 "$bg" 2>/dev/null; then + echo "cua-driver serve exited" >&2 + cat "${log}-cua.log" >&2 || true + return 1 + fi + sleep 0.1 + done + echo "cua-driver socket was not ready" >&2 + cat "${log}-cua.log" >&2 || true + return 1 +} + start_xterm() { local display="$1" local log="$2" @@ -286,6 +328,7 @@ ensure_slot() { exit 1 } start_desktop "$display" "$xfce_home" "$log" + start_cua_driver "$display" "$log" || true start_xterm "$display" "$log" start_vnc "$display" "$vnc_port" "$view_port" "$log" || exit 1 if [[ -n "$profile" ]]; then diff --git a/scripts/cua-smoke-inner.py b/scripts/cua-smoke-inner.py new file mode 100755 index 0000000..d29ec75 --- /dev/null +++ b/scripts/cua-smoke-inner.py @@ -0,0 +1,785 @@ +#!/usr/bin/env python3 +"""In-container Cua Driver smoke test for the LazyBoy XFCE + Xvfb desktop.""" + +from __future__ import annotations + +import argparse +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +ROOT = Path("/tmp/lazyboy") +SOCKET = ROOT / "cua.sock" +REPORT = ROOT / "cua-smoke-report" +CLICKED = ROOT / "cua-smoke-clicked" +TYPED = ROOT / "cua-smoke-typed" +GTK_SCRIPT = Path("/usr/local/bin/lazyboy-cua-smoke-gtk") +HTML = Path("/usr/share/lazyboy/cua-smoke.html") +FIXTURE_PORT = 8765 +FIXTURE_URL = f"http://127.0.0.1:{FIXTURE_PORT}/cua-smoke.html" +TYPED_TEXT = "hello-cua" + + +class SmokeError(RuntimeError): + pass + + +def log(message: str) -> None: + print(message, flush=True) + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, default=str) + "\n", encoding="utf-8") + + +def run( + argv: list[str], + timeout: int = 60, + env: dict[str, str] | None = None, + input_text: str | None = None, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + argv, + check=False, + capture_output=True, + text=True, + timeout=timeout, + env=env, + input=input_text, + ) + + +def parse_jsonish(text: str) -> Any: + text = text.strip() + if not text: + return None + try: + return json.loads(text) + except json.JSONDecodeError: + pass + start = text.find("{") + end = text.rfind("}") + if start >= 0 and end > start: + try: + return json.loads(text[start : end + 1]) + except json.JSONDecodeError: + return None + start = text.find("[") + end = text.rfind("]") + if start >= 0 and end > start: + try: + return json.loads(text[start : end + 1]) + except json.JSONDecodeError: + return None + return None + + +def walk(value: Any) -> list[Any]: + found = [value] + if isinstance(value, dict): + for item in value.values(): + found.extend(walk(item)) + elif isinstance(value, list): + for item in value: + found.extend(walk(item)) + return found + + +def first_list_of_dicts(value: Any, required_key: str) -> list[dict[str, Any]]: + for node in walk(value): + if isinstance(node, list) and node and all(isinstance(item, dict) for item in node): + if any(required_key in item for item in node): + return node + if isinstance(node, dict) and required_key in node and isinstance(node[required_key], list): + items = node[required_key] + if items and all(isinstance(item, dict) for item in items): + return items + return [] + + +def cua_env() -> dict[str, str]: + env = os.environ.copy() + env.setdefault("DISPLAY", ":1") + env.setdefault("CUA_DRIVER_RS_HOME", str(ROOT / "cua-home")) + env["PATH"] = "/usr/local/bin:/usr/local/lib/cua-driver:" + env.get("PATH", "") + dbus = ROOT / "screen-1.dbus" + if dbus.exists() and "DBUS_SESSION_BUS_ADDRESS" not in env: + env["DBUS_SESSION_BUS_ADDRESS"] = dbus.read_text(encoding="utf-8").strip() + runtime = ROOT / "screen-1.runtime" + if runtime.exists() and "XDG_RUNTIME_DIR" not in env: + env["XDG_RUNTIME_DIR"] = runtime.read_text(encoding="utf-8").strip() + env.setdefault("GTK_MODULES", "atk-bridge") + env.setdefault("GTK_A11Y", "atspi") + env.setdefault("NO_AT_BRIDGE", "0") + return env + + +def cua_bin(args: list[str], timeout: int = 60) -> subprocess.CompletedProcess[str]: + return run(["cua-driver", *args], timeout=timeout, env=cua_env()) + + +def cua_call(tool: str, payload: dict[str, Any] | None = None, extra: list[str] | None = None, timeout: int = 90) -> dict[str, Any]: + argv = ["call", "--socket", str(SOCKET), tool] + if extra: + argv.extend(extra) + argv.append(json.dumps(payload or {})) + proc = cua_bin(argv, timeout=timeout) + parsed = parse_jsonish(proc.stdout) or parse_jsonish(proc.stderr) + result = { + "tool": tool, + "code": proc.returncode, + "stdout": proc.stdout, + "stderr": proc.stderr, + "parsed": parsed, + } + if proc.returncode != 0: + raise SmokeError(f"{tool} failed ({proc.returncode}): {proc.stderr or proc.stdout}") + combined = f"{proc.stdout or ''}\n{proc.stderr or ''}" + if "❌" in combined: + raise SmokeError(f"{tool} reported an error: {combined[-2000:]}") + return result + + +def wait_ready(timeout: int) -> None: + deadline = time.time() + timeout + ready = ROOT / "ready" + while time.time() < deadline: + if ready.exists() and SOCKET.is_socket(): + status = cua_bin(["status", "--socket", str(SOCKET)], timeout=15) + text = (status.stdout or "") + (status.stderr or "") + if status.returncode == 0 and "not running" not in text.lower(): + return + time.sleep(0.4) + raise SmokeError("desktop or cua-driver socket was not ready") + + +def png_ok(path: Path) -> None: + data = path.read_bytes() + if len(data) < 32 or data[:8] != b"\x89PNG\r\n\x1a\n": + raise SmokeError(f"{path} is not a PNG ({len(data)} bytes)") + + +def kill_matching(pattern: str) -> None: + run(["pkill", "-f", pattern], timeout=10) + + +def launch_gtk() -> subprocess.Popen[bytes]: + kill_matching("lazyboy-cua-smoke-gtk") + for path in (CLICKED, TYPED): + if path.exists(): + path.unlink() + log_path = REPORT / "gtk.log" + log_file = log_path.open("w", encoding="utf-8") + proc = subprocess.Popen( + [str(GTK_SCRIPT)], + env=cua_env(), + stdout=log_file, + stderr=log_file, + start_new_session=True, + ) + deadline = time.time() + 15 + while time.time() < deadline: + if proc.poll() is not None: + log_file.close() + detail = log_path.read_text(encoding="utf-8")[-1500:] + raise SmokeError(f"GTK smoke window exited immediately: {detail}") + windows = list_windows() + if find_gtk_window(windows): + return proc + time.sleep(0.3) + log_file.close() + raise SmokeError(f"GTK smoke window did not appear; windows={list_windows()}") + + +def start_fixture_server() -> subprocess.Popen[bytes]: + proc = subprocess.Popen( + [ + "python3", + "-m", + "http.server", + str(FIXTURE_PORT), + "--bind", + "127.0.0.1", + "--directory", + str(HTML.parent), + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + deadline = time.time() + 8 + while time.time() < deadline: + check = run( + [ + "python3", + "-c", + f"import urllib.request; urllib.request.urlopen('{FIXTURE_URL}', timeout=1).read()", + ], + timeout=5, + ) + if check.returncode == 0: + return proc + if proc.poll() is not None: + raise SmokeError("fixture HTTP server exited") + time.sleep(0.2) + raise SmokeError("fixture HTTP server did not become ready") + + +def launch_browser() -> None: + env = cua_env() + subprocess.Popen( + ["lazyboy-browser", FIXTURE_URL], + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + deadline = time.time() + 25 + while time.time() < deadline: + windows = list_windows() + if find_browser_window(windows): + return + time.sleep(0.4) + raise SmokeError("Chromium window did not appear") + + +def list_windows() -> list[dict[str, Any]]: + result = cua_call("list_windows", {"on_screen_only": True}) + windows = first_list_of_dicts(result["parsed"], "window_id") + if not windows: + windows = first_list_of_dicts(result["parsed"], "title") + return windows + + +def window_title(window: dict[str, Any]) -> str: + return str(window.get("title") or window.get("app_name") or "") + + +def find_window(windows: list[dict[str, Any]], title_part: str) -> dict[str, Any] | None: + needle = title_part.lower() + for window in windows: + if needle in window_title(window).lower(): + return window + return None + + +def find_gtk_window(windows: list[dict[str, Any]]) -> dict[str, Any] | None: + for window in windows: + title = window_title(window) + app = str(window.get("app_name") or "") + if title == "LazyBoy Cua Smoke" and "chrom" not in app.lower(): + return window + return None + + +def find_browser_window(windows: list[dict[str, Any]]) -> dict[str, Any] | None: + for window in windows: + blob = " ".join( + str(window.get(key) or "") + for key in ("title", "app_name", "application", "wm_class") + ).lower() + if "chrom" in blob or "lazyboy cua smoke" in blob: + return window + return None + + +def snapshot(pid: int, window_id: int, name: str) -> dict[str, Any]: + out = REPORT / f"{name}.png" + result = cua_call( + "get_window_state", + { + "pid": pid, + "window_id": window_id, + "include_screenshot": True, + "screenshot_out_file": str(out), + }, + timeout=120, + ) + parsed = result["parsed"] or {} + elements = first_list_of_dicts(parsed, "element_index") or first_list_of_dicts( + parsed, "element_token" + ) + snapshot_id = None + for node in walk(parsed): + if isinstance(node, dict) and node.get("snapshot_id"): + snapshot_id = node["snapshot_id"] + break + return { + "result": result, + "elements": elements, + "snapshot_id": snapshot_id, + "png": out if out.exists() else None, + } + + +def element_by_label(elements: list[dict[str, Any]], label: str) -> dict[str, Any]: + needle = label.lower() + for element in elements: + hay = " ".join( + str(element.get(key) or "") + for key in ("label", "name", "title", "role", "value") + ).lower() + if needle in hay: + return element + raise SmokeError(f"no accessibility element matching {label!r}") + + +def click_element(pid: int, window_id: int, element: dict[str, Any], snapshot_id: Any) -> None: + payload: dict[str, Any] = {"pid": pid, "window_id": window_id} + if element.get("element_token"): + payload["element_token"] = element["element_token"] + elif element.get("element_index") is not None and snapshot_id: + payload["element_index"] = element["element_index"] + payload["snapshot_id"] = snapshot_id + else: + raise SmokeError(f"element has no token/index: {element}") + cua_call("click", payload) + + +def type_element(pid: int, window_id: int, element: dict[str, Any], snapshot_id: Any, text: str) -> None: + payload: dict[str, Any] = {"pid": pid, "window_id": window_id, "text": text} + if element.get("element_token"): + payload["element_token"] = element["element_token"] + elif element.get("element_index") is not None and snapshot_id: + payload["element_index"] = element["element_index"] + payload["snapshot_id"] = snapshot_id + cua_call("type_text", payload) + + +def wait_file(path: Path, expect: str | None, timeout: float = 8.0) -> str: + deadline = time.time() + timeout + while time.time() < deadline: + if path.exists(): + text = path.read_text(encoding="utf-8").strip() + if expect is None or expect in text: + return text + time.sleep(0.2) + raise SmokeError(f"{path} did not contain {expect!r}") + + +def desktop_screenshot() -> None: + out = REPORT / "desktop.png" + cua_call( + "get_desktop_state", + {"screenshot_out_file": str(out)}, + extra=["--screenshot-out-file", str(out)], + ) + if not out.exists(): + # Some builds only honour the JSON field or only the flag. + time.sleep(0.2) + if not out.exists(): + raise SmokeError("get_desktop_state did not write a screenshot") + png_ok(out) + + +def collect_diagnostics() -> dict[str, Any]: + version = cua_bin(["--version"]) + doctor = cua_bin(["doctor", "--json"], timeout=90) + if doctor.returncode != 0: + doctor = cua_bin(["doctor"], timeout=90) + tools = cua_bin(["list-tools"], timeout=30) + status = cua_bin(["status", "--socket", str(SOCKET)], timeout=15) + report = { + "version": (version.stdout or version.stderr).strip(), + "doctor_code": doctor.returncode, + "doctor_stdout": doctor.stdout, + "doctor_stderr": doctor.stderr, + "doctor_json": parse_jsonish(doctor.stdout), + "list_tools": tools.stdout, + "status": (status.stdout or "") + (status.stderr or ""), + "display": os.environ.get("DISPLAY", ":1"), + "socket": str(SOCKET), + } + write_json(REPORT / "diagnostics.json", report) + (REPORT / "list-tools.txt").write_text(tools.stdout or "", encoding="utf-8") + (REPORT / "doctor.txt").write_text( + (doctor.stdout or "") + (doctor.stderr or ""), encoding="utf-8" + ) + if version.returncode != 0: + raise SmokeError("cua-driver --version failed") + return report + + +def native_round(iteration: int) -> None: + for path in (CLICKED, TYPED): + if path.exists(): + path.unlink() + gtk = launch_gtk() + try: + windows = list_windows() + window = find_gtk_window(windows) + if not window: + raise SmokeError(f"GTK window missing: {windows}") + pid = int(window["pid"]) + window_id = int(window["window_id"]) + state = snapshot(pid, window_id, f"native-{iteration}") + click_el = element_by_label(state["elements"], "Smoke Click") + click_element(pid, window_id, click_el, state["snapshot_id"]) + wait_file(CLICKED, "clicked") + state = snapshot(pid, window_id, f"native-type-{iteration}") + entry = element_by_label(state["elements"], "Smoke Entry") + type_element(pid, window_id, entry, state["snapshot_id"], TYPED_TEXT) + state = snapshot(pid, window_id, f"native-save-{iteration}") + save = element_by_label(state["elements"], "Smoke Save") + click_element(pid, window_id, save, state["snapshot_id"]) + wait_file(TYPED, TYPED_TEXT) + finally: + gtk.send_signal(signal.SIGTERM) + try: + gtk.wait(timeout=5) + except subprocess.TimeoutExpired: + gtk.kill() + + +def extract_text(value: Any) -> str: + chunks: list[str] = [] + for node in walk(value): + if isinstance(node, str) and 0 < len(node) < 400: + chunks.append(node) + elif isinstance(node, dict): + for key in ("text", "value", "label", "name", "url", "title", "result", "ref"): + item = node.get(key) + if isinstance(item, str) and 0 < len(item) < 400: + chunks.append(item) + return "\n".join(chunks) + + +def strip_heavy(value: Any) -> Any: + if isinstance(value, dict): + out = {} + for key, item in value.items(): + if isinstance(item, str) and len(item) > 400: + out[key] = f"<{len(item)} chars>" + else: + out[key] = strip_heavy(item) + return out + if isinstance(value, list): + return [strip_heavy(item) for item in value] + return value + + +def find_browser_ref(parsed: Any, needle: str) -> str | None: + needle = needle.lower() + for node in walk(parsed): + if not isinstance(node, dict): + continue + ref = node.get("ref") + if not isinstance(ref, str) or not ref: + continue + blob = " ".join( + str(node.get(key) or "") + for key in ( + "name", + "label", + "text", + "role", + "description", + "accessible_name", + "selector", + "value", + ) + ).lower() + if needle in blob or needle in json.dumps(strip_heavy(node), default=str).lower(): + return ref + return None + + +def browser_ids(parsed: Any) -> tuple[Any, Any]: + target_id = None + tab_id = None + for node in walk(parsed): + if not isinstance(node, dict): + continue + target_id = target_id or node.get("target_id") + tab_id = tab_id or node.get("tab_id") + tabs = node.get("tabs") + if isinstance(tabs, list) and tabs and isinstance(tabs[0], dict): + tab_id = tab_id or tabs[0].get("tab_id") or tabs[0].get("id") + return target_id, tab_id + + +def browser_prepare(pid: int, window_id: int) -> dict[str, Any]: + try: + return cua_call( + "browser_prepare", + { + "pid": pid, + "window_id": window_id, + "session": "lazyboy-smoke", + "strategy": {"kind": "existing_profile"}, + "allow_launch": False, + }, + timeout=180, + ) + except SmokeError as error: + return {"error": str(error)} + + +def browser_round(iteration: int) -> dict[str, Any]: + windows = list_windows() + window = find_browser_window(windows) + if not window: + launch_browser() + windows = list_windows() + window = find_browser_window(windows) + if not window: + raise SmokeError(f"no Chromium window: {windows}") + pid = int(window["pid"]) + window_id = int(window["window_id"]) + bind_payload = { + "pid": pid, + "window_id": window_id, + "include_screenshot": False, + "session": "lazyboy-smoke", + } + state = cua_call("get_browser_state", bind_payload, timeout=120) + note = extract_text(state["parsed"]) + state["stdout"] + prepared = None + if any( + marker in note + for marker in ( + "browser_requires_setup", + "browser_consent_required", + "requires_grant", + "existing_profile", + ) + ): + prepared = browser_prepare(pid, window_id) + write_json(REPORT / "browser-prepare.json", prepared) + prep_text = extract_text(prepared.get("parsed")) + str(prepared.get("stdout") or "") + str(prepared.get("error") or "") + if prepared.get("error") or "❌" in prep_text or "refused" in prep_text: + raise SmokeError(f"browser_prepare failed: {prep_text[-2000:]}") + state = cua_call("get_browser_state", bind_payload, timeout=120) + note = extract_text(state["parsed"]) + state["stdout"] + target_id, tab_id = browser_ids(state["parsed"]) + if not target_id or not tab_id: + raise SmokeError(f"browser bind missing target/tab: {note[-2000:]}") + cua_call( + "browser_navigate", + { + "target_id": target_id, + "tab_id": tab_id, + "url": FIXTURE_URL, + "session": "lazyboy-smoke", + }, + timeout=60, + ) + time.sleep(0.8) + snap = cua_call( + "get_browser_state", + { + "target_id": target_id, + "tab_id": tab_id, + "include_screenshot": False, + "session": "lazyboy-smoke", + "snapshot_format": "semantic_v2", + "query": "Smoke", + }, + timeout=90, + ) + write_json(REPORT / f"browser-snap-{iteration}.json", strip_heavy(snap["parsed"])) + click_ref = find_browser_ref(snap["parsed"], "smoke click") + type_ref = find_browser_ref(snap["parsed"], "smoke entry") + save_ref = find_browser_ref(snap["parsed"], "smoke save") + if not type_ref: + type_ref = find_browser_ref(snap["parsed"], "textbox") or find_browser_ref( + snap["parsed"], "input" + ) + if not click_ref: + raise SmokeError(f"browser snapshot has no Smoke Click ref: {extract_text(snap['parsed'])[-1500:]}") + click_result = cua_call( + "browser_click", + { + "target_id": target_id, + "tab_id": tab_id, + "ref": click_ref, + "session": "lazyboy-smoke", + "input_route": "dom_event", + }, + timeout=60, + ) + write_json(REPORT / f"browser-click-{iteration}.json", strip_heavy(click_result["parsed"] or click_result["stdout"])) + time.sleep(0.4) + after_click = cua_call( + "get_browser_state", + { + "target_id": target_id, + "tab_id": tab_id, + "session": "lazyboy-smoke", + "snapshot_format": "semantic_v2", + "include_screenshot": False, + }, + timeout=90, + ) + write_json(REPORT / f"browser-after-click-{iteration}.json", strip_heavy(after_click["parsed"])) + click_text = json.dumps(strip_heavy(after_click["parsed"]), default=str) + if "clicked-ok" not in click_text: + raise SmokeError(f"browser click did not change page text to clicked-ok: {click_text[-1500:]}") + type_ref = find_browser_ref(after_click["parsed"], "smoke entry") or find_browser_ref( + after_click["parsed"], "textbox" + ) + save_ref = find_browser_ref(after_click["parsed"], "smoke save") + if not type_ref: + raise SmokeError("browser snapshot has no Smoke Entry ref") + cua_call( + "browser_type", + { + "target_id": target_id, + "tab_id": tab_id, + "ref": type_ref, + "text": TYPED_TEXT, + "replace": True, + "session": "lazyboy-smoke", + }, + timeout=60, + ) + if not save_ref: + raise SmokeError("browser snapshot has no Smoke Save ref") + cua_call( + "browser_click", + { + "target_id": target_id, + "tab_id": tab_id, + "ref": save_ref, + "session": "lazyboy-smoke", + "input_route": "dom_event", + }, + timeout=60, + ) + time.sleep(0.4) + after_type = cua_call( + "get_browser_state", + { + "target_id": target_id, + "tab_id": tab_id, + "session": "lazyboy-smoke", + "snapshot_format": "semantic_v2", + "include_screenshot": False, + }, + timeout=90, + ) + write_json(REPORT / f"browser-after-type-{iteration}.json", strip_heavy(after_type["parsed"])) + type_text = json.dumps(strip_heavy(after_type["parsed"]), default=str) + if f"typed:{TYPED_TEXT}" not in type_text: + raise SmokeError(f"browser type did not land in the page: {type_text[-1500:]}") + return { + "pid": pid, + "window_id": window_id, + "target_id": target_id, + "tab_id": tab_id, + "prepared": prepared, + "click_ref": click_ref, + } + + +def novnc_ok() -> None: + proc = run( + [ + "python3", + "-c", + "import socket; s=socket.create_connection(('127.0.0.1',6080),2); s.close()", + ], + timeout=10, + ) + if proc.returncode != 0: + raise SmokeError("noVNC port 6080 is not accepting connections") + + +def extra_cua_processes() -> list[str]: + proc = run(["ps", "-eo", "pid,cmd"], timeout=10) + lines = [] + for line in (proc.stdout or "").splitlines(): + if "cua-driver" in line and "serve" not in line and "cua-smoke" not in line: + if "cua-driver serve" in line: + continue + if str(os.getpid()) in line.split()[:1]: + continue + lines.append(line.strip()) + return lines + + +def one_iteration(iteration: int) -> dict[str, Any]: + log(f"iteration {iteration}: screenshot") + desktop_screenshot() + log(f"iteration {iteration}: windows") + windows = list_windows() + if not windows: + raise SmokeError("list_windows returned no windows") + write_json(REPORT / f"windows-{iteration}.json", windows) + log(f"iteration {iteration}: native click/type") + native_round(iteration) + log(f"iteration {iteration}: browser") + browser = browser_round(iteration) + log(f"iteration {iteration}: noVNC") + novnc_ok() + return {"windows": len(windows), "browser": browser} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repeat", type=int, default=10) + parser.add_argument("--ready-timeout", type=int, default=90) + args = parser.parse_args() + REPORT.mkdir(parents=True, exist_ok=True) + summary: dict[str, Any] = { + "repeat": args.repeat, + "passed": 0, + "failed": 0, + "iterations": [], + "ok": False, + } + fixture: subprocess.Popen[bytes] | None = None + try: + wait_ready(args.ready_timeout) + fixture = start_fixture_server() + diagnostics = collect_diagnostics() + summary["diagnostics"] = { + "version": diagnostics["version"], + "doctor_code": diagnostics["doctor_code"], + "status": diagnostics["status"], + } + for iteration in range(1, args.repeat + 1): + try: + detail = one_iteration(iteration) + summary["passed"] += 1 + summary["iterations"].append({"n": iteration, "ok": True, **detail}) + log(f"iteration {iteration}: ok") + except Exception as error: # noqa: BLE001 — smoke must record any failure + summary["failed"] += 1 + summary["iterations"].append( + {"n": iteration, "ok": False, "error": str(error)} + ) + summary["error"] = str(error) + log(f"iteration {iteration}: FAIL {error}") + raise + leftovers = extra_cua_processes() + summary["leftover_cua_processes"] = leftovers + if leftovers: + raise SmokeError(f"extra cua-driver processes: {leftovers}") + summary["ok"] = True + log(f"passed {args.repeat}/{args.repeat}") + return 0 + except subprocess.TimeoutExpired as error: + summary["error"] = f"timeout: {error}" + log(summary["error"]) + return 1 + except SmokeError as error: + summary["error"] = str(error) + log(f"FAIL {error}") + return 1 + finally: + write_json(REPORT / "result.json", summary) + kill_matching("lazyboy-cua-smoke-gtk") + if fixture is not None: + fixture.kill() + kill_matching(f"http.server {FIXTURE_PORT}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/cua-smoke-test.sh b/scripts/cua-smoke-test.sh new file mode 100755 index 0000000..fd3797b --- /dev/null +++ b/scripts/cua-smoke-test.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Prove Cua Driver can control the LazyBoy XFCE + Xvfb desktop. +# Host usage: scripts/cua-smoke-test.sh --docker [--repeat 10] +# In-container: lazyboy-cua-smoke --repeat 10 +set -euo pipefail + +repeat=10 +ready_timeout=120 +image="${COMPUTER_IMAGE:-lazyboy/computer:local}" +name="lazyboy-cua-smoke" +docker_mode=0 + +usage() { + echo "usage: $0 [--docker] [--repeat N] [--image NAME]" >&2 + exit 2 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --docker) docker_mode=1; shift ;; + --repeat) repeat="${2:-}"; shift 2 ;; + --image) image="${2:-}"; shift 2 ;; + --ready-timeout) ready_timeout="${2:-}"; shift 2 ;; + -h|--help) usage ;; + *) usage ;; + esac +done + +if [[ "$docker_mode" -eq 0 ]] && [[ -x /usr/local/bin/lazyboy-cua-smoke ]] && [[ -S /tmp/lazyboy/cua.sock || -f /tmp/lazyboy/ready ]]; then + exec /usr/local/bin/lazyboy-cua-smoke --repeat "$repeat" --ready-timeout "$ready_timeout" +fi + +if [[ "$docker_mode" -eq 0 ]]; then + echo "not inside a LazyBoy desktop; passing --docker to run the computer image" >&2 + docker_mode=1 +fi + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required" >&2 + exit 1 +fi + +if ! docker image inspect "$image" >/dev/null 2>&1; then + echo "building $image (first desktop image build is slow)" >&2 + root="$(cd "$(dirname "$0")/.." && pwd)" + docker build -f "$root/image/computer/Dockerfile" -t "$image" "$root" +fi + +cleanup() { + docker rm -f "$name" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +cleanup +docker run -d --name "$name" --shm-size=512m \ + -e DISPLAY=:1 \ + "$image" >/dev/null + +echo "waiting for desktop + cua-driver in $name" +for _ in $(seq 1 "$ready_timeout"); do + if docker exec -u 1000:1000 "$name" test -f /tmp/lazyboy/ready \ + && docker exec -u 1000:1000 "$name" test -S /tmp/lazyboy/cua.sock; then + break + fi + if ! docker inspect -f '{{.State.Running}}' "$name" 2>/dev/null | grep -q true; then + echo "computer container exited" >&2 + docker logs "$name" >&2 || true + exit 1 + fi + sleep 1 +done + +if ! docker exec -u 1000:1000 "$name" test -f /tmp/lazyboy/ready; then + echo "desktop did not become ready" >&2 + docker exec -u 1000:1000 "$name" sh -c 'ls -la /tmp/lazyboy; tail -n 80 /tmp/lazyboy/*.log 2>/dev/null' >&2 || true + exit 1 +fi + +echo "running $repeat Cua smoke iterations" +set +e +docker exec -u 1000:1000 "$name" /usr/local/bin/lazyboy-cua-smoke --repeat "$repeat" --ready-timeout "$ready_timeout" +code=$? +set -e + +out="${CUA_SMOKE_OUT:-/tmp/lazyboy-cua-smoke-last}" +mkdir -p "$out" +docker cp "$name:/tmp/lazyboy/cua-smoke-report/." "$out/" 2>/dev/null || true +docker exec -u 1000:1000 "$name" sh -c 'tail -n 80 /tmp/lazyboy/screen-1-cua.log 2>/dev/null || true' \ + >"$out/cua-driver.log" || true + +echo "smoke artifacts copied to $out" +exit "$code"