From 2bd00a471647248481cc7f0e298d69ab4e646ca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=80=A7=E9=A9=8A?= Date: Sat, 5 Sep 2026 17:41:44 +0800 Subject: [PATCH 1/9] =?UTF-8?q?feat:=20=E6=8E=92=E7=A8=8B=E3=80=81?= =?UTF-8?q?=E4=BF=9D=E9=9A=AA=E7=AE=B1=E3=80=81=E7=86=B1=E6=A9=9F=E6=A1=8C?= =?UTF-8?q?=E9=9D=A2=E8=88=87=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent 可建 cron 排程，登入帳號進每 bot 保險箱。閒置桌面改 docker pause， 分頁心跳保活，喚醒不再整套重開；啟動／換手用左上頭像小卡。原生 GUI 走 AT-SPI，VNC 貼上與唯讀不拆 iframe。README 補硬體、對 Grok Bot、 流程圖與二次開發目錄。 --- .dockerignore | 10 + .env.example | 6 +- .gitignore | 3 + Cargo.lock | 258 ++++++- Cargo.toml | 5 + Makefile | 26 +- README.md | 562 ++++++++++++-- a.txt | 51 -- apps/web/package-lock.json | 7 + apps/web/package.json | 1 + apps/web/src/App.tsx | 147 +++- apps/web/src/animated-icons.tsx | 2 +- apps/web/src/avatar.css | 31 + apps/web/src/chat.css | 59 ++ apps/web/src/computer.css | 57 ++ apps/web/src/locales/zh-TW.ts | 15 + apps/web/src/main.tsx | 5 + apps/web/src/refinements.css | 760 ++++++++++++++++--- apps/web/src/responsive.css | 28 + apps/web/src/schedule.css | 37 + apps/web/src/schedule.tsx | 198 +++++ apps/web/src/styles.css | 142 +++- apps/web/src/types.ts | 2 +- apps/web/src/use-animations.tsx | 7 + apps/web/vnc.html | 66 +- crates/api/Cargo.toml | 6 + crates/api/src/attachments.rs | 206 ++++-- crates/api/src/auth.rs | 195 ++++- crates/api/src/computer.rs | 243 +++++-- crates/api/src/main.rs | 16 +- crates/api/src/mcp.rs | 15 + crates/api/src/routes.rs | 37 +- crates/api/src/runs.rs | 966 +++++++++++++++++++++++-- crates/api/src/schedules.rs | 794 ++++++++++++++++++++ crates/api/src/screen_proxy.rs | 104 +-- crates/api/src/skills.rs | 3 + crates/api/src/tools.rs | 660 ++++++++++++++--- crates/api/src/vault.rs | 383 ++++++++++ crates/contracts/src/action.rs | 30 +- crates/contracts/src/computer.rs | 4 + crates/control/src/a11y.py | 353 +++++++++ crates/control/src/a11y.rs | 201 +++++ crates/control/src/actions.rs | 297 +++++++- crates/control/src/cdp.py | 171 +++-- crates/control/src/cdp.rs | 34 +- crates/control/src/clipboard.py | 51 ++ crates/control/src/lib.rs | 2 + crates/control/src/sandbox.rs | 23 +- crates/control/src/x11.rs | 43 +- crates/controld/src/main.rs | 65 +- crates/harness/Cargo.toml | 2 + crates/harness/src/resolve.rs | 9 +- crates/sandbox/src/docker.rs | 63 +- crates/sandbox/src/fake.rs | 18 + crates/supervisor/Cargo.toml | 4 + crates/supervisor/src/docker.rs | 364 +++++++--- crates/supervisor/src/main.rs | 74 ++ docker-compose.dev.yml | 5 + docker-compose.yml | 49 +- docs/hero.html | 442 +++++++++++ docs/readme-hero.png | Bin 0 -> 269517 bytes image/api/Dockerfile | 44 +- image/computer/Dockerfile | 48 +- image/computer/lazyboy-browser | 2 +- image/computer/lazyboy-screen | 40 + image/computer/start.sh | 4 + image/supervisor/Dockerfile | 12 +- migrations/011_schedules_and_vault.sql | 34 + reference/rakazo | 2 +- scripts/dev.sh | 2 +- scripts/init-env.py | 21 + scripts/render-readme-hero.sh | 30 + tests/control.test.py | 43 ++ tests/frontend.test.mjs | 33 + 74 files changed, 7816 insertions(+), 916 deletions(-) delete mode 100644 a.txt create mode 100644 apps/web/src/avatar.css create mode 100644 apps/web/src/chat.css create mode 100644 apps/web/src/computer.css create mode 100644 apps/web/src/responsive.css create mode 100644 apps/web/src/schedule.css create mode 100644 apps/web/src/schedule.tsx create mode 100644 apps/web/src/use-animations.tsx create mode 100644 crates/api/src/schedules.rs create mode 100644 crates/api/src/vault.rs create mode 100644 crates/control/src/a11y.py create mode 100644 crates/control/src/a11y.rs create mode 100644 crates/control/src/clipboard.py create mode 100644 docker-compose.dev.yml create mode 100644 docs/hero.html create mode 100644 docs/readme-hero.png create mode 100644 migrations/011_schedules_and_vault.sql create mode 100644 scripts/init-env.py create mode 100755 scripts/render-readme-hero.sh create mode 100644 tests/control.test.py create mode 100644 tests/frontend.test.mjs diff --git a/.dockerignore b/.dockerignore index 520d968..b03ca8f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,13 @@ data reference **/*.md apps/web/node_modules + +.env +.env.* +!.env.example +**/*.pem +**/*.key +**/node_modules +apps/web/dist +**/__pycache__ +.DS_Store diff --git a/.env.example b/.env.example index b8e088c..8edce9a 100644 --- a/.env.example +++ b/.env.example @@ -4,10 +4,14 @@ OPENAI_API_KEY= # Generate independent values with: openssl rand -hex 32 LAZYBOY_APP_TOKEN= SANDBOX_SUPERVISOR_TOKEN= +# Keep this key stable when rotating the app login token. +LAZYBOY_VAULT_KEY= +POSTGRES_PASSWORD=lazyboy +LAZYBOY_BIND_IP=127.0.0.1 SANDBOX_PROVIDER=docker DATABASE_URL=postgres://lazyboy:lazyboy@127.0.0.1:5434/lazyboy DATA_DIR=./data -API_BIND=0.0.0.0:3101 +API_BIND=127.0.0.1:3101 SANDBOX_SUPERVISOR_URL=http://127.0.0.1:7091 LAZYBOY_SECURE_COOKIE=false LAZYBOY_COMPUTER_MEMORY_MB=2048 diff --git a/.gitignore b/.gitignore index 7c1fa46..e137c3b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ node_modules dist .DS_Store .gstack/ + +__pycache__/ +*.pyc diff --git a/Cargo.lock b/Cargo.lock index 9203251..e9a590b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,41 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.12" @@ -55,6 +90,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "ambient-authority" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" + [[package]] name = "android_system_properties" version = "0.1.6" @@ -435,6 +476,36 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "cap-primitives" +version = "3.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e0bf07d379916947be6c4a07f43684153d710a2896c31f9e97781362895596c" +dependencies = [ + "ambient-authority", + "fs-set-times", + "io-extras", + "io-lifetimes", + "ipnet", + "maybe-owned", + "rustix", + "rustix-linux-procfs", + "windows-sys 0.52.0", + "winx", +] + +[[package]] +name = "cap-std" +version = "3.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a59e59fa26472d29680ece6a9f8ee8b0551a719a33df2f5240bde065ecbddfd7" +dependencies = [ + "cap-primitives", + "io-extras", + "io-lifetimes", + "rustix", +] + [[package]] name = "castaway" version = "0.2.4" @@ -491,6 +562,26 @@ dependencies = [ "windows-link", ] +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "cmake" version = "0.1.58" @@ -655,6 +746,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "cron" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5877d3fbf742507b66bc2a1945106bd30dd8504019d596901ddd012a4dd01740" +dependencies = [ + "chrono", + "once_cell", + "winnow 0.6.26", +] + [[package]] name = "crossbeam-deque" version = "0.8.7" @@ -702,9 +804,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "darling" version = "0.20.11" @@ -1141,6 +1253,17 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-set-times" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" +dependencies = [ + "io-lifetimes", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -1305,6 +1428,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gif" version = "0.14.2" @@ -1843,6 +1976,15 @@ dependencies = [ "web-time", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "interpolate_name" version = "0.2.4" @@ -1854,6 +1996,22 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "io-extras" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2285ddfe3054097ef4b2fe909ef8c3bcd1ea52a8f0d274416caebeef39f04a65" +dependencies = [ + "io-lifetimes", + "windows-sys 0.52.0", +] + +[[package]] +name = "io-lifetimes" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" + [[package]] name = "ipnet" version = "2.12.1" @@ -2011,10 +2169,14 @@ dependencies = [ name = "lazyboy-api" version = "0.1.0" dependencies = [ + "aes-gcm", "async-trait", "axum", "base64 0.22.1", + "cap-std", "chrono", + "chrono-tz", + "cron", "dotenvy", "fastembed", "futures-util", @@ -2026,6 +2188,7 @@ dependencies = [ "lazyboy-control", "lazyboy-harness", "lazyboy-sandbox", + "rand 0.8.8", "reqwest 0.12.28", "rig-core", "rmcp", @@ -2089,6 +2252,7 @@ name = "lazyboy-harness" version = "0.1.0" dependencies = [ "lazyboy-contracts", + "reqwest 0.12.28", "rig-core", "serde", "thiserror", @@ -2120,11 +2284,14 @@ dependencies = [ "base64 0.22.1", "bollard", "futures-util", + "hex", + "hmac", "lazyboy-contracts", "lazyboy-control", "reqwest 0.12.28", "serde", "serde_json", + "sha2", "thiserror", "tokio", "tracing", @@ -2277,6 +2444,12 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -2602,6 +2775,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "openssl" version = "0.10.81" @@ -2755,6 +2934,24 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -2827,6 +3024,18 @@ dependencies = [ "miniz_oxide 0.8.9", ] +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.15.0" @@ -3536,6 +3745,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustix-linux-procfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" +dependencies = [ + "once_cell", + "rustix", +] + [[package]] name = "rustls" version = "0.23.43" @@ -3930,6 +4149,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.12" @@ -4568,7 +4793,7 @@ dependencies = [ "indexmap 2.14.1", "toml_datetime", "toml_parser", - "winnow", + "winnow 1.0.4", ] [[package]] @@ -4577,7 +4802,7 @@ version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ - "winnow", + "winnow 1.0.4", ] [[package]] @@ -4823,6 +5048,16 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -5420,6 +5655,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e90edd2ac1aa278a5c4599b1d89cf03074b610800f866d4026dc199d7929a28" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "1.0.4" @@ -5429,6 +5673,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winx" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" +dependencies = [ + "bitflags 2.13.1", + "windows-sys 0.52.0", +] + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/Cargo.toml b/Cargo.toml index 3988c62..182524a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,11 @@ version = "0.1.0" license = "MIT" publish = false +[profile.release] +lto = true +codegen-units = 1 +strip = "symbols" + [workspace.dependencies] lazyboy-contracts = { path = "crates/contracts" } lazyboy-control = { path = "crates/control" } diff --git a/Makefile b/Makefile index 6fb0f2d..183f70b 100644 --- a/Makefile +++ b/Makefile @@ -53,26 +53,12 @@ help: ## Show this help # --- Environment ----------------------------------------------------------- -env: ## Create .env from example with fresh random tokens (no-op if .env exists) - @if [ -f .env ]; then \ - echo ".env already exists — leaving it untouched (use 'make env-force' to regenerate tokens)."; \ - else \ - cp .env.example .env; \ - APP=$$(openssl rand -hex 32); SUP=$$(openssl rand -hex 32); \ - sed "s|^LAZYBOY_APP_TOKEN=.*|LAZYBOY_APP_TOKEN=$$APP|" .env > .env.tmp; \ - sed "s|^SANDBOX_SUPERVISOR_TOKEN=.*|SANDBOX_SUPERVISOR_TOKEN=$$SUP|" .env.tmp > .env.tmp2; \ - mv .env.tmp2 .env; rm -f .env.tmp; \ - echo "created .env with generated LAZYBOY_APP_TOKEN / SANDBOX_SUPERVISOR_TOKEN (64 hex chars)."; \ - echo "next: set XAI_API_KEY in .env, then run 'make up'."; \ - fi +env: ## Create .env with independent random secrets (preserve existing .env) + @python3 scripts/init-env.py -env-force: ## Regenerate .env (wipes XAI_API_KEY — you will set it again) - @cp .env.example .env; \ - APP=$$(openssl rand -hex 32); SUP=$$(openssl rand -hex 32); \ - sed "s|^LAZYBOY_APP_TOKEN=.*|LAZYBOY_APP_TOKEN=$$APP|" .env > .env.tmp; \ - sed "s|^SANDBOX_SUPERVISOR_TOKEN=.*|SANDBOX_SUPERVISOR_TOKEN=$$SUP|" .env.tmp > .env.tmp2; \ - mv .env.tmp2 .env; rm -f .env.tmp; \ - echo "regenerated .env tokens (XAI_API_KEY reset — set it again)." +env-force: ## Refuse destructive key regeneration; existing vault keys must be preserved + @echo "Refusing to overwrite .env: this can orphan saved passwords. See docs/security-and-harness-review.md for safe rotation." + @exit 1 # --- Full Docker stack ----------------------------------------------------- @@ -104,7 +90,7 @@ computer: ## Build the Debian desktop image used to spawn bot computers docker build -f image/computer/Dockerfile -t $(COMPUTER_IMAGE) . postgres: ## Start only postgres and wait until it is ready - $(COMPOSE) up -d postgres + $(COMPOSE) -f docker-compose.yml -f docker-compose.dev.yml up -d postgres @echo "waiting for postgres (127.0.0.1:5434) to be ready..." @for i in $$(seq 1 40); do if $(COMPOSE) exec -T postgres pg_isready -U lazyboy >/dev/null 2>&1; then echo "postgres ready"; break; fi; sleep 0.5; done diff --git a/README.md b/README.md index f27a97e..78a3ed9 100644 --- a/README.md +++ b/README.md @@ -1,72 +1,528 @@ # LazyBoy -Create a bot in the browser, give it a Team or Private computer, and let it drive a Linux desktop. +![給 Agent 一台真的電腦](./docs/readme-hero.png) -## Run (full Docker, recommended) +在瀏覽器裡開 Agent。每個都有自己的 Linux 桌面：開網頁、敲指令、學你示範過的流程。金鑰、模型、檔案都留在你這台機器上。 -Everything (postgres, desktop image build, supervisor, API + web UI) runs in -Docker Compose, driven by the Makefile: +--- + +## 需要的硬體 + +LazyBoy 是本機系統，不是雲端沙盒。機器要跑四件事：**Postgres**、**API（含網頁）**、**supervisor**、以及每個 Agent 的 **Debian 桌面容器**。 + +| 項目 | 最低能跑 | 建議（一台桌面常開） | +| --- | --- | --- | +| 作業系統 | macOS / Linux，已裝 Docker Engine + Compose | 同上，磁碟給 Docker 至少 30 GB | +| CPU | 4 核 | 8 核以上。每個桌面預設吃 2 核（`LAZYBOY_COMPUTER_CPUS`） | +| 記憶體 | 8 GB（只能開一台、還會卡） | **16 GB**。每個桌面預設 2 GB（`LAZYBOY_COMPUTER_MEMORY_MB`）再加上 API、Postgres、嵌入模型 | +| 磁碟 | 約 15 GB（桌面映像 + Postgres） | 家目錄 `data/homes/` 會隨瀏覽器設定檔長大 | +| GPU | 不需要 | 模型走網路 API，畫面是 CPU 上的 Xvfb | +| 網路 | 第一次建映像、拉套件需要 | 之後離線也能開 UI；聊天要模型金鑰能連外 | + +預設一個 Team 電腦容器可同時掛最多 **8** 個螢幕（`TEAM_SCREEN_LIMIT`）。再開私人電腦就是再一個容器、再 2 GB。分頁開著時心跳會讓桌面保持熱機；關掉分頁約 10 分鐘後凍結（記憶體還在），約 6 小時後才真正停機。 + +--- + +## 怎麼快速啟動 + +要有 Docker（含 Compose 外掛）和 Make。第一次會編 `lazyboy/computer:local`（Debian + XFCE + Chromium），會比較久。 ```bash -make env # creates .env with two generated 64-hex tokens (no-op if .env exists) -# then set XAI_API_KEY in .env -make up # builds all images and starts the stack +make env +# 在 .env 填 XAI_API_KEY +# （也可以之後在「本機工作區 → 設定」接 xAI / OpenCode Go / OpenAI 相容端點） +make up make health ``` -Open `http://127.0.0.1:3101` and sign in with `LAZYBOY_APP_TOKEN`. +打開 [http://127.0.0.1:3101](http://127.0.0.1:3101)，用 `.env` 裡的 `LAZYBOY_APP_TOKEN` 登入，建一個 Agent，傳一句話。 -Useful targets (see `make help`): - -| target | what it does | -| --- | --- | -| `make up` / `make down` / `make purge` | start / stop (keep data) / stop + delete postgres volume | -| `make logs`, `make ps`, `make health` | observe the stack | -| `make computer` | build only the heavy desktop image `lazyboy/computer:local` | -| `make postgres`, `make postgres-down` | start/stop just the database on `127.0.0.1:5434` | -| `make build`, `make fmt`, `make clippy`, `make test` | cargo workspace tasks | -| `make web` | build `apps/web` with npm (needs node) | -| `make dev`, `make dev-supervisor`, `make dev-api` | local dev: postgres in Docker, Rust services on the host in two terminals | - -Without make, the full-stack equivalent is: - -```bash -cp .env.example .env # fill in the two tokens + XAI_API_KEY -docker compose up -d --build -``` - -## Run (local) - -Postgres is on `127.0.0.1:5434` so it does not collide with other stacks. +沒有 Make： ```bash cp .env.example .env -# Set XAI_API_KEY, then generate two different secrets: -# openssl rand -hex 32 -# openssl rand -hex 32 -# Put them in LAZYBOY_APP_TOKEN and SANDBOX_SUPERVISOR_TOKEN. -docker compose up -d postgres -./scripts/build-computer-image.sh -SANDBOX_SUPERVISOR_TOKEN= \ -DATA_DIR=/root/LazyBoy/data cargo run -p lazyboy-supervisor -DATABASE_URL=postgres://lazyboy:lazyboy@127.0.0.1:5434/lazyboy \ -SANDBOX_PROVIDER=docker \ -SANDBOX_SUPERVISOR_URL=http://127.0.0.1:7091 \ -SANDBOX_SUPERVISOR_TOKEN= \ -LAZYBOY_APP_TOKEN= \ -DATA_DIR=/root/LazyBoy/data \ -LAZYBOY_WEB_DIR=apps/web \ -API_BIND=0.0.0.0:3101 \ -cargo run -p lazyboy-api +# openssl rand -hex 32 → LAZYBOY_APP_TOKEN +# openssl rand -hex 32 → SANDBOX_SUPERVISOR_TOKEN +# openssl rand -hex 32 → LAZYBOY_VAULT_KEY +docker compose up -d --build ``` -Open `http://:3101` and sign in with `LAZYBOY_APP_TOKEN`. The computer is a real Debian container: fluxbox toolbar, Chromium with tabs and URL bar, and xterm. The display is proxied through the authenticated API; the supervisor is internal-only in Docker Compose and must not be published to the LAN. +前端熱重載：`apps/web` 裡 `npm install && npm run dev`，開 [http://127.0.0.1:5173](http://127.0.0.1:5173)。Vite 把 `/api` 和 `/view` 轉到 3101。 -For LAN use, put LazyBoy behind HTTPS whenever possible. A shared token sent over plain HTTP can be observed by other devices on an untrusted network. Set `LAZYBOY_SECURE_COOKIE=true` when HTTPS terminates at LazyBoy or a trusted reverse proxy. The API refuses a non-loopback bind unless `LAZYBOY_APP_TOKEN` is at least 32 characters; the supervisor likewise rejects missing, short, or default tokens. +本機 Rust 開發（Postgres 仍在 Docker）： -For frontend development, run `npm install && npm run dev` in `apps/web`, then open -`http://127.0.0.1:5173`. Vite proxies API and computer-screen traffic to the Rust API on port 3101. +```bash +make dev # 準備 .env、Postgres、桌面映像 +make dev-supervisor # 終端 1 +make dev-api # 終端 2 +``` -The standalone supervisor listens on `127.0.0.1:7091` by default. Docker Compose reaches it internally as `supervisor:7091`; there is intentionally no host port `7092`. +| 指令 | 做什麼 | +| --- | --- | +| `make up` / `make down` / `make purge` | 啟動／停止（留資料）／連 Postgres 一起清 | +| `make logs` `make ps` `make health` | 看狀態 | +| `make computer` | 只重建桌面映像 | +| `make postgres` | 只開資料庫 `127.0.0.1:5434` | -Model providers: set the workspace default in 本機工作區 → 設定. v1 supports xAI (`XAI_API_KEY`), OpenCode Go (`OPENCODE_GO_API_KEY` or a key saved in settings), and a self-hosted OpenAI-compatible endpoint (base URL + optional key). Keys saved in the UI are stored on the local workspace and take precedence over env vars. +--- + +## 跟 Grok Bot 比，好在哪 + +[Grok Bot](https://grok.com) 是 xAI 的雲端隊友：對話、電腦、排程都在他們的機器上。LazyBoy 走同一類產品（本機開源實作對齊 Rakazo 那條線），差在**誰擁有執行環境**。 + +| | Grok Bot | LazyBoy | +| --- | --- | --- | +| 跑在哪 | xAI 雲端 | 你的 Docker | +| 模型 | Grok | 你帶金鑰：xAI、OpenCode Go、或任何 OpenAI 相容端點 | +| 電腦 | 廠商提供的桌面 | 你映像裡的 Debian／XFCE／Chromium，家目錄在 `data/homes/` | +| 資料 | 在服務端 | 對話、記憶、保險箱、瀏覽器設定檔都在本機 Postgres + 磁碟 | +| 登入帳號 | 跟雲端工作流程走 | 每個 Agent 自己的保險箱（AES-256-GCM），模型只看到帳號 id | +| 客製 | 封閉 | 開源。工具、MCP、技能 JSON 可改可搬 | +| 費用形態 | 訂閱／用量 | 電費與硬體；模型金鑰另計 | +| 多 Agent 同桌 | 產品內建 | Team 電腦一個容器最多 8 螢幕；私人電腦一人一容器 | +| 教會它 | 看產品當下提供什麼 | 你示範一次，CDP 記語意事件，模型整理成技能 | + +適合 LazyBoy 的情況：資料不能出門、要自己選模型、要看它點了哪個控制項、或想把「看完訓練影片交測驗」這種流程做成可匯出的技能。 + +Grok Bot 適合的情況：不想養 Docker、要官方託管、機器不夠力。 + +--- + +## 每個功能簡介 + +**對話與 Session** +每個 Agent 多則對話。訊息進 Postgres，同一則用 `clientNonce` 去重。問候、閒聊走純文字，**不會**為了「看一下螢幕」去開 Docker。 + +**Team / 私人電腦** +Team：工作區共用一個家目錄，每個 bot 有自己的 `DISPLAY`（`:1`、`:2`…）和瀏覽器設定檔。私人：這個 bot 獨佔一個容器。 + +**即時畫面** +右側預覽是 noVNC。瀏覽器連 `/view/{botId}/vnc.html`，API 用已登入的 cookie 轉到容器裡的 websockify。模型看到的截圖另走 `computer_observe`，上面會蓋黃字編號；你盯著的 VNC **沒有**那些編號。 + +**接管 / 釋放** +人按接管就拿到控制租約（預設 15 分鐘，心跳續約）。進行中的 run 會進 `waiting_takeover`，放開後從目前畫面接著做。模型遇到登入牆、2FA、驗證碼會呼叫 `request_takeover`。 + +**觀察與操作** +- Chromium 網頁：`browser`（CDP，點 element id） +- 原生視窗（對話框、檔案管理員、XFCE）：`computer_act`（AT-SPI id，不行再退 xdotool 座標） +- 檔案與指令：`list_files` / `read_file` / `write_file` / `shell` +點到 `[disabled]` 的控制項會最多等 45 秒等它亮。模型用文字回「我在等」會結束整段 run，所以等待必須是 `wait` 工具。 + +**教技能** +你示範，容器內 CDP 錄「點了哪個控制項、填了什麼、去了哪一頁」，再抽幾個關鍵畫面。停下來後模型整理成意圖級 playbook，之後用普通工具在**當下畫面**找控制項，不是重播座標。密碼欄不錄。技能可匯出 JSON。 + +**記憶** +`pgvector` + MiniLM（384 維）。只有你叫它記住、或它呼叫 `remember` 的內容會進長期記憶。密碼與 token 會被拒。清除對話不會清記憶。 + +**保險箱** +每個 bot 自己的站名／帳號／密碼。模型用 `list_accounts` 只看到站與使用者名稱，`use_saved_login` 在 Chromium 登入表單填入。金鑰用 `LAZYBOY_VAULT_KEY` 加密。 + +**排程** +五欄 cron，預設 `Asia/Taipei`。對話裡講「以後每天九點」或側欄新增。tick 迴圈把到期列變成普通 queued run。 + +**MCP** +工作區級外掛。市集或自訂 stdio／HTTP／SSE。stdio 跑在 API 容器裡。 + +**群組** +多個 Agent 同一個 thread。Team 電腦上各用各的螢幕。同一 bot 同時只跑一個 run，後面的訊息排隊。 + +**附件** +圖片給當則模型看，不進歷史二進位。要讓電腦開原檔會放 `inbox/`，兩小時後刪。 + +**頭像與狀態** +Blobatar 色塊＋眼睛。啟動、喚醒、連線、換手時，預覽左上角與思考列會顯示對應文字。分頁開著時心跳保住容器，換手不拆 VNC。 + +--- + +## 系統怎麼疊起來 + +```text +瀏覽器 + ├─ React UI（apps/web） + └─ /view/{bot} → API 畫面代理（cookie）→ 容器 websockify → x11vnc → Xvfb + │ + ▼ + lazyboy-api + ├─ Postgres + pgvector + ├─ runs worker（最多 16 條並行，每個 bot 同時一個） + ├─ computer idle_loop（熱機／凍結／停放） + ├─ schedules tick + ├─ MCP hub + └─ harness：選模型、接金鑰 + │ + ▼ HTTP（SANDBOX_SUPERVISOR_TOKEN） + lazyboy-supervisor :7091 + │ Docker socket + ▼ + lazyboy/computer 容器 + ├─ start.sh → Xvfb :1 + XFCE + x11vnc + websockify + ├─ lazyboy-screen 額外 Team 螢幕 :2…:8 + └─ lazyboy-controld :7070 容器內觀察／動作 + data/homes/ bind 到 /home/lazyboy +``` + +Compose 裡 supervisor **不**對主機開埠。API 在容器網路連 `supervisor:7091`。 + +--- + +## 時序：你送一則訊息 + +```mermaid +sequenceDiagram + participant U as 瀏覽器 + participant A as API + participant DB as Postgres + participant W as run worker + participant H as harness + participant M as 模型 + participant S as supervisor + participant C as 桌面容器 + + U->>A: POST /api/sessions/{id}/messages + A->>DB: 寫 user 訊息（clientNonce 去重） + A->>DB: INSERT run status=queued + A-->>U: 202 + 輪詢 messages / computer status + + loop 每 200ms + W->>DB: 租下一筆 queued（同 bot 沒有別人在跑） + end + W->>H: resolve_backend（bot → 工作區 → 環境變數） + H-->>W: provider + model + key + W->>M: 系統提示 + 歷史 + 記憶 + 工具定義 + + alt 純聊天（問候、閒聊、知識問答） + M-->>W: 純文字 + W->>DB: 寫 assistant 訊息，run=completed + else 要用電腦 + M-->>W: tool_call（browser / computer_act / shell…） + W->>DB: busy_step（電腦啟動中／喚醒中／實際動作） + W->>A: boot() 若尚未 running + A->>S: provision 或 unpause + S->>C: 等 /tmp/lazyboy/ready + W->>C: 執行工具 + C-->>W: 截圖／DOM／stdout + W->>M: tool result（畫面變了才帶圖） + M-->>W: 下一動或結束文字 + end + W->>DB: completed，放開螢幕租約 + U->>A: GET status／messages（2 秒一次） +``` + +同一 bot 已有 `leased`／`running`／`waiting_takeover` 時，新訊息會 `queuedBehindActive`。人正在接管時，後面的話只排隊，思考轉圈不會假裝它還在動。 + +--- + +## 時序：電腦從開機到你看到畫面 + +```mermaid +sequenceDiagram + participant U as 瀏覽器 + participant A as API + participant S as supervisor + participant D as Docker + participant C as 容器 PID 1 + + U->>A: POST /api/computer/{bot}/boot + alt 已是 running 且容器還在 + A-->>U: state=running + else 休眠中（docker pause） + A->>S: POST /computers/{id}/unpause + S->>D: unpause + S->>C: 確認 /tmp/lazyboy/ready（約一秒內） + A-->>U: state=running + else 已關或沒有容器 + A->>DB: state=booting + A->>S: POST /computers + S->>D: 找到可重用的就 start，否則 create + D->>C: start.sh + C->>C: Xvfb :1、XFCE、x11vnc、websockify、controld + C->>C: touch /tmp/lazyboy/ready + A->>S: ensure_screen（Team 再掛 :2…） + A-->>U: state=running + end + U->>A: GET /api/computer/{bot}/screen + A-->>U: /view/{bot}/vnc.html + U->>A: WebSocket /view/{bot}/websockify + A->>C: 轉到該 slot 的 6080+N + Note over U,C: 分頁每 2 秒 heartbeat，更新 computers.updated_at +``` + +閒置（`crates/api/src/computer.rs` `idle_loop`）： + +1. 執行中、超過 10 分鐘沒人看、也沒有進行中的 run／示範 → `docker pause`，狀態 `suspended` +2. 休眠超過 6 小時 → `docker stop`，狀態 `stopped` +3. 分頁還在就心跳，不會進 1 + +--- + +## Harness 流程 + +`crates/harness` 不管滑鼠，只負責**這次 run 要用哪一家模型**。真正的 agent 迴圈在 `crates/api/src/runs.rs`。 + +```mermaid +flowchart TD + A[execute_run] --> B[讀 bot 與工作區設定] + B --> C[CredentialChain] + C --> C1[bot 自己存的 key] + C --> C2[工作區設定的 key] + C --> C3[環境變數 XAI_API_KEY 等] + C1 --> D[resolve_backend] + C2 --> D + C3 --> D + D --> E{provider} + E -->|xai| F[rig xAI CompletionModel] + E -->|opencode-go 且 gpt/grok/muse| G[OpenAI Responses API] + E -->|其他 OpenAI 相容| H[OpenAI Chat Completions] + F --> I[connect_model] + G --> I + H --> I + I --> J[帶工具定義進 complete_once] + J --> K{回傳} + K -->|純文字| L[聊天結束或 skill 檢查沒過就 nudge] + K -->|tool_calls| M[dispatch] + M --> N[結果寫回 history] + N --> J +``` + +金鑰優先順序：**bot → 工作區 → 環境變數**。API 跑在 Docker 時，迴圈位址 `127.0.0.1` 會被改成 `host.docker.internal`，才能打到你本機的相容端點。 + +`execute_run` 每一輪： + +1. 續租約（5 分鐘），否則 halt +2. 寫 `busy_step`（思考中／電腦啟動中／browser click…） +3. `complete_once`：system + 記憶 + 技能目錄 + 歷史 +4. 沒有 tool call：問候就結束；技能 run 若檢查沒過，把現在畫面塞回去再逼一次（最多數次） +5. 有 tool call：需要沙盒才 `prepare_run_computer`（boot／解凍、拿螢幕執行租約、瀏覽器 profile lock） +6. `dispatch` 跑工具，畫面沒變就不重複塞圖 +7. 直到文字結束、halt（停止／接管）、或達到回合上限（聊天 4、一般 40、技能 80） + +問候路徑會把工具表清空，從源頭避免「哈囉」去 `ls` 家目錄。 + +--- + +## 控制電腦的原理 + +模型**從不**直接連 VNC。它只打 API 工具；工具經 sandbox HTTP 進 supervisor，再 `docker exec` 或打容器內 `controld`。 + +```mermaid +flowchart LR + subgraph 模型側 + T1[browser] + T2[computer_observe / computer_act] + T3[shell / files] + end + subgraph 容器內 + CDP[Chromium CDP] + ATSPI[AT-SPI] + XD[xdotool / Xvfb] + VNC[x11vnc] + end + T1 --> CDP + T2 --> ATSPI + T2 --> XD + T3 --> XD + U[你的瀏覽器] --> VNC +``` + +**三層找得到什麼、點得了什麼** + +1. **CDP（網頁）** + Chromium 開著時，`browser` 拿 DOM：可點的控制項編成 1…N，截圖上蓋黃字。`click {element:N}` 會捲到視窗外的節點。這是訓練系統、信箱、後台的主路徑。`computer_act` 點在瀏覽器視窗上會被拒，避免用像素點網頁。 + +2. **AT-SPI（原生 GUI）** + 沒有 DOM 時（檔案選取、XFCE 對話框），`computer_observe` 走無障礙樹。編號是控制項，不是視窗外框。 + +3. **座標（最後）** + 畫布、無樹的 widget 才用 `computer_act` 的 x,y。解析度契約是 **1280×800**。 + +**編號從哪來** +`overlay_elements` 只畫在給模型的 JPEG 上。VNC 是乾淨桌面。每次 navigation／snapshot 會重編號，舊 id 作廢。 + +**誰可以動滑鼠** +每個 bot 一個 `computer_screens` 列：slot、DISPLAY、執行租約 `execution_run_id`、控制租約 `control_holder`。run 要 GUI 時 `take_screen_execution` 把 fence +1，較新的 fence 贏。人接管寫 `control_holder=user`，worker 在回合邊界停，不跟你搶滑鼠。 + +**你看到的畫面** +`screen_proxy` 只接受已登入的 GET／WebSocket。頁面本體是 `apps/web/vnc.html`。`view_only` 用 postMessage 切，不重掛 iframe，所以換手時預覽不會黑掉。 + +**容器內 controld** +`lazyboy-controld` 聽 `127.0.0.1:7070`，要 `LAZYBOY_CONTROL_TOKEN`。supervisor 的觀察／動作能打通就走它，否則退回 `import`／`xdotool` 指令。 + +**Team 多螢幕** +slot 0 = `:1` / VNC 5900 / 畫面 6080。slot N = `:N+1` / 5900+N / 6080+N。`lazyboy-screen ensure` 在同一個容器裡再長一組 Xvfb。瀏覽器設定檔預設 per-bot；兩個 bot 搶同一份 shared profile 會拿到「profile locked」，檔案與 shell 仍可用。 + +--- + +## 教技能怎麼做 + +```mermaid +sequenceDiagram + participant H as 你 + participant A as API + participant C as 桌面 + participant M as 模型 + + H->>A: POST /skills/start {goal} + A->>C: 開電腦、把控制權給人 + A->>C: 啟動 CDP recorder + loop 約 1.5s + A->>C: 視窗標題 + 粗略畫面簽名 + Note over A: 簽名沒變就不存，避免閒置把 60 幀用完 + end + H->>C: 正常操作（密碼欄不錄） + H->>A: POST /skills/stop + A->>C: 停 recorder + A->>M: 目標 + 語意事件 + 最多 8 張關鍵畫面 + M-->>A: playbook（意圖、輸入、步驟、怎麼驗收） + A-->>H: 草稿，可改名、試跑、匯出 JSON +``` + +之後 run 若 prompt 對得上技能名，會把完整 playbook 塞進當則，並清掉舊聊天以免模型複誦上次的「還在倒數」。執行仍用 `browser`／`computer_act`，在**現在**的畫面上找「Next」，不是記像素。 + +--- + +## 排程怎麼進 run + +```mermaid +flowchart LR + A[對話 create_schedule 或側欄新增] --> B[(schedules 表)] + B --> C[tick_loop] + C -->|next_run_at 到了| D[INSERT runs queued] + D --> E[同一個 worker_loop] + E --> F[普通 execute_run，prompt 是 instructions] +``` + +Cron 五欄。時區寫在列上，預設台北。`run now` 只是立刻插一筆 run，不改下一拍時間。 + +--- + +## 專案目錄（二次開發從這裡找） + +Cargo workspace。契約在 `contracts`，畫面邏輯在 `control`，HTTP 與 agent 迴圈在 `api`，Docker 生命週期在 `supervisor`。前端是獨立的 Vite app，由 API 把 `apps/web/dist`（或開發時的 `apps/web`）端出去。 + +```text +LazyBoy/ +├── apps/web/ 瀏覽器 UI（Vite + React） +│ ├── src/App.tsx 幾乎全部畫面：側欄、聊天、電腦、設定 +│ ├── src/schedule.tsx 排程面板 +│ ├── src/avatar.tsx Blobatar 頭像 +│ ├── src/api.ts fetch 包裝、401 +│ ├── src/types.ts 跟 API JSON 對齊的型別 +│ ├── src/locales/zh-TW.ts 所有使用者看得到的字 +│ ├── src/*.css 樣式（styles / chat / computer / refinements…） +│ └── vnc.html 內嵌桌面（noVNC）；API 的 /view 會讀這一檔 +├── crates/ +│ ├── contracts/ 跨 crate 的型別：Bot、Run、ComputerState、動作 JSON +│ ├── harness/ 模型後端：CredentialChain、resolve_backend、connect_model +│ ├── control/ 桌面契約：螢幕 slot、租約、CDP/AT-SPI/xdotool、overlay +│ │ 含 a11y.py / cdp.py（容器裡被 exec 的腳本） +│ ├── sandbox/ API 打 supervisor 的 HTTP 客戶端；fake 給測試 +│ ├── supervisor/ Docker：provision / pause / unpause / exec / observe / act +│ ├── controld/ 打進容器的小 HTTP（127.0.0.1:7070） +│ └── api/ 唯一對外程序：路由、worker、idle、排程 tick、靜態網頁 +│ └── src/ +│ ├── main.rs 啟動、三條背景迴圈 +│ ├── routes.rs 組 router；bot / computer HTTP 也在這 +│ ├── runs.rs agent 迴圈（租約、complete_once、nudge） +│ ├── tools.rs tool_definitions + dispatch（加工具從這裡） +│ ├── computer.rs boot / 凍結 / 心跳 / 螢幕租約 +│ ├── sessions.rs 對話 CRUD、送訊息、SSE +│ ├── skills.rs 示範錄製與蒸馏 +│ ├── schedules.rs cron +│ ├── vault.rs 登入保險箱 +│ ├── memory.rs pgvector 記憶 +│ ├── mcp.rs MCP 連線 +│ ├── screen_proxy.rs /view 反代 +│ └── db.rs SQL 與列定義 +├── image/ +│ ├── api/Dockerfile +│ ├── supervisor/Dockerfile +│ └── computer/ 桌面映像 +│ ├── Dockerfile +│ ├── start.sh PID 1：controld + Xvfb/XFCE/VNC +│ └── lazyboy-screen Team 額外 DISPLAY +├── migrations/ sqlx，檔名流水號；API 啟動時自動 migrate +├── data/homes/ 每個電腦的家目錄（bind 進容器 /home/lazyboy） +├── tests/ 跨語言的小測試（node:test、Python） +├── scripts/ init-env、build-computer-image、dev +├── docker-compose.yml 正式堆疊（Postgres + supervisor + API） +├── Makefile make up / dev / test +└── reference/rakazo/ 上游參考實作，不要當 runtime 依賴 +``` + +crate 依賴方向（不要倒過來 import）： + +```text +api → harness, control, sandbox, contracts +sandbox → control, contracts +supervisor → control, contracts +controld → control, contracts +harness → contracts +control → contracts +``` + +### 想改什麼，開哪個檔 + +| 你要做的事 | 先開 | +| --- | --- | +| 加一個模型工具（例如 `screenshot_region`） | `crates/api/src/tools.rs`（`tool_definitions` + `dispatch`）；若要 GUI，`runs.rs` 的 `tool_needs_sandbox` / `tool_needs_gui` | +| 工具對應的滑鼠／鍵盤／CDP | `crates/control/src/{actions,x11,cdp,a11y}.rs` 與同目錄 `.py` | +| 新的 HTTP 端點 | 功能模組自己的 `router()`（如 `schedules.rs`），在 `routes.rs` `.merge(...)`；電腦／bot 則直接寫在 `routes.rs` | +| 新狀態、動作 JSON、Run 狀態機 | `crates/contracts/src/`（改完 `api` / 前端 `types.ts` 一起對） | +| 換模型供應商或金鑰解析 | `crates/harness/src/resolve.rs`、`crates/contracts/src/model.rs`、`workspace.rs` | +| 容器怎麼開、凍結、等 ready | `crates/supervisor/src/docker.rs`、`crates/api/src/computer.rs` | +| 桌面裡多裝套件、改 XFCE、開機腳本 | `image/computer/`，然後 `make computer` | +| 對話 UI、電腦預覽、頭像小卡 | `apps/web/src/App.tsx` + 對應 css | +| 畫面上的中文 | `apps/web/src/locales/zh-TW.ts`（key 加了 `tsc` 才會過） | +| 內嵌 VNC 行為（貼上、唯讀） | `apps/web/vnc.html` | +| 新資料表 | `migrations/0xx_....sql`；列定義補 `crates/api/src/db.rs` | +| 排程 UI | `apps/web/src/schedule.tsx` | +| MCP 市集清單 | `crates/api/src/mcp_catalog.rs` | + +### 加一支工具的最短路徑 + +1. `tools.rs` 的 `tool_definitions` 加 `ToolDefinition`（名稱、說明、JSON Schema）。說明是寫給模型看的。 +2. 同一個檔的 `dispatch` 加 match arm，回 `ToolOutcome { text, image, pause, blocks }`。 +3. 若會動到桌面：`runs.rs` 裡 `tool_needs_sandbox` / `tool_needs_gui` 把名字加進去，否則不會 boot、也拿不到螢幕租約。 +4. 需要新的容器指令就放 `control`（Rust 組 argv，Python 做 CDP/AT-SPI），supervisor 的 `exec` 已經會把 `DISPLAY` 帶進去。 +5. 前端若要顯示步驟文字，`runs.rs` 的 `describe_step` 加一列。 +6. `SANDBOX_PROVIDER=fake cargo test -p lazyboy-api` 先過，再對真容器看。 + +### 本機二次開發迴圈 + +```bash +make env && make postgres # 資料庫 +make computer # 桌面映像有改才需要 +make dev-supervisor # 終端 1，:7091 +make dev-api # 終端 2，:3101，會自動跑 migrations +# 前端另開： +cd apps/web && npm install && npm run dev # :5173 +``` + +- 改 `apps/web/src`：Vite 熱更新。 +- 改 `crates/api`：停掉 `dev-api` 再 `make dev-api`。 +- 改 `crates/supervisor`：同樣重跑 supervisor。 +- 改 `crates/control` 的 `.py`：映像沒重建的話，執行中的容器還是舊腳本；要嘛 `make computer` 後重開電腦，要嘛確認 supervisor exec 讀的是映像內檔案。 +- 改 `image/computer`：一定 `make computer`，再在 UI 重啟該台電腦。 +- 契約改了：同時改 `contracts`、呼叫端、`apps/web/src/types.ts`。 + +檢查： + +```bash +make fmt && make clippy && make test +node --test tests/frontend.test.mjs # 排程 cron、VNC 貼上、登入填表防護 +``` + +`SANDBOX_PROVIDER=fake` 時 API 不碰 Docker，適合先測 run／工具契約。 + +`reference/rakazo/` 是對齊用的上游，不要在 LazyBoy runtime import 它。 + +--- + +## 安全（操作時要記得） + +- Supervisor 只在 Compose 內網，不要對 LAN 開埠 +- 畫面走已登入 API，VNC 密碼不進瀏覽器 URL +- 區網請走 HTTPS；終端是 HTTPS 時設 `LAZYBOY_SECURE_COOKIE=true` +- API 綁非本機時 `LAZYBOY_APP_TOKEN` 至少 32 字；supervisor 拒絕空白、過短、`dev-token` +- 保險箱用 `LAZYBOY_VAULT_KEY`；換登入 token 時這把 key 要留著 +- 模型看不到密碼本文；2FA／CAPTCHA 一定要人在**它的**畫面上處理 diff --git a/a.txt b/a.txt deleted file mode 100644 index aaa90fc..0000000 --- a/a.txt +++ /dev/null @@ -1,51 +0,0 @@ -我這邊體感滑，是因為多數工作走 Shell／API／瀏覽器 DOM，很少靠截圖猜 XY。你 repo 裡 瀏覽器這條其實已經對了；卡的是 桌面原生 App。 - -你已經有的（別拆） -能力 在哪 -Docker 真實 Linux 桌面 + Screen／lease／takeover supervisor / sandbox / control -shell 工具 crates/api/src/tools.rs -browser（CDP DOM snapshot／click by id／selector） control/cdp.rs + cdp.py；工具描述已寫「網頁優先用 browser」 -編號元素、apply_element_targets contracts/action.rs、control/actions.rs -畫面沒變就不重傳圖（frame signature） control/observe.rs -原生動作執行：xdotool / wmctrl control/x11.rs、image/computer/Dockerfile -這已經比純「截圖 computer-use」產品完整一截。 - -現在實際主路徑（為什麼不順） -computer_observe → 截圖 +（瀏覽器時）CDP 元素 - →（否則）只有 wmctrl 視窗級元素 -computer_act → element id 先換成中心點 XY → xdotool 點像素 - -關鍵缺口： - -P0｜沒有 AT-SPI／無障礙樹 -Dockerfile 有 xdotool、wmctrl，沒有 at-spi2／python3-pyatspi 之類。 -原生「elements」在 window_list_command 只是 整窗標題，不是按鈕／輸入框。模型對 Thunar、對話框、面板仍要看圖或瞎點。 - -P0｜element 最終還是 XY -apply_element_targets 填的是中心座標，執行仍走 ComputerAction::Pointer → xdotool。 -視窗一動、動畫、縮放就歪。agent-desktop／Cua 是 對 element 做 AX／AT-SPI action，不是點中心像素。 - -P1｜ComputerAction 契約只有像素／鍵 -action.rs 沒有 ActByRef { id }、SetValue、A11yPress 這類語意動作。擴 CDP／a11y 都會卡在契約層。 - -P1｜控的是容器桌面，不是你 Mac -對「操控我電腦」若指本機 Finder／系統設定，現在這條 stack 接不到 Mac AX。那是另一條 provider（agent-desktop／Cua），不是再調 xdotool。 - -P2｜模型路由依賴 prompt -tools 已寫「Chromium 用 browser」，但若模型仍狂 computer_observe＋座標，體感就跟我差一截。缺硬性策略（例如：有 CDP page 時禁止對 browser 窗做 computer_act click）。 - -跟「我為什麼滑」的對照 -路徑 我（Grok Bot） LazyBoy feat/test -Shell／檔案 預設 有 -瀏覽器 DOM／element ref 有 CDP，方向對 -桌面原生 另有 a11y／委派；截圖是備援 截圖 + xdotool 仍是主幹 -本機 Mac 受限／核准 Shell 尚未做 Host provider -所以：你換好幾套「控制」若不順，多半都還停在 視覺／座標層；你 repo 差的是把 AT-SPI（容器內） 接到 observe／act，並讓 browser／shell 硬優先。 - -建議下一步（只做這 3 件） -容器加 AT-SPI：image 裝 at-spi；observe 回傳 button／textfield 樹（精簡 YAML／JSON + stable id）；act 對 id 做 DoAction／set text，失敗才 fallback XY。 -擴 ComputerAction／UiElement：kind: a11y|dom|window，ref 語意動作；computer_act 有 selector／a11y ref 時不要先轉座標。 -路由硬規則：Chromium 前台 → 強制 browser；能 shell 完成 → 禁止截圖迴圈；並加「連續同點無畫面變化 → 強制換策略」（你已有一點 stale-click 提示，可再硬一點）。 -本機 Mac 若也要滑：另開 HostComputerProvider（agent-desktop／Cua），別硬塞進現在的 X11 xdotool 路徑。 - -要的話我可以直接幫你開一張 P0 AT-SPI observe/act 的實作規格（改哪些檔、契約長怎樣），或直接在 feat/test 上開工。 \ No newline at end of file diff --git a/apps/web/package-lock.json b/apps/web/package-lock.json index 33a1897..c71e1cb 100644 --- a/apps/web/package-lock.json +++ b/apps/web/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "@blobatar/react": "^2.7.0", "@fontsource/huninn": "^5.3.0", + "@novnc/novnc": "1.7.0", "blobatar": "^2.7.0", "react": "^19.0.0", "react-dom": "^19.0.0", @@ -836,6 +837,12 @@ "node": "^22.20 || ^24.12 || >=25" } }, + "node_modules/@novnc/novnc": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@novnc/novnc/-/novnc-1.7.0.tgz", + "integrity": "sha512-ucEJOx4T2avIRCleodk7YobZj5O2Ga2AeLfQ69A/yjG9HHba2+PDgwSkN3FttrmG+70ZGx21sElNFouK13RzyA==", + "license": "MPL-2.0" + }, "node_modules/@rolldown/pluginutils": { "version": "1.0.0-beta.27", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", diff --git a/apps/web/package.json b/apps/web/package.json index c960204..555fd9e 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -11,6 +11,7 @@ "dependencies": { "@blobatar/react": "^2.7.0", "@fontsource/huninn": "^5.3.0", + "@novnc/novnc": "1.7.0", "blobatar": "^2.7.0", "react": "^19.0.0", "react-dom": "^19.0.0", diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index 81c9fba..8dc8b91 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -1,8 +1,7 @@ import { FormEvent, KeyboardEvent as ReactKeyboardEvent, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { BotIcon, Brain, ChevronDown, ChevronsRight, CircleHelp, ClipboardPaste, Computer, Download, Ellipsis, Info, LogOut, Megaphone, Paperclip, Pencil, Pin, Plug, Plus, RefreshCw, Settings, Smartphone, Sparkle, Square, Upload, Users, X } from "./animated-icons"; -import UseAnimations from "react-useanimations"; +import UseAnimations from "./use-animations"; import loading from "react-useanimations/lib/loading"; -import loading2 from "react-useanimations/lib/loading2"; import arrowUp from "react-useanimations/lib/arrowUp"; import bookmark from "react-useanimations/lib/bookmark"; import copy from "react-useanimations/lib/copy"; @@ -19,22 +18,32 @@ import { api, ApiError } from "./api"; import { Avatar, AvatarLookProvider, AvatarStack, BLOBATAR_BACKGROUNDS, BLOBATAR_EXPRESSIONS, BLOBATAR_SHAPES, DEFAULT_LOOK, persistBlobatarShape, readAvatarLooks, resolveBlobatarShape, writeAvatarLook, type AvatarBackground, type AvatarExpression, type AvatarLook } from "./avatar"; import { t, type MessageKey } from "./i18n"; import type { AvatarShape, Bot, ComputerMode, ComputerStatus, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, MessageFile, ModelProviderId, Playbook, PlaybookInput, PlaybookStep, Room, RoomMember, Session, TaughtSkill, WorkspaceSettings } from "./types"; +import { ScheduleEditor, ScheduleList, cronFromPreset, defaultCronPreset, presetFromCron, type CronPreset, type ScheduleItem } from "./schedule"; -const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,busyStep:null,waitingRunId:null,waitingSessionId:null,queuedRuns:0,display:null,profileMode:"per-bot",screenAvailable:false}; +const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,busyStep:null,usingComputer:false,waitingRunId:null,waitingSessionId:null,queuedRuns:0,display:null,profileMode:"per-bot",screenAvailable:false}; const SESSION_STORE="lazyboy.sessionByBot"; const PANE_STORE="lazyboy.rightPane"; const WORKSPACE_STORE="lazyboy.workspace"; -type RightPart="computer"|"memory"|"settings"|"plugins"; +type RightPart="computer"|"memory"|"settings"|"plugins"|"accounts"; type AccountDialog="phone"|"settings"|"model"|"about"|"help"|"feedback"|null; function readSessionStore():Record{try{const raw=localStorage.getItem(SESSION_STORE);return raw?JSON.parse(raw) as Record:{}}catch{return {}}} function writeSessionStore(botId:string,sessionId:string){const store=readSessionStore();store[botId]=sessionId;localStorage.setItem(SESSION_STORE,JSON.stringify(store))} -function readPaneStore():{collapsed:boolean;part:RightPart}{try{const raw=localStorage.getItem(PANE_STORE);if(!raw)return{collapsed:false,part:"computer"};const value=JSON.parse(raw) as {collapsed?:boolean;part?:string};return{collapsed:Boolean(value.collapsed),part:value.part==="memory"||value.part==="settings"||value.part==="plugins"?value.part:"computer"}}catch{return{collapsed:false,part:"computer"}}} +function readPaneStore():{collapsed:boolean;part:RightPart}{try{const raw=localStorage.getItem(PANE_STORE);if(!raw)return{collapsed:false,part:"computer"};const value=JSON.parse(raw) as {collapsed?:boolean;part?:string};return{collapsed:Boolean(value.collapsed),part:value.part==="memory"||value.part==="settings"||value.part==="plugins"||value.part==="accounts"?value.part:"computer"}}catch{return{collapsed:false,part:"computer"}}} type WorkspacePrefs={name:string;showHidden:boolean}; function readWorkspace():WorkspacePrefs{try{const raw=localStorage.getItem(WORKSPACE_STORE);if(!raw)return{name:t("localWorkspace"),showHidden:false};const value=JSON.parse(raw) as {name?:string;showHidden?:boolean};const name=value.name?.trim();return{name:name&&name!=="Local workspace"?name:t("localWorkspace"),showHidden:Boolean(value.showHidden)}}catch{return{name:t("localWorkspace"),showHidden:false}}} function WorkspaceAvatar({name}:{name:string}){const parts=name.trim().split(/\s+/).filter(Boolean);const initials=(parts.length>1?parts.map(part=>part[0]).join(""):parts[0]?.slice(0,2)||"LB").slice(0,2).toUpperCase();return } function modeLabel(mode:ComputerMode){return mode==="team"?t("sharedComputer"):t("privateComputer")} function stateLabel(state:ComputerStatus["state"]){return ({stopped:t("stopped"),booting:t("booting"),running:t("running"),suspended:t("suspended"),error:t("error")})[state]} +function isTransitionStep(step?:string|null){return step==="電腦啟動中"||step==="喚醒中"||step==="換手中"} +function hudLabel(computer:ComputerStatus,connecting:boolean,handingOff:boolean){ + const step=computer.busyStep||""; + if(computer.state==="booting"||step==="電腦啟動中")return t("hudBooting"); + if(computer.state==="suspended"||step==="喚醒中")return t("hudWaking"); + if(handingOff||step==="換手中")return t("hudHandoff"); + if(connecting)return t("hudConnecting"); + return null; +} function inboxTime(value:string|null){if(!value)return "";const date=new Date(value),now=new Date();if(date.toDateString()===now.toDateString())return new Intl.DateTimeFormat("zh-TW",{hour:"2-digit",minute:"2-digit",hour12:false}).format(date);const days=Math.floor((new Date(now.getFullYear(),now.getMonth(),now.getDate()).getTime()-new Date(date.getFullYear(),date.getMonth(),date.getDate()).getTime())/86400000);if(days<7)return new Intl.DateTimeFormat("zh-TW",{weekday:"long"}).format(date);return new Intl.DateTimeFormat("zh-TW",{month:"numeric",day:"numeric"}).format(date)} const ATTACH_MAX=4; const ATTACH_MAX_BYTES=10*1024*1024; @@ -45,6 +54,7 @@ function readAsBase64(file:File){return new Promise((resolve,reject)=>{c function formatBytes(size:number){if(size<1024)return `${size} B`;if(size<1024*1024)return `${Math.round(size/102.4)/10} KB`;return `${Math.round(size/104857.6)/10} MB`} function fileExt(name:string){const dot=name.lastIndexOf(".");const ext=dot>=0?name.slice(dot+1).replace(/[^a-z0-9]/gi,""):"";return (ext||"FILE").slice(0,4).toUpperCase()} function messageFiles(blocks:unknown):MessageFile[]{if(!Array.isArray(blocks))return [];return blocks.flatMap(block=>{if(!block||typeof block!=="object")return [];const value=block as {kind?:string;name?:string;mimeType?:string;size?:number};if(value.kind!=="file"&&value.kind!=="image")return [];return [{kind:value.kind,name:value.name||"file",mimeType:value.mimeType,size:value.size}]})} +function chipBlocks(blocks:unknown){if(!Array.isArray(blocks))return [] as {kind:string;site?:string;why?:string;name?:string;human?:string}[];return blocks.flatMap(block=>{if(!block||typeof block!=="object")return [];const value=block as {kind?:string;site?:string;why?:string;name?:string;human?:string};if(value.kind==="login"||value.kind==="schedule"||value.kind==="scheduleRun")return [value];return []})} function isAutoAttachCaption(body:string,files:MessageFile[]){const text=body.trim();if(!files.length)return false;if(!text)return true;return files.some(file=>text===file.name||text===`附件 ${file.name}`||text===t("attachedFile",{name:file.name}))} function FileCard({file,preview,onRemove}:{file:{name:string;size?:number};preview?:string|null;onRemove?:()=>void}){const ext=fileExt(file.name);return
{preview?:}
{file.name}{typeof file.size==="number"?formatBytes(file.size):ext}
{onRemove&&}
} function clientNonce(){ @@ -79,9 +89,14 @@ export function App(){ const [roomToDelete,setRoomToDelete]=useState(null); const [mcpServers,setMcpServers]=useState([]); const [accountOpen,setAccountOpen]=useState(false); const [accountDialog,setAccountDialog]=useState(null); const [skills,setSkills]=useState([]); const [plusOpen,setPlusOpen]=useState(false); const [skillQuery,setSkillQuery]=useState(""); const [teachOpen,setTeachOpen]=useState(false); const [editingSkillId,setEditingSkillId]=useState(null); + const [schedules,setSchedules]=useState([]); const [scheduleDraft,setScheduleDraft]=useState<{name:string;instructions:string;enabled:boolean;preset:CronPreset;id?:string;timezone?:string;threadId?:string|null}|null>(null); + const [scheduleError,setScheduleError]=useState(null); const [scheduleSaving,setScheduleSaving]=useState(false); const [runningScheduleId,setRunningScheduleId]=useState(null); const [workspaceName,setWorkspaceName]=useState(workspaceStart.name); const [looks,setLooks]=useState(readAvatarLooks); const sendingRef=useRef(false); const refreshSeqRef=useRef(0); const importRef=useRef(null); const attachRef=useRef(null); + const desktopFrameRef=useRef(null); + const holderRef=useRef(computer.controlHolder); const paneBotRef=useRef(null); const skipHandoffRef=useRef(true); + const [desktopReady,setDesktopReady]=useState(false); const [handingOff,setHandingOff]=useState(false); const [pendingFiles,setPendingFiles]=useState([]); const messageEndRef=useRef(null); const sentHistoryRef=useRef([]); const historyIndexRef=useRef(null); const historyDraftRef=useRef(""); @@ -92,7 +107,10 @@ export function App(){ const sections=useMemo(()=>{const map=new Map();for(const bot of filtered){const key=bot.pinned?t("pinned"):bot.groupName||t("agentGroup");map.set(key,[...(map.get(key)||[]),bot])}return [...map.entries()]},[filtered]); const paneBotId=busyMembers[0]?.id||activeRoom?.members[0]?.id||activeId; const paneBot=bots.find(bot=>bot.id===paneBotId)||active; - const workingMembers=activeRoom?busyMembers:active&&computer.busySessionId===activeSessionId?[{id:active.id,name:active.name,avatarColor:active.avatarColor,avatarShape:active.avatarShape}]:[]; + const currentPaneRef=useRef(paneBotId);currentPaneRef.current=paneBotId; + const pasteQueueRef=useRef>(Promise.resolve()); + const [clipboardStatus,setClipboardStatus]=useState(""); + const workingMembers=activeRoom?busyMembers:active&&activeSessionId&&computer.busySessionId===activeSessionId?[{id:active.id,name:active.name,avatarColor:active.avatarColor,avatarShape:active.avatarShape}]:[]; const lastMessageId=messages[messages.length-1]?.id||""; // The bot is parked in waiting_takeover: nothing moves (including queued // messages) until the human releases the screen, so say so loudly. @@ -131,14 +149,40 @@ export function App(){ useEffect(()=>{if(!plusOpen)setSkillQuery("")},[plusOpen]); useEffect(()=>{messageEndRef.current?.scrollIntoView({block:"end",behavior:"smooth"})},[activeSessionId,lastMessageId,workingMembers.length,pausedForUser]); useEffect(()=>{if(!activeSessionId||(!activeId&&!activeRoomId)){refreshSeqRef.current+=1;setMessages([]);setScreenUrl(null);if(!activeId&&!activeRoomId)setComputer(blankComputer);return}setScreenUrl(null);refresh().catch(e=>setError(e.message));const timer=setInterval(()=>{refresh().catch(()=>{});const beat=roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||activeId;if(beat)api(`/api/computer/${beat}/heartbeat`,{method:"POST",body:"{}"}).catch(()=>{})},2000);return()=>{clearInterval(timer);refreshSeqRef.current+=1}},[activeId,activeRoomId,activeSessionId,refresh]); - useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard.writeText(text).catch(()=>{})}};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)}); - useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.data?.type!=="lazyboy-request-control"||!paneBotId)return;void action(()=>api(`/api/computer/${paneBotId}/takeover`,{method:"POST",body:"{}"}))};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)},[paneBotId]); + useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard?.writeText(text).then(()=>setClipboardStatus("剪貼簿已同步")).catch(()=>setClipboardStatus("瀏覽器未允許同步，請按複製按鈕"))}if(event.data.type==="lazyboy-copy-request"&&computer.controlHolder==="user")void copySelection();if(event.data.type==="lazyboy-paste-text"&&typeof event.data.text==="string")pasteText(event.data.text);if(event.data.type==="lazyboy-paste-request"&&computer.controlHolder==="user")setClipboardOpen(true);if(event.data.type==="lazyboy-desktop-ready")setDesktopReady(true);if(event.data.type==="lazyboy-desktop-lost")setDesktopReady(false)};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)}); + useEffect(()=>{setDesktopReady(false)},[screenUrl,paneBotId]); + useEffect(()=>{if(skipHandoffRef.current){skipHandoffRef.current=false;holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId;return}if((holderRef.current!==computer.controlHolder||paneBotRef.current!==paneBotId)&&computer.state==="running")setHandingOff(true);holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId},[computer.controlHolder,paneBotId,computer.state]); + useEffect(()=>{if(!handingOff)return;const timer=setTimeout(()=>setHandingOff(false),1600);return()=>clearTimeout(timer)},[handingOff]); + useEffect(()=>{const frame=desktopFrameRef.current;if(!frame?.contentWindow||!screenUrl)return;frame.contentWindow.postMessage({type:"lazyboy-view-only",viewOnly:computer.controlHolder!=="user"},location.origin)},[computer.controlHolder,screenUrl,desktopReady]); + useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||event.data?.type!=="lazyboy-request-control"||!paneBotId)return;void action(()=>api(`/api/computer/${paneBotId}/takeover`,{method:"POST",body:"{}"}))};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)},[paneBotId]); useEffect(()=>{const close=(event:MouseEvent)=>{const target=event.target;if(target instanceof Element&&target.closest(".create-menu-wrap,.account-wrap,.session-picker,.context-menu,.plus-menu-wrap"))return;setContext(null);setRoomContext(null);setSessionMenuOpen(false);setAccountOpen(false);setCreateMenuOpen(false);setPlusOpen(false)};window.addEventListener("click",close);return()=>window.removeEventListener("click",close)},[]); useEffect(()=>{const onKey=(event:KeyboardEvent)=>{if(event.key!=="Escape")return;setAccountOpen(false);setAccountDialog(null);setCreateMenuOpen(false);setPlusOpen(false);setTeachOpen(false);setEditingSkillId(null);setSessionMenuOpen(false);setContext(null);setRoomContext(null)};window.addEventListener("keydown",onKey);return()=>window.removeEventListener("keydown",onKey)},[]); useEffect(()=>{localStorage.setItem(WORKSPACE_STORE,JSON.stringify({name:workspaceName,showHidden}))},[workspaceName,showHidden]); useEffect(()=>{if(!sessionStoreKey||!activeSessionId)return;if(!sessions.some(session=>session.id===activeSessionId))return;if(!activeRoomId&&!sessions.some(session=>session.id===activeSessionId&&session.botId===activeId))return;writeSessionStore(sessionStoreKey,activeSessionId)},[sessionStoreKey,activeId,activeRoomId,activeSessionId,sessions]); useEffect(()=>{localStorage.setItem(PANE_STORE,JSON.stringify({collapsed:rightCollapsed,part:rightPart}))},[rightCollapsed,rightPart]); + useEffect(()=>{if(!paneBotId){setSchedules([]);return}api(`/api/bots/${paneBotId}/schedules`).then(setSchedules).catch(()=>setSchedules([]))},[paneBotId]); + useEffect(()=>{if(computer.takeoverRequested){setRightPart("computer");setRightCollapsed(false)}},[computer.takeoverRequested]); function openPane(part:RightPart){setRightPart(part);setRightCollapsed(false)} + async function openLoginScreen(){ + const id=paneBot?.id||active?.id;if(!id)return; + setRightPart("computer");setRightCollapsed(false);setComputerOpen(true); + await action(async()=>{ + try{await api(`/api/computer/${id}/boot`,{method:"POST",body:"{}"})}catch{/* already up */} + await api(`/api/computer/${id}/takeover`,{method:"POST",body:"{}"}).catch(()=>{}); + }); + } + async function reloadSchedules(){if(!paneBotId)return;setSchedules(await api(`/api/bots/${paneBotId}/schedules`))} + async function saveScheduleDraft(){ + if(!paneBot||!scheduleDraft)return; + setScheduleSaving(true);setScheduleError(null); + try{ + const body={name:scheduleDraft.name.trim(),cron:cronFromPreset(scheduleDraft.preset),instructions:scheduleDraft.instructions.trim(),timezone:scheduleDraft.timezone||Intl.DateTimeFormat().resolvedOptions().timeZone||"Asia/Taipei",enabled:scheduleDraft.enabled,threadId:scheduleDraft.id?scheduleDraft.threadId:activeRoomId?undefined:activeSessionId}; + if(scheduleDraft.id)await api(`/api/schedules/${scheduleDraft.id}`,{method:"PATCH",body:JSON.stringify(body)}); + else await api(`/api/bots/${paneBot.id}/schedules`,{method:"POST",body:JSON.stringify(body)}); + setScheduleDraft(null);await reloadSchedules(); + }catch(e){setScheduleError(e instanceof Error?e.message:t("operationFailed"))} + finally{setScheduleSaving(false)} + } const sessionBusy=workingMembers.length>0; const otherSessionBusy=Boolean(computer.busyBotName&&!sessionBusy); @@ -166,7 +210,23 @@ export function App(){ async function clearSession(){if(!activeSessionId)return;setClearOpen(false);setSessionMenuOpen(false);await action(async()=>{await api(`/api/sessions/${activeSessionId}/messages`,{method:"DELETE"});setMessages([]);await loadSessions()})} async function deleteSession(id:string){if(!sessionsPath||!sessionStoreKey)return;setSessionMenuOpen(false);setBusy(true);setError(null);try{await api(`/api/sessions/${id}`,{method:"DELETE"});const next=await api(sessionsPath);setSessions(next);const pick=id===activeSessionId||!next.some(session=>session.id===activeSessionId)?next[0]?.id||null:activeSessionId;setActiveSessionId(pick);if(pick)writeSessionStore(sessionStoreKey,pick)}catch(e){setError(e instanceof Error?e.message:t("operationFailed"))}finally{setBusy(false)}} async function rememberMessage(message:Message){const botId=message.speakerBotId||active?.id||activeRoom?.members[0]?.id;if(!botId||!message.body.trim())return;try{await api(`/api/bots/${botId}/memories`,{method:"POST",body:JSON.stringify({content:message.body,sessionId:activeSessionId})});setRemembered(current=>({...current,[message.id]:true}))}catch(e){setError(e instanceof Error?e.message:t("rememberFailed"))}} - async function pasteClipboard(){try{const text=await navigator.clipboard.readText();document.querySelectorAll(".desktop-frame").forEach(frame=>frame.contentWindow?.postMessage({type:"lazyboy-host-clipboard",text},location.origin))}catch{setClipboardOpen(true)}} + function pasteText(text:string){ + const botId=paneBotId; + if(!botId||computer.controlHolder!=="user")return; + pasteQueueRef.current=pasteQueueRef.current.catch(()=>{}).then(async()=>{ + if(currentPaneRef.current!==botId)return; + await api(`/api/computer/${botId}/input`,{method:"POST",body:JSON.stringify({kind:"clipboard",text})}); + if(currentPaneRef.current===botId)setClipboardStatus("已貼上文字"); + }).catch(()=>setClipboardStatus("貼上失敗，請確認已接管電腦後重試")); + } + async function copySelection(){ + const botId=paneBotId;if(!botId)return; + try{const result=await api<{text:string}>(`/api/computer/${botId}/input`,{method:"POST",body:JSON.stringify({kind:"copy"})}); + if(currentPaneRef.current!==botId)return; + setDesktopClipboard(result.text);await navigator.clipboard.writeText(result.text);setClipboardStatus("已複製選取文字"); + }catch{setClipboardStatus("複製未同步；請按複製按鈕，或在遠端使用 Ctrl+Shift+C");} + } + async function pasteClipboard(){try{pasteText(await navigator.clipboard.readText())}catch{setClipboardOpen(true)}} async function copyClipboard(){try{await navigator.clipboard.writeText(desktopClipboard)}catch{setError(t("clipboardWriteBlocked"))}} async function inbox(bot:Bot,actionName:string,groupName?:string|null){await api(`/api/bots/${bot.id}/inbox`,{method:"POST",body:JSON.stringify({action:actionName,groupName})});await loadBots()} function openBot(bot:Bot){setMobileNav(false);setSessionMenuOpen(false);if(bot.unreadCount>0)void inbox(bot,"read");if(bot.id===activeId&&!activeRoomId){if(!activeSessionId){const stored=readSessionStore()[bot.id];if(stored)setActiveSessionId(stored);else void loadSessions()}return}setMessages([]);setActiveSessionId(null);setActiveRoomId(null);setBusyMembers([]);setActiveId(bot.id)} @@ -174,13 +234,27 @@ export function App(){ async function deleteRoom(room:Room){setRoomToDelete(null);await action(async()=>{await api(`/api/rooms/${room.id}`,{method:"DELETE"});if(activeRoomId===room.id){setActiveRoomId(null);setActiveSessionId(null);setMessages([]);setBusyMembers([])}await loadBots()})} function openAccount(dialog:AccountDialog){setAccountOpen(false);setAccountDialog(dialog)} async function logout(){setAccountOpen(false);await api("/api/session",{method:"DELETE",body:"{}"}).catch(()=>{});setBots([]);setRooms([]);setMcpServers([]);setActiveId(null);setActiveRoomId(null);setAuthRequired(true)} - // Re-mount noVNC after every status transition so a freshly booted Docker - // display cannot remain stuck on the previous disconnected iframe. - const startBoot=async()=>{setScreenUrl(null);setComputer(current=>({...current,state:"booting"}));await api(`/api/computer/${paneBot?.id||active?.id}/boot`,{method:"POST",body:"{}"})}; - const restartComputer=async()=>{setScreenUrl(null);setComputer(current=>({...current,state:"booting"}));await api(`/api/computer/${paneBot?.id||active?.id}/restart`,{method:"POST",body:"{}"})}; - const frame=screenUrl?