feat/cua-driver-poc #7
22
.env.example
22
.env.example
|
|
@ -7,6 +7,8 @@ SANDBOX_SUPERVISOR_TOKEN=
|
|||
# Keep this key stable when rotating the app login token.
|
||||
LAZYBOY_VAULT_KEY=
|
||||
POSTGRES_PASSWORD=lazyboy
|
||||
# 127.0.0.1 = 只有本機；0.0.0.0 = 開放區網（需 LAZYBOY_APP_TOKEN >= 32 字元）；
|
||||
# 也可填單一網卡的 IP，把監聽限制在那個介面。
|
||||
LAZYBOY_BIND_IP=127.0.0.1
|
||||
SANDBOX_PROVIDER=docker
|
||||
DATABASE_URL=postgres://lazyboy:lazyboy@127.0.0.1:5434/lazyboy
|
||||
|
|
@ -19,9 +21,23 @@ LAZYBOY_COMPUTER_CPUS=2
|
|||
LAZYBOY_COMPUTER_PIDS=2048
|
||||
# Only affects the Agent desktop container. Disabled by default.
|
||||
LAZYBOY_COMPUTER_SUDO=false
|
||||
# Computer-control backend inside each desktop container. Rebuild/recreate
|
||||
# computers after upgrading. Cua is the only supported computer controller.
|
||||
LAZYBOY_COMPUTER_DRIVER=cua
|
||||
# Linux only (optional): point this at the host's LXCFS root to make htop/free
|
||||
# report the per-Agent cgroup quota. Leave the default empty directory on macOS.
|
||||
LAZYBOY_LXCFS_ROOT=./data/lxcfs
|
||||
# 任務長度政策：不再用固定輪數掐掉任務。正常任務一路做到驗證完成，只有
|
||||
# 真的鬼打牆（同一個動作重複、同一個錯誤一直失敗、很久沒有新的成功）才會被
|
||||
# 提示、接著暫停等你決定；最後兩個是防迴圈失控烧 token 的保險絲，不是額度。
|
||||
# soft turns：第 60 輪起，之後每 soft every 輪請模型自我交代「已完成/還缺/下一步」
|
||||
# cap turns / hard minutes：最後防火牆，正常任務不該碰到
|
||||
LAZYBOY_RUN_SOFT_TURNS=60
|
||||
LAZYBOY_RUN_SOFT_EVERY=120
|
||||
LAZYBOY_RUN_CAP_TURNS=1000
|
||||
LAZYBOY_RUN_SOFT_MINUTES=75
|
||||
LAZYBOY_RUN_HARD_MINUTES=240
|
||||
|
||||
LAZYBOY_MEMORY_ENABLED=true
|
||||
LAZYBOY_MEMORY_MODEL_CACHE=./data/fastembed
|
||||
LAZYBOY_MEMORY_TOP_K=8
|
||||
|
|
@ -31,6 +47,12 @@ LAZYBOY_MEMORY_BYTE_BUDGET=6000
|
|||
LAZYBOY_EVENT_RETENTION_DAYS=30
|
||||
LAZYBOY_CHECKPOINT_RETENTION_DAYS=7
|
||||
LAZYBOY_RUN_RETENTION_DAYS=90
|
||||
# Per-run trace shown when hovering the thinking avatar (round, tool results, errors).
|
||||
LAZYBOY_RUN_ACTIVITY_RETENTION_DAYS=7
|
||||
LAZYBOY_RECORDING_RETENTION_DAYS=30
|
||||
LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS=90
|
||||
LAZYBOY_DB_WARN_MB=1024
|
||||
|
||||
# Docker 網路名稱：API 與 Agent 電腦的 noVNC 透過它相通（容器內用，不對外）。
|
||||
# 同時跑多組 LazyBoy 時改這個名字避免相撞。
|
||||
LAZYBOY_SCREEN_NETWORK=lazyboy_screen
|
||||
|
|
|
|||
|
|
@ -1889,7 +1889,6 @@ name = "lazyboy-api"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64 0.22.1",
|
||||
"cap-std",
|
||||
|
|
@ -1900,9 +1899,7 @@ dependencies = [
|
|||
"fastembed",
|
||||
"futures-util",
|
||||
"hex",
|
||||
"hmac",
|
||||
"http",
|
||||
"http-body-util",
|
||||
"lazyboy-contracts",
|
||||
"lazyboy-control",
|
||||
"lazyboy-harness",
|
||||
|
|
@ -1915,7 +1912,6 @@ dependencies = [
|
|||
"serde_json",
|
||||
"sha2",
|
||||
"sqlx",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-tungstenite 0.26.2",
|
||||
"tower-http",
|
||||
|
|
@ -1940,6 +1936,7 @@ name = "lazyboy-control"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"hex",
|
||||
"image",
|
||||
|
|
@ -1948,6 +1945,8 @@ dependencies = [
|
|||
"serde_json",
|
||||
"sha2",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1955,12 +1954,8 @@ name = "lazyboy-controld"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"base64 0.22.1",
|
||||
"lazyboy-contracts",
|
||||
"lazyboy-control",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
|
|
@ -1979,7 +1974,6 @@ dependencies = [
|
|||
"rig-core",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
|
|
@ -1992,39 +1986,30 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"hex",
|
||||
"lazyboy-contracts",
|
||||
"lazyboy-control",
|
||||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazyboy-supervisor"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64 0.22.1",
|
||||
"bollard",
|
||||
"futures-util",
|
||||
"hex",
|
||||
"hmac",
|
||||
"lazyboy-contracts",
|
||||
"lazyboy-control",
|
||||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
20
Cargo.toml
20
Cargo.toml
|
|
@ -12,10 +12,28 @@ members = [
|
|||
|
||||
[workspace.package]
|
||||
edition = "2024"
|
||||
rust-version = "1.98"
|
||||
version = "0.1.0"
|
||||
license = "Apache-2.0"
|
||||
publish = false
|
||||
|
||||
# --- Lint policy -----------------------------------------------------------
|
||||
# A single place decides how the whole workspace is linted; every crate opts in
|
||||
# with `[lints] workspace = true`. `make lint` runs clippy with -D warnings, so a
|
||||
# new warning has to be fixed (or relaxed at the call site with a reason) before
|
||||
# it reaches main. Thresholds live in clippy.toml, which cargo-clippy only reads
|
||||
# from the directory it was started in - keep running it at the workspace root.
|
||||
[workspace.lints.rust]
|
||||
# Safety-relevant code (the vault, the docker socket) has to stay explicit.
|
||||
unsafe_code = "warn"
|
||||
unused_must_use = "deny"
|
||||
unused_crate_dependencies = "warn"
|
||||
|
||||
[workspace.lints.clippy]
|
||||
# Debug leftovers and placeholder implementations must not reach a branch.
|
||||
dbg_macro = "warn"
|
||||
todo = "warn"
|
||||
|
||||
[profile.release]
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
|
|
@ -34,7 +52,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock", "serd
|
|||
image = { version = "0.25", default-features = false, features = ["jpeg", "png"] }
|
||||
rig-core = "0.42"
|
||||
async-trait = "0.1"
|
||||
tokio = { version = "1", features = ["macros", "rt-multi-thread", "process", "io-util", "fs", "signal", "time"] }
|
||||
tokio = { version = "1", features = ["macros", "rt-multi-thread", "process", "io-util", "fs", "signal", "time", "sync"] }
|
||||
axum = { version = "0.8", features = ["ws"] }
|
||||
tower-http = { version = "0.6", features = ["cors", "trace", "fs"] }
|
||||
tracing = "0.1"
|
||||
|
|
|
|||
68
Makefile
68
Makefile
|
|
@ -8,12 +8,17 @@ COMPOSE ?= docker compose
|
|||
COMPUTER_IMAGE ?= lazyboy/computer:local
|
||||
WEB_DIR ?= apps/web
|
||||
DATA_DIR ?= ./data
|
||||
BUILDX_BUILDER ?= lazyboy
|
||||
# PUSH=1 publishes the manifest list instead of writing an OCI archive.
|
||||
MULTI_FLAGS ?=
|
||||
$(if $(filter 1,$(PUSH)),$(eval MULTI_FLAGS := --push))
|
||||
|
||||
.PHONY: help env env-force \
|
||||
up logs ps health down purge \
|
||||
computer postgres postgres-down \
|
||||
computer computer-multi images-multi postgres postgres-down pg-collation \
|
||||
cua-smoke \
|
||||
build build-api build-supervisor build-controld \
|
||||
fmt clippy test clean \
|
||||
fmt fmt-check clippy lint audit test clean \
|
||||
web \
|
||||
dev dev-supervisor dev-api
|
||||
|
||||
|
|
@ -31,8 +36,12 @@ help: ## Show this help
|
|||
@echo ""
|
||||
@echo " Individual pieces:"
|
||||
@echo " make computer Build the heavy Debian desktop image (lazyboy/computer:local)"
|
||||
@echo " make computer-multi Cross-build the desktop image for amd64 + arm64 (PUSH=1 to publish)"
|
||||
@echo " make images-multi Cross-build desktop + api + supervisor for amd64 + arm64"
|
||||
@echo " make cua-smoke Run Cua Driver smoke test in a disposable desktop container"
|
||||
@echo " make postgres Start only postgres (127.0.0.1:5434) and wait for ready"
|
||||
@echo " make postgres-down Stop postgres"
|
||||
@echo " make pg-collation Repair a Postgres collation version mismatch (see docs)"
|
||||
@echo ""
|
||||
@echo " Local dev (postgres in Docker, Rust services on the host):"
|
||||
@echo " make dev Prep .env + postgres + computer image, then print run steps"
|
||||
|
|
@ -43,8 +52,11 @@ help: ## Show this help
|
|||
@echo " make build cargo build --release (whole workspace)"
|
||||
@echo " make build-api cargo build --release -p lazyboy-api"
|
||||
@echo " make fmt cargo fmt --all"
|
||||
@echo " make clippy cargo clippy (deny warnings)"
|
||||
@echo " make test cargo test --workspace"
|
||||
@echo " make fmt-check Report files rustfmt would change (legacy drift exists)"
|
||||
@echo " make clippy cargo clippy (deny warnings, reads clippy.toml)"
|
||||
@echo " make lint The Rust gate: clippy with -D warnings"
|
||||
@echo " make audit cargo deny: RustSec advisories, licenses, sources"
|
||||
@echo " make test cargo test --workspace (DB tests need: make postgres)"
|
||||
@echo " make web Build the frontend in $(WEB_DIR) (needs node/npm)"
|
||||
@echo " make clean cargo clean"
|
||||
@echo ""
|
||||
|
|
@ -87,7 +99,26 @@ purge: ## Stop containers and delete the postgres data volume
|
|||
# --- Individual pieces -----------------------------------------------------
|
||||
|
||||
computer: ## Build the Debian desktop image used to spawn bot computers
|
||||
docker build -f image/computer/Dockerfile -t $(COMPUTER_IMAGE) .
|
||||
./scripts/build-image.sh --tag $(COMPUTER_IMAGE)
|
||||
|
||||
# Cross-builds every supported CPU architecture into one manifest list. Needs a
|
||||
# docker-container builder + QEMU binfmt; both are bootstrapped by the script.
|
||||
# PUSH=1 publishes to a registry, otherwise an OCI archive is written.
|
||||
computer-multi: ## Cross-build the desktop image for all CPU architectures
|
||||
./scripts/build-image.sh --file image/computer/Dockerfile --multi $(MULTI_FLAGS)
|
||||
|
||||
# The api and supervisor images carry per-architecture binaries as well (ONNX
|
||||
# Runtime, node, the distroless libc), so they get the same treatment as the
|
||||
# desktop image instead of only ever existing for the build host.
|
||||
images-multi: ## Cross-build every shipped image for all CPU architectures
|
||||
@for dockerfile in image/computer/Dockerfile image/supervisor/Dockerfile \
|
||||
image/api/Dockerfile; do \
|
||||
echo "== $$dockerfile"; \
|
||||
./scripts/build-image.sh --file "$$dockerfile" --multi $(MULTI_FLAGS) || exit 1; \
|
||||
done
|
||||
|
||||
cua-smoke: computer ## Run the Cua Driver smoke test inside a disposable desktop container
|
||||
./scripts/cua-smoke-test.sh --docker --image $(COMPUTER_IMAGE)
|
||||
|
||||
postgres: ## Start only postgres and wait until it is ready
|
||||
$(COMPOSE) -f docker-compose.yml -f docker-compose.dev.yml up -d postgres
|
||||
|
|
@ -97,6 +128,14 @@ postgres: ## Start only postgres and wait until it is ready
|
|||
postgres-down: ## Stop postgres
|
||||
$(COMPOSE) down postgres
|
||||
|
||||
# A pgvector image rebuilt on another glibc leaves every database recording the old
|
||||
# collation version; Postgres then refuses CREATE DATABASE and `cargo test` hangs on
|
||||
# PoolTimedOut. Stop the api first, then reindex + refresh each database in place.
|
||||
pg-collation: ## Repair a Postgres collation version mismatch after an image update
|
||||
$(COMPOSE) exec -T postgres psql -X -v ON_ERROR_STOP=1 -U lazyboy -d template1 -c "REINDEX DATABASE template1;" -c "ALTER DATABASE template1 REFRESH COLLATION VERSION;"
|
||||
$(COMPOSE) exec -T postgres psql -X -v ON_ERROR_STOP=1 -U lazyboy -d postgres -c "REINDEX DATABASE postgres;" -c "ALTER DATABASE postgres REFRESH COLLATION VERSION;"
|
||||
$(COMPOSE) exec -T postgres psql -X -v ON_ERROR_STOP=1 -U lazyboy -d lazyboy -c "REINDEX DATABASE lazyboy;" -c "ALTER DATABASE lazyboy REFRESH COLLATION VERSION;"
|
||||
|
||||
# --- Rust / web ------------------------------------------------------------
|
||||
|
||||
build: ## Release build of the whole workspace
|
||||
|
|
@ -114,8 +153,23 @@ build-controld: ## Release build of the controld binary
|
|||
fmt: ## Format all Rust code
|
||||
cargo fmt --all
|
||||
|
||||
clippy: ## Run clippy, denying warnings
|
||||
cargo clippy --all-targets -- -D warnings
|
||||
fmt-check: ## Check formatting without touching files
|
||||
cargo fmt --all --check
|
||||
|
||||
# Lint policy lives in [workspace.lints] in the root Cargo.toml; the thresholds
|
||||
# (e.g. too-many-arguments-threshold) live in clippy.toml, which cargo-clippy
|
||||
# only reads from the directory it is started in - keep running this at the root.
|
||||
clippy: ## Run clippy over the workspace, denying warnings
|
||||
cargo clippy --workspace --all-targets -- -D warnings
|
||||
|
||||
lint: clippy ## The Rust quality gate used by CI
|
||||
|
||||
# cargo-deny is a separate CLI: cargo install --locked cargo-deny
|
||||
audit: ## Supply-chain check (RustSec advisories, licenses, dependency sources)
|
||||
@command -v cargo-deny >/dev/null 2>&1 || { \
|
||||
echo "cargo-deny is not installed: cargo install --locked cargo-deny"; exit 1; }
|
||||
@echo "(advisories need the RustSec advisory DB, cloned on first run)"
|
||||
cargo deny check
|
||||
|
||||
test: ## Run the test suite
|
||||
cargo test --workspace
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@ The guides below are currently in Traditional Chinese.
|
|||
| [Architecture](./docs/architecture.md) | Task flow, system architecture, computer lifecycle |
|
||||
| [Interactive diagram](./docs/workflow.html) | Zoomable, searchable HTML chart; download and open |
|
||||
| [Operations](./docs/operations.md) | Resources, env vars, security, site checks, sudo |
|
||||
| [Agent experience](./docs/agent-experience.md) | Turn limits, persistent terminal, live chat |
|
||||
| [Development](./docs/development.md) | Local dev, checks and tests, directory layout |
|
||||
| [Env example](./.env.example) | Environment variables and defaults |
|
||||
|
||||
|
|
|
|||
|
|
@ -111,6 +111,7 @@ npm run dev
|
|||
| [架構與流程](./docs/architecture.md) | 任務流程圖、系統架構、電腦生命週期狀態機 |
|
||||
| [互動流程圖](./docs/workflow.html) | 可縮放、搜尋的 HTML 圖表；下載後開啟 |
|
||||
| [部署與操作](./docs/operations.md) | 資源、環境變數、安全設定、網站驗證、sudo |
|
||||
| [AI 使用體驗](./docs/agent-experience.md) | 輪次政策、持久終端機、聊天即時推送 |
|
||||
| [開發指南](./docs/development.md) | 本機開發、檢查與測試、目錄結構 |
|
||||
| [設定範例](./.env.example) | 環境變數與預設值 |
|
||||
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -175,7 +175,7 @@ export function CallOverlay({
|
|||
<button type="button" className="call-hangup" onClick={hangUp}>{t("hangUp")}</button>
|
||||
</div>
|
||||
{showTakeover ? (
|
||||
<button type="button" className="primary call-takeover" onClick={onTakeOver}>{t("takeOverNow")}</button>
|
||||
<button type="button" className="primary call-takeover" onClick={onTakeOver}>{t("loginOpenScreen")}</button>
|
||||
) : null}
|
||||
<p className="call-hint">{t("callShortcuts")}</p>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
.composer textarea{flex:1;min-height:42px;max-height:140px;resize:none;border:0;outline:0;background:transparent;color:var(--ink);padding:11px 4px}
|
||||
.composer-plus{background:transparent;color:var(--muted)}
|
||||
.composer svg{width:18px}
|
||||
.messages{min-width:0;padding-bottom:128px;padding-left:var(--chat-gutter);padding-right:var(--chat-gutter)}
|
||||
.messages{min-width:0;padding-bottom:24px;padding-left:var(--chat-gutter);padding-right:var(--chat-gutter)}
|
||||
.message{box-sizing:border-box;width:var(--chat-col);max-width:none;margin-inline:auto;overflow-wrap:anywhere}
|
||||
.message>span{max-width:82%}
|
||||
.thinking-row{display:flex;align-items:center;gap:10px;width:min(760px,100%);margin:0}
|
||||
|
|
@ -24,7 +24,7 @@
|
|||
.thinking-dots i{width:5px;height:5px;border-radius:50%;background:var(--muted);animation:thinking-dot 1.15s ease-in-out infinite}
|
||||
.thinking-dots i:nth-child(2){animation-delay:.16s}
|
||||
.thinking-dots i:nth-child(3){animation-delay:.32s}
|
||||
.composer-dock{position:absolute;left:0;right:0;bottom:0;z-index:6;display:flex;flex-direction:column;align-items:center;gap:20px;padding:28px var(--chat-gutter) 20px;background:var(--main);pointer-events:none}
|
||||
.composer-dock{position:relative;z-index:6;display:flex;flex-direction:column;align-items:center;gap:20px;padding:28px var(--chat-gutter) 20px;background:var(--main);pointer-events:none;flex:none;width:100%}
|
||||
.composer-dock:before{content:"";position:absolute;left:0;right:0;bottom:100%;height:32px;background:linear-gradient(180deg,transparent,var(--main));pointer-events:none}
|
||||
.composer-dock>*{box-sizing:border-box;pointer-events:auto;width:var(--chat-col);max-width:760px;margin-inline:auto}
|
||||
.composer{position:relative;left:auto;right:auto;bottom:auto;width:var(--chat-col);min-height:62px;align-items:center;padding:9px 10px;transform:none}
|
||||
|
|
@ -73,3 +73,7 @@
|
|||
|
||||
.slash-suggestions button[aria-selected="true"]{background:rgba(255,255,255,.08)}
|
||||
.slash-suggestions small{flex-shrink:0;color:var(--muted);font-size:11px}
|
||||
|
||||
.message{padding-bottom:21px}
|
||||
.message-time{position:absolute;bottom:0;left:0;font-size:11px;line-height:17px;color:var(--muted);font-variant-numeric:tabular-nums}
|
||||
.message.user .message-time{left:auto;right:28px}
|
||||
|
|
|
|||
|
|
@ -28,9 +28,13 @@
|
|||
.computer-part .preview .desktop-frame{display:block;aspect-ratio:16 / 10}
|
||||
.overlay-screen .overlay-desktop{position:relative;width:min(100%,1440px);height:100%}
|
||||
.overlay-screen .overlay-desktop>.desktop-frame,.overlay-screen .overlay-desktop>.empty-computer{width:100%;height:100%;border:1px solid var(--border);border-radius:14px;overflow:hidden;background:var(--main)}
|
||||
.computer-hud{position:absolute;top:10px;left:10px;z-index:3;display:grid;justify-items:center;gap:6px;padding:10px 12px 8px;border-radius:16px;background:rgba(10,10,12,.72);backdrop-filter:blur(8px);pointer-events:none;box-shadow:0 10px 30px rgba(0,0,0,.35)}
|
||||
.computer-hud .avatar.blobatar.thinking{animation:computer-hud-bob 1.6s ease-in-out infinite,avatar-rainbow-glow 1.8s linear infinite}
|
||||
.computer-hud-label{font-size:12px;letter-spacing:.02em;line-height:1.3;white-space:nowrap;background:linear-gradient(90deg,#7a8088 0%,#7a8088 28%,#fff 50%,#7a8088 72%,#7a8088 100%);background-size:220% 100%;-webkit-background-clip:text;background-clip:text;color:transparent;animation:working-shimmer 1.35s linear infinite}
|
||||
/* The waiting veil sits over the remote pixels, not beside them: while the
|
||||
desktop is booting, waking, or changing hands, the signal mascot breathes
|
||||
under its amber orbit and the label shimmers. It never takes a click, so the
|
||||
mouse is live the instant the veil lifts, and it fades rather than blinks. */
|
||||
.computer-hud{position:absolute;inset:0;z-index:3;display:grid;align-content:center;justify-items:center;gap:14px;padding:10px 12px 8px;border-radius:inherit;background:radial-gradient(ellipse at center,#142722e8,#101012ed);backdrop-filter:blur(8px);pointer-events:none;opacity:1;transition:opacity .22s ease}
|
||||
.computer-hud.is-leaving{opacity:0}
|
||||
.computer-hud-label{font-size:12px;letter-spacing:.02em;line-height:1.5;white-space:normal;text-align:center;max-width:90%;color:#c9ddd5}
|
||||
.empty-computer.is-waiting{min-height:100%;background:var(--inset)}
|
||||
.computer-actions{display:flex;align-items:center;justify-content:flex-end;gap:8px;min-width:0;flex-wrap:wrap}
|
||||
.computer-actions .restart-computer{font-size:12px;padding-inline:10px;color:var(--muted)}
|
||||
|
|
@ -47,12 +51,10 @@
|
|||
.control-bar{flex-wrap:wrap}
|
||||
.control-bar>.computer-actions{flex:1 1 180px}
|
||||
/* Independent monitor mascot, with mint halo and amber orbit. */
|
||||
.computer-hud{inset:0;align-content:center;border-radius:inherit;gap:14px;background:radial-gradient(ellipse at center,#142722e8,#101012ed);box-shadow:none}
|
||||
.computer-signal{position:relative;display:grid;place-items:center;width:64px;height:64px;border:1px solid #65dcb34d;border-radius:24px;animation:monitor-breathe 2.8s ease-in-out infinite}
|
||||
.computer-signal::before{content:"";position:absolute;inset:-8px;border:1px solid #65dcb326;border-top-color:#efbe72;border-radius:50%;animation:monitor-orbit 4s linear infinite}
|
||||
.computer-signal-face{display:flex;align-items:center;justify-content:center;gap:12px;width:46px;height:38px;border-radius:15px;background:#85dfbb;color:#143429;transform:rotate(-6deg)}
|
||||
.computer-signal-face i{width:5px;height:11px;border-radius:5px;background:currentColor;animation:monitor-blink 4.6s ease-in-out infinite}
|
||||
.computer-hud-label{color:#c9ddd5;background:none;animation:none;white-space:normal;text-align:center;max-width:90%;line-height:1.5}
|
||||
|
||||
.clipboard-status{margin:0;min-height:1.4em;font-size:12px;line-height:1.4;color:var(--muted)}
|
||||
|
||||
|
|
@ -68,3 +70,10 @@
|
|||
.computer-power-menu .computer-power-stop{color:var(--danger-soft)}
|
||||
.computer-power-menu small{display:block;padding:8px 10px 6px;border-top:1px solid var(--line);color:var(--muted);font-size:11px;line-height:1.6}
|
||||
.computer-overlay .computer-power-menu{top:calc(100% + 8px);bottom:auto}
|
||||
|
||||
/* The overlay tracks the visible viewport so the veil and the controls stay
|
||||
above the keyboard; the veil itself belongs to the desktop box below. */
|
||||
.computer-overlay{top:var(--visible-top,0px);bottom:auto;height:var(--visible-height,100dvh)}
|
||||
.computer-overlay>header{flex-shrink:0}
|
||||
.overlay-error{position:static;transform:none;flex:none;margin:0 8px 8px}
|
||||
@media(max-width:700px){.overlay-screen{padding:4px}.computer-overlay>header{min-height:44px;gap:4px}.computer-overlay>header>div{gap:5px}.computer-overlay>header strong{font-size:12px}}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
// Everything that decides when the screen veil shows, and when the viewer may
|
||||
// be thrown away. Both jobs used to be a fixed timer plus "null the url", which
|
||||
// is exactly what made starting and handing over feel like a stall: the mascot
|
||||
// popped, the desktop went black, and noVNC booted from scratch.
|
||||
|
||||
import type { ComputerState } from "./types";
|
||||
|
||||
/** The viewer url is the proxy path for a bot and never carries a nonce, so the
|
||||
* frame can mount while the desktop is still coming up and let noVNC dial into
|
||||
* it on retry instead of waiting for a status round trip. */
|
||||
export function viewerPath(botId: string): string {
|
||||
return `/view/${encodeURIComponent(botId)}/vnc.html`;
|
||||
}
|
||||
|
||||
/** Only a computer that is really gone may drop the pixels. While it boots,
|
||||
* wakes, or a poll blinks, the mounted frame keeps its VNC session; remounting
|
||||
* costs a page load, a handshake, and a black flash for no information. */
|
||||
export function keepScreenUrl(current: string | null, next: string | null, state: ComputerState): string | null {
|
||||
if (next) return next;
|
||||
if (state === "stopped" || state === "error") return null;
|
||||
return current;
|
||||
}
|
||||
|
||||
/** A handoff should read as one deliberate beat, not as a wait. This is the
|
||||
* ceiling, used only when the server never confirms; agreement on the new
|
||||
* holder usually releases the veil far sooner. */
|
||||
export const HANDOFF_MS = 900;
|
||||
/** The floor. Fast as the reply may land, the swap is worth one visible beat,
|
||||
* otherwise the mascot strobes instead of gesturing. */
|
||||
export const HANDOFF_MIN_MS = 320;
|
||||
/** Kept in step with the veil's CSS fade so it never vanishes mid-animation. */
|
||||
export const VEIL_FADE_MS = 220;
|
||||
|
||||
/** How much longer the veil has to stay after the server agreed on a holder:
|
||||
* zero once the beat has been served, and never more than the ceiling timer,
|
||||
* which is what a lost reply falls back on. */
|
||||
export function handoffRemaining(startedAt: number, now: number): number {
|
||||
const held = now - startedAt;
|
||||
if (held >= HANDOFF_MS) return 0;
|
||||
return Math.max(HANDOFF_MIN_MS - held, 0);
|
||||
}
|
||||
|
||||
export interface Veil { label: string | null; leaving: boolean }
|
||||
|
||||
/** Fade in with a label, fade out once the label clears, and stay out when
|
||||
* there was never anything to show. Pure, so a status blip can be tested. */
|
||||
export function nextVeil(label: string | null, current: Veil): Veil {
|
||||
if (label) return current.label === label && !current.leaving ? current : { label, leaving: false };
|
||||
if (!current.label) return current;
|
||||
return current.leaving ? current : { label: current.label, leaving: true };
|
||||
}
|
||||
|
||||
/** Shared desktop input stays enabled while the agent works or waits for help.
|
||||
* Older servers still use the exclusive holder gate. */
|
||||
export function viewOnlyFor(holder: "none" | "bot" | "user", sharedInput = false): boolean {
|
||||
return !sharedInput && holder !== "user";
|
||||
}
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
// Live session feed. The API keeps a durable, ordered event cursor per session
|
||||
// and streams it as server-sent events, so the browser does not have to guess
|
||||
// when the transcript changed. This module owns only the transport half:
|
||||
// subscribe, decode, drop replays, and collapse a burst into one refresh.
|
||||
|
||||
export type SessionEventKind =
|
||||
| "message.created"
|
||||
| "run.started"
|
||||
| "run.paused"
|
||||
| "run.failed"
|
||||
| "run.completed"
|
||||
| "session.cleared";
|
||||
|
||||
/**
|
||||
* Kinds the app listens for. An SSE source cannot subscribe to "everything", so
|
||||
* a kind added on the server needs to appear here to be delivered live; the
|
||||
* caller's safety poll still picks anything new up within a few seconds.
|
||||
*/
|
||||
export const SESSION_EVENT_TYPES: SessionEventKind[] = [
|
||||
"message.created",
|
||||
"run.started",
|
||||
"run.paused",
|
||||
"run.failed",
|
||||
"run.completed",
|
||||
"session.cleared",
|
||||
];
|
||||
|
||||
export interface SessionEvent {
|
||||
kind: string;
|
||||
/** Server cursor (`events.seq`); 0 when the source sent no id. */
|
||||
id: number;
|
||||
payload: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** The slice of `EventSource` this module uses, so tests can fake the socket. */
|
||||
export interface EventSourceLike {
|
||||
addEventListener(type: string, listener: (event: { data?: string; lastEventId?: string }) => void): void;
|
||||
close(): void;
|
||||
}
|
||||
|
||||
export interface LiveFeed {
|
||||
/** Stop listening. Safe to call more than once. */
|
||||
close(): void;
|
||||
}
|
||||
|
||||
export function sessionEventsUrl(sessionId: string): string {
|
||||
return `/api/sessions/${encodeURIComponent(sessionId)}/events`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the event stream for one session. `onStatus` reports whether the live
|
||||
* path is up, which lets the caller decide how often it needs to poll as a
|
||||
* backup; it stays silent about the browser's own reconnect attempts.
|
||||
*/
|
||||
export function subscribeToSession(
|
||||
sessionId: string,
|
||||
onEvent: (event: SessionEvent) => void,
|
||||
options: {
|
||||
source?: (url: string) => EventSourceLike;
|
||||
onStatus?: (connected: boolean) => void;
|
||||
} = {},
|
||||
): LiveFeed {
|
||||
const open = options.source ?? ((url: string) => new EventSource(url) as unknown as EventSourceLike);
|
||||
const source = open(sessionEventsUrl(sessionId));
|
||||
// A reconnect replays from the last id the browser saw, and the server replays
|
||||
// from its cursor, so the same event can legitimately arrive twice. Applying
|
||||
// it twice is only harmless work, but the cursor is free and keeps the caller
|
||||
// honest about doing real work once per event.
|
||||
let cursor = 0;
|
||||
const receive = (kind: string) => (event: { data?: string; lastEventId?: string }) => {
|
||||
const id = Number(event.lastEventId);
|
||||
if (Number.isFinite(id) && id > 0) {
|
||||
if (id <= cursor) return;
|
||||
cursor = id;
|
||||
}
|
||||
let payload: Record<string, unknown> = {};
|
||||
try {
|
||||
const parsed: unknown = event.data ? JSON.parse(event.data) : {};
|
||||
if (parsed && typeof parsed === "object") payload = parsed as Record<string, unknown>;
|
||||
} catch {
|
||||
// A malformed frame is a dropped frame: the next event, or the poll,
|
||||
// delivers the same state.
|
||||
}
|
||||
onEvent({ kind, id: Number.isFinite(id) ? id : 0, payload });
|
||||
};
|
||||
for (const kind of SESSION_EVENT_TYPES) source.addEventListener(kind, receive(kind));
|
||||
source.addEventListener("open", () => options.onStatus?.(true));
|
||||
source.addEventListener("error", () => options.onStatus?.(false));
|
||||
let closed = false;
|
||||
return {
|
||||
close() {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
source.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export interface Coalescer {
|
||||
/** Ask for a run; a request inside an open window joins the pending one. */
|
||||
kick(): void;
|
||||
/** Forget a pending run, e.g. because the session was switched away from. */
|
||||
cancel(): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapse a burst of "something changed" into a single run. One turn can move
|
||||
* several events at once, and each of them would otherwise fetch the whole
|
||||
* transcript again.
|
||||
*/
|
||||
export function createCoalescer(
|
||||
run: () => void,
|
||||
windowMs: number,
|
||||
schedule: (callback: () => void, ms: number) => number = (callback, ms) => setTimeout(callback, ms) as unknown as number,
|
||||
dismiss: (handle: number) => void = handle => clearTimeout(handle),
|
||||
): Coalescer {
|
||||
let handle: number | null = null;
|
||||
return {
|
||||
kick() {
|
||||
if (handle !== null) return;
|
||||
handle = schedule(() => {
|
||||
handle = null;
|
||||
run();
|
||||
}, windowMs);
|
||||
},
|
||||
cancel() {
|
||||
if (handle === null) return;
|
||||
dismiss(handle);
|
||||
handle = null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -2,6 +2,9 @@ import type { zhTW } from "./zh-TW";
|
|||
|
||||
/** English UI copy. Keys must stay in lockstep with zh-TW. */
|
||||
export const en: { [K in keyof typeof zhTW]: string } = {
|
||||
sharedDesktop: "Shared control",
|
||||
sharedNeedsUser: "{name} is waiting for you to finish the steps on screen. Then press “Done, continue”.",
|
||||
doneContinue: "Done, continue",
|
||||
search: "Search",
|
||||
sharedComputer: "Shared computer",
|
||||
privateComputer: "Private computer",
|
||||
|
|
@ -253,6 +256,7 @@ export const en: { [K in keyof typeof zhTW]: string } = {
|
|||
computerPower: "Computer menu",
|
||||
sharedPowerHint: "This is a shared computer. The action affects every agent using it.",
|
||||
stopAndTakeOver: "Stop and take over",
|
||||
takeoverBusy: "Stop the task first, then take the mouse.",
|
||||
hudBooting: "Computer starting…",
|
||||
hudWaking: "Waking…",
|
||||
hudConnecting: "Connecting…",
|
||||
|
|
@ -453,5 +457,60 @@ export const en: { [K in keyof typeof zhTW]: string } = {
|
|||
schedHumanElapsed: "Every {n} minutes (elapsed)",
|
||||
schedCalendar: "Calendar schedule: {expr}",
|
||||
teachInProgress: "A demo is in progress. Finish or cancel it before sending a message.",
|
||||
aiTimeout: "The model timed out (120 seconds).",
|
||||
aiTimeout: "The model timed out (150 seconds).",
|
||||
resumeMidTask: "Stopped halfway — your call",
|
||||
resumeBudget: "Turn budget spent, result unverified",
|
||||
resumeLoop: "Kept repeating itself — needs a nudge",
|
||||
resumeProgress: "{turns}/{limit} turns",
|
||||
resumeContinue: "Keep going",
|
||||
resumeStop: "Stop here",
|
||||
resumeSent: "Keep going and finish it.", monitorTitle: "Live log",
|
||||
monitorQueued: "Queued",
|
||||
monitorTurnOf: "turn {turn}/{limit}",
|
||||
monitorTurn: "turn {turn}",
|
||||
monitorElapsed: "up {time}",
|
||||
monitorNow: "Now: {step}",
|
||||
monitorEmpty: "Nothing yet — the trail fills in as I work.",
|
||||
monitorLoadFailed: "Couldn’t load the log",
|
||||
monitorCopy: "Copy log",
|
||||
monitorCopied: "Copied",
|
||||
monitorPinHint: "Folds away when the mouse leaves",
|
||||
monitorPinned: "Pinned — press Esc to close",
|
||||
monitorClose: "Close",
|
||||
monitorKindModel: "Model",
|
||||
monitorKindTool: "Action",
|
||||
monitorKindRetry: "Retry",
|
||||
monitorKindNotice: "Note",
|
||||
monitorKindRun: "Status",
|
||||
monitorRunStarted: "Started: {task}",
|
||||
monitorRunCompleted: "Done in {turns} turns",
|
||||
monitorRunFailed: "Failed",
|
||||
monitorRunWaiting: "Waiting for you: {reason}",
|
||||
monitorRunPaused: "Paused: {reason}",
|
||||
monitorRunRetry: "Re-queued",
|
||||
monitorModelTurn: "turn {turn}, thought for {time}",
|
||||
monitorRetryAttempt: "attempt {attempt} failed, retrying",
|
||||
monitorGaveUp: "gave up",
|
||||
monitorStatusOk: "ok",
|
||||
monitorStatusError: "failed",
|
||||
monitorStatusTimedOut: "timed out",
|
||||
monitorStatusPaused: "needs you",
|
||||
errorTitleInterrupted: "Interrupted",
|
||||
errorTitleToolTimeout: "Action timed out",
|
||||
errorTitleModelKey: "Model API key rejected",
|
||||
errorTitleModelQuota: "Rate limited or out of quota",
|
||||
errorTitleModelUnknown: "Model not found",
|
||||
errorTitleModelTimeout: "Model took too long",
|
||||
errorTitleNetwork: "Can’t reach the model",
|
||||
errorTitleComputerGone: "Computer is gone",
|
||||
errorTitleLeaseLost: "Work picked up elsewhere",
|
||||
errorTitleUnknown: "Something broke",
|
||||
errorChip: "Task failed",
|
||||
errorRetry: "Retry",
|
||||
errorRetryHint: "Continue where it stopped",
|
||||
errorOpenSettings: "Model settings",
|
||||
errorDetails: "Show the log",
|
||||
errorRetried: "Back in the queue",
|
||||
errorRetryFailed: "Can’t re-queue right now — another run may still be busy.",
|
||||
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
/** Traditional Chinese UI copy. Keep keys stable when adding another locale. */
|
||||
export const zhTW = {
|
||||
sharedDesktop: "共同操作",
|
||||
sharedNeedsUser: "{name} 已暫停等你完成畫面上的步驟。完成後按「完成，繼續」。",
|
||||
doneContinue: "完成，繼續",
|
||||
search: "搜尋", sharedComputer: "共用電腦", privateComputer: "私人電腦",
|
||||
stopped: "已關閉", booting: "啟動中", running: "執行中", suspended: "休眠中", error: "發生錯誤",
|
||||
openComputer: "開啟電腦", stopTask: "停止任務", takeControl: "取得控制", takeOverNow: "接手操作", releaseControl: "釋放控制", done: "完成", skip: "略過",
|
||||
|
|
@ -64,7 +67,7 @@ export const zhTW = {
|
|||
mcpNoResults: "找不到符合的 MCP。換個關鍵字，或改用自訂接入。", mcpCustom: "自訂接入", mcpBackToList: "回到列表",
|
||||
mcpConnectNamed: "接入 {name}", mcpKeyHint: "這個 MCP 需要憑證才能連。",
|
||||
noMcp: "還沒有 MCP。點上面的「選擇 MCP」從市集接入。", toolsCount: "{count} 個工具", disabled: "已關閉", disconnected: "未連線", noTools: "沒有可用工具", reconnect: "重新連線", disable: "停用", enable: "啟用",
|
||||
preparingDesktop: "正在準備 Agent 的獨立桌面…", computerPreviewHint: "開啟電腦後，畫面會顯示在這裡。", bootingProgress: "啟動中…", restartComputer: "重啟", shutDownComputer: "關閉電腦", computerPower: "電腦選單", sharedPowerHint: "此為共用電腦，操作會影響使用它的所有 Agent。", stopAndTakeOver: "停止並接管",
|
||||
preparingDesktop: "正在準備 Agent 的獨立桌面…", computerPreviewHint: "開啟電腦後，畫面會顯示在這裡。", bootingProgress: "啟動中…", restartComputer: "重啟", shutDownComputer: "關閉電腦", computerPower: "電腦選單", sharedPowerHint: "此為共用電腦，操作會影響使用它的所有 Agent。", stopAndTakeOver: "停止並接管", takeoverBusy: "請先停止任務，再接手滑鼠。",
|
||||
hudBooting: "電腦啟動中…", hudWaking: "喚醒中…", hudConnecting: "連線中…", hudHandoff: "換手中…",
|
||||
pasteToRemoteComputer: "貼到遠端電腦", pasteRemoteHelp: "把外面的文字貼在這裡，再送進 VNC。這個方式在區網 HTTP 也能使用。", pasteTextPlaceholder: "在此貼上文字…", pasteIntoVnc: "貼入 VNC",
|
||||
botNamePlaceholder: "例如：研究助理", sharedComputerHint: "與其他機器人共用環境", privateComputerHint: "全新的獨立 Docker", create: "建立",
|
||||
|
|
@ -148,5 +151,60 @@ export const zhTW = {
|
|||
schedHumanElapsed: "每隔 {n} 分鐘（固定間隔）",
|
||||
schedCalendar: "日曆排程：{expr}",
|
||||
teachInProgress: "示範進行中：先按「完成示範」或「取消」，再送訊息。",
|
||||
aiTimeout: "AI 回應逾時（120 秒）",
|
||||
aiTimeout: "AI 回應逾時（150 秒）",
|
||||
resumeMidTask: "做到一半，需要你決定",
|
||||
resumeBudget: "輪次用盡，尚未確認完成",
|
||||
resumeLoop: "卡在同一個動作，需要你給方向",
|
||||
resumeProgress: "{turns}/{limit} 輪",
|
||||
resumeContinue: "繼續",
|
||||
resumeStop: "就到這裡",
|
||||
resumeSent: "繼續，把它做完。", monitorTitle: "即時記錄",
|
||||
monitorQueued: "排隊中",
|
||||
monitorTurnOf: "第 {turn}/{limit} 輪",
|
||||
monitorTurn: "第 {turn} 輪",
|
||||
monitorElapsed: "已運行 {time}",
|
||||
monitorNow: "現在：{step}",
|
||||
monitorEmpty: "還沒有記錄。開始動手後會一條條出現。",
|
||||
monitorLoadFailed: "記錄載入失敗",
|
||||
monitorCopy: "複製記錄",
|
||||
monitorCopied: "已複製",
|
||||
monitorPinHint: "滑鼠移開會自動收起來",
|
||||
monitorPinned: "已釘住（按 Esc 關閉）",
|
||||
monitorClose: "關閉",
|
||||
monitorKindModel: "模型",
|
||||
monitorKindTool: "動作",
|
||||
monitorKindRetry: "重試",
|
||||
monitorKindNotice: "提醒",
|
||||
monitorKindRun: "狀態",
|
||||
monitorRunStarted: "開始工作：{task}",
|
||||
monitorRunCompleted: "完成了（{turns} 輪）",
|
||||
monitorRunFailed: "失敗了",
|
||||
monitorRunWaiting: "停下來等你：{reason}",
|
||||
monitorRunPaused: "暫停：{reason}",
|
||||
monitorRunRetry: "重新排進佇列",
|
||||
monitorModelTurn: "第 {turn} 輪，想了 {time}",
|
||||
monitorRetryAttempt: "第 {attempt} 次嘗試失敗，正在重試",
|
||||
monitorGaveUp: "放棄重試",
|
||||
monitorStatusOk: "成功",
|
||||
monitorStatusError: "失敗",
|
||||
monitorStatusTimedOut: "逾時",
|
||||
monitorStatusPaused: "要你接手",
|
||||
errorTitleInterrupted: "被打斷",
|
||||
errorTitleToolTimeout: "動作逾時",
|
||||
errorTitleModelKey: "模型金鑰不被接受",
|
||||
errorTitleModelQuota: "限流或額度不足",
|
||||
errorTitleModelUnknown: "模型名稱找不到",
|
||||
errorTitleModelTimeout: "模型太久沒回話",
|
||||
errorTitleNetwork: "連不到模型",
|
||||
errorTitleComputerGone: "電腦不見了",
|
||||
errorTitleLeaseLost: "工作被另一邊接手",
|
||||
errorTitleUnknown: "出錯了",
|
||||
errorChip: "任務失敗",
|
||||
errorRetry: "重試",
|
||||
errorRetryHint: "從中斷的地方接著做",
|
||||
errorOpenSettings: "開模型設定",
|
||||
errorDetails: "看詳細記錄",
|
||||
errorRetried: "已重新排進佇列",
|
||||
errorRetryFailed: "現在重試排不進去，可能還有別的工作在跑。",
|
||||
|
||||
} as const;
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import "./avatar.css";
|
|||
import "./chat.css";
|
||||
import "./computer.css";
|
||||
import "./schedule.css";
|
||||
import "./monitor.css";
|
||||
import "./call.css";
|
||||
import "./responsive.css";
|
||||
import { App } from "./App";
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
/* The live run bubble. It floats above whatever opened it, so it never has to
|
||||
fit inside the composer or a message bubble. */
|
||||
.run-probe{position:relative;display:inline-flex;align-items:center;border-radius:10px;outline:none}
|
||||
.run-probe:focus-visible{box-shadow:0 0 0 2px var(--focus)}
|
||||
.run-probe.inactive{cursor:default}
|
||||
.run-monitor{position:fixed;z-index:60;display:flex;flex-direction:column;gap:8px;max-height:min(64vh,460px);padding:12px 13px 10px;border:1px solid var(--border);border-radius:14px;background:var(--menu);box-shadow:0 18px 44px rgba(0,0,0,.55);color:var(--ink);font-size:12px;line-height:1.45;text-align:left;cursor:default}
|
||||
.run-monitor-head{display:flex;flex-wrap:wrap;align-items:baseline;justify-content:space-between;gap:8px}
|
||||
.run-monitor-head b{font-size:13px}
|
||||
.run-monitor-turn{color:var(--accent);font-variant-numeric:tabular-nums}
|
||||
.run-monitor-step{color:var(--muted);overflow-wrap:anywhere}
|
||||
.run-monitor-step.bad{color:var(--error-fg)}
|
||||
.run-monitor-list{display:flex;flex-direction:column;gap:5px;min-height:44px;max-height:min(38vh,300px);overflow-y:auto;padding-right:2px;border-top:1px solid var(--line);border-bottom:1px solid var(--line);padding-top:7px;padding-bottom:7px}
|
||||
.run-monitor-empty{color:var(--faint)}
|
||||
.run-line{display:grid;grid-template-columns:58px 40px 1fr;gap:7px;align-items:baseline}
|
||||
.run-line-clock{color:var(--faint);font-variant-numeric:tabular-nums}
|
||||
.run-line-kind{color:var(--muted)}
|
||||
.run-line-text{overflow-wrap:anywhere}
|
||||
.run-line.is-error .run-line-text,.run-line.is-timed_out .run-line-text{color:var(--error-fg)}
|
||||
.run-line.is-paused .run-line-text{color:var(--warn-fg)}
|
||||
.run-line.kind-run .run-line-text{color:var(--ink)}
|
||||
.run-line.kind-retry .run-line-text{color:var(--warn-fg)}
|
||||
.run-monitor-error{display:flex;flex-direction:column;gap:4px;padding:9px 10px;border:1px solid var(--error-border);border-radius:11px;background:var(--error-bg);color:var(--error-fg)}
|
||||
.run-monitor-error code{display:block;max-height:110px;overflow:auto;white-space:pre-wrap;overflow-wrap:anywhere;color:var(--error-fg);font-size:11px;opacity:.85}
|
||||
.run-monitor-foot{display:flex;align-items:center;gap:9px}
|
||||
.run-monitor-foot button{display:inline-flex;align-items:center;gap:5px;padding:5px 9px;border:1px solid var(--border);border-radius:9px;background:var(--elevated);color:var(--ink);font-size:12px;cursor:pointer}
|
||||
.run-monitor-foot button:hover{background:var(--surface-hover)}
|
||||
.run-monitor-foot small{flex:1;color:var(--faint)}
|
||||
.run-monitor-close{padding:3px 8px !important;color:var(--muted)}
|
||||
|
||||
/* Failure card in the chat: what broke, and the one thing to do next. */
|
||||
.error-chip{border-color:var(--error-border);background:var(--error-bg)}
|
||||
.error-chip .login-label{display:flex;align-items:center;gap:7px;color:var(--error-fg);font-weight:600}
|
||||
.error-chip .error-body{color:var(--ink);overflow-wrap:anywhere}
|
||||
.error-chip .error-actions{display:flex;flex-wrap:wrap;align-items:center;gap:8px;margin-top:2px}
|
||||
.error-chip .error-actions button{padding:6px 12px;border:1px solid var(--border);border-radius:9px;background:var(--elevated);color:var(--ink);font-size:13px;cursor:pointer}
|
||||
.error-chip .error-actions button.primary{border-color:transparent;background:var(--cream);color:var(--on-cream)}
|
||||
.error-chip .error-actions button:disabled{opacity:.55;cursor:default}
|
||||
.error-chip .error-probe{margin-left:auto;color:var(--error-fg);cursor:pointer}
|
||||
.error-chip .error-said{color:var(--error-fg);font-size:12px}
|
||||
|
|
@ -40,7 +40,8 @@
|
|||
.chat-panel{--chat-gutter:max(34px,7vw);--chat-col:min(760px,100%)}
|
||||
|
||||
|
||||
.chat-panel:has(.composer-dock.has-status) .messages{padding-bottom:188px}
|
||||
.composer-dock .error-banner,.composer-dock .queue-hint{position:relative;left:auto;bottom:auto;transform:none;z-index:auto}
|
||||
.composer-dock .queue-hint{text-align:center}
|
||||
|
||||
|
||||
.spinner{width:17px;height:17px;animation:spin .8s linear infinite}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
@media(max-width:1050px){.app-shell{grid-template-columns:250px 1fr}.computer-panel{display:none}}
|
||||
@media(max-width:700px){.app-shell{display:block}.sidebar{position:fixed;z-index:40;inset:0 auto 0 0;width:min(300px,88vw);transform:translateX(-105%);transition:.2s;box-shadow:20px 0 60px #000}.sidebar.open{transform:none}.chat-panel{height:100%}.mobile-menu{display:inline-flex}.topbar{padding:0 12px}.messages{padding:24px 18px 130px}.composer{left:12px;right:12px}.computer-overlay>header{height:auto;min-height:64px;flex-wrap:wrap;padding:10px}.computer-overlay>header>div:last-child{flex-wrap:wrap;justify-content:flex-end}.overlay-screen{padding:8px}.mode-grid{grid-template-columns:1fr}}
|
||||
@media(max-width:700px){.app-shell{display:block}.sidebar{position:fixed;z-index:40;inset:0 auto 0 0;width:min(300px,88vw);transform:translateX(-105%);transition:.2s;box-shadow:20px 0 60px #000}.sidebar.open{transform:none}.chat-panel{height:100%}.mobile-menu{display:inline-flex}.topbar{padding:0 12px}.messages{padding:24px 18px 16px}.composer{left:12px;right:12px}.computer-overlay>header{height:auto;min-height:64px;flex-wrap:wrap;padding:10px}.computer-overlay>header>div:last-child{flex-wrap:wrap;justify-content:flex-end}.overlay-screen{padding:8px}.mode-grid{grid-template-columns:1fr}}
|
||||
@media(max-width:1200px){.app-shell{grid-template-columns:230px minmax(0,1fr) 330px}}
|
||||
@media(max-width:1050px){.app-shell{grid-template-columns:250px minmax(0,1fr)}}
|
||||
@media(max-width:700px){.chat-panel{--chat-gutter:18px}.app-shell{display:block}.message{width:100%}.message>span,.message>.message-body,.message-stack{max-width:90%}.composer-dock{padding:20px var(--chat-gutter) 16px}.composer{width:var(--chat-col);min-height:60px}.composer-files{padding:2px 4px 8px 8px}.file-card{max-width:100%;height:52px}.file-card-remove{flex-basis:32px;width:32px;height:32px}.message.with-files .msg-attachments{max-width:100%}.messages{padding-bottom:118px}.chat-panel:has(.composer-dock.has-status) .messages{padding-bottom:178px}}
|
||||
@media(max-width:700px){.chat-panel{--chat-gutter:18px}.app-shell{display:block}.message{width:100%}.message>span,.message>.message-body,.message-stack{max-width:90%}.composer-dock{padding:20px var(--chat-gutter) 16px}.composer{width:var(--chat-col);min-height:60px}.composer-files{padding:2px 4px 8px 8px}.file-card{max-width:100%;height:52px}.file-card-remove{flex-basis:32px;width:32px;height:32px}.message.with-files .msg-attachments{max-width:100%}.messages{padding-bottom:16px}}
|
||||
@media(prefers-reduced-motion:reduce){.avatar.blobatar.thinking:before,.avatar.blobatar.thinking:after,.thinking-dots i{animation:none!important}}
|
||||
@media(max-width:1200px){.app-shell.right-open{grid-template-columns:230px minmax(0,1fr) 340px}.app-shell.right-collapsed{grid-template-columns:230px minmax(0,1fr) 52px}}
|
||||
@media(max-width:1050px){
|
||||
|
|
|
|||
|
|
@ -0,0 +1,372 @@
|
|||
import { useCallback, useEffect, useRef, useState, type KeyboardEvent as ReactKeyboardEvent, type ReactNode } from "react";
|
||||
import { api } from "./api";
|
||||
import { t, type MessageKey } from "./i18n";
|
||||
import type { RunActivity, RunActivityEntry } from "./types";
|
||||
|
||||
/**
|
||||
* The observability bubble: hover (or tap, or focus) an avatar to watch the run
|
||||
* that avatar is doing, turn by turn. The trail is polled rather than streamed
|
||||
* because it is only interesting while somebody is looking at it.
|
||||
*/
|
||||
const POLL_MS = 1500;
|
||||
const CLOSE_MS = 300;
|
||||
const KEEP = 200;
|
||||
const PANEL_WIDTH = 380;
|
||||
const DONE = ["completed", "failed", "cancelled"];
|
||||
|
||||
export type RunActionId = "retry" | "screen" | "settings";
|
||||
|
||||
/**
|
||||
* Which buttons actually help for a failure code, best first. A bad API key is
|
||||
* fixed in settings and a lost computer is checked on screen — retrying alone
|
||||
* would only burn another turn on the same wall.
|
||||
*/
|
||||
const ACTIONS: Record<string, RunActionId[]> = {
|
||||
interrupted: ["screen", "retry"],
|
||||
tool_timeout: ["screen", "retry"],
|
||||
model_key: ["settings", "retry"],
|
||||
model_unknown: ["settings", "retry"],
|
||||
model_quota: ["retry", "settings"],
|
||||
model_timeout: ["retry", "settings"],
|
||||
network: ["retry", "settings"],
|
||||
computer_gone: ["screen", "retry"],
|
||||
lease_lost: ["retry"],
|
||||
unknown: ["retry"],
|
||||
};
|
||||
|
||||
const TITLES: Record<string, MessageKey> = {
|
||||
interrupted: "errorTitleInterrupted",
|
||||
tool_timeout: "errorTitleToolTimeout",
|
||||
model_key: "errorTitleModelKey",
|
||||
model_quota: "errorTitleModelQuota",
|
||||
model_unknown: "errorTitleModelUnknown",
|
||||
model_timeout: "errorTitleModelTimeout",
|
||||
network: "errorTitleNetwork",
|
||||
computer_gone: "errorTitleComputerGone",
|
||||
lease_lost: "errorTitleLeaseLost",
|
||||
unknown: "errorTitleUnknown",
|
||||
};
|
||||
|
||||
export function errorActions(code?: string | null): RunActionId[] {
|
||||
return ACTIONS[code || ""] || ACTIONS.unknown;
|
||||
}
|
||||
|
||||
/** Short label for a failure code; the long sentence lives in the message body. */
|
||||
export function errorTitle(code?: string | null): string {
|
||||
return t(TITLES[code || ""] || TITLES.unknown);
|
||||
}
|
||||
|
||||
/** 1:05 / 1:02:05 — how long the run has been at it. */
|
||||
export function formatElapsed(ms: number): string {
|
||||
const seconds = Number.isFinite(ms) ? Math.max(0, Math.floor(ms / 1000)) : 0;
|
||||
const pad = (value: number) => String(value).padStart(2, "0");
|
||||
const hours = Math.floor(seconds / 3600);
|
||||
const minutes = Math.floor((seconds % 3600) / 60);
|
||||
return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
|
||||
}
|
||||
|
||||
/** 340ms / 6.4s / 1:05 — durations inside the trail stay glanceable. */
|
||||
export function shortDuration(ms?: number | null): string {
|
||||
if (typeof ms !== "number" || !Number.isFinite(ms) || ms < 0) return "";
|
||||
if (ms < 1000) return `${Math.round(ms)}ms`;
|
||||
if (ms < 10_000) return `${(ms / 1000).toFixed(1)}s`;
|
||||
return formatElapsed(ms);
|
||||
}
|
||||
|
||||
function kindLabel(kind: string): string {
|
||||
if (kind === "model") return t("monitorKindModel");
|
||||
if (kind === "tool") return t("monitorKindTool");
|
||||
if (kind === "retry") return t("monitorKindRetry");
|
||||
if (kind === "notice") return t("monitorKindNotice");
|
||||
return t("monitorKindRun");
|
||||
}
|
||||
|
||||
function statusLabel(status?: string | null): string {
|
||||
if (status === "ok") return t("monitorStatusOk");
|
||||
if (status === "timed_out") return t("monitorStatusTimedOut");
|
||||
if (status === "paused") return t("monitorStatusPaused");
|
||||
if (status === "error") return t("monitorStatusError");
|
||||
return "";
|
||||
}
|
||||
|
||||
/** One trail line as plain text — what the panel shows and what gets copied. */
|
||||
export function trailText(entry: RunActivityEntry): string {
|
||||
if (entry.kind === "run") {
|
||||
if (entry.event === "started") return t("monitorRunStarted", { task: entry.task || "" });
|
||||
if (entry.event === "completed") return t("monitorRunCompleted", { turns: entry.turns ?? 0 });
|
||||
if (entry.event === "failed") return t("monitorRunFailed");
|
||||
if (entry.event === "waiting_input") return t("monitorRunWaiting", { reason: entry.reason || "" });
|
||||
if (entry.event === "paused") return t("monitorRunPaused", { reason: entry.reason || "" });
|
||||
if (entry.event === "retry") return t("monitorRunRetry");
|
||||
return entry.text || entry.reason || t("monitorKindRun");
|
||||
}
|
||||
if (entry.kind === "model") {
|
||||
const head = t("monitorModelTurn", { turn: entry.turn ?? 0, time: shortDuration(entry.elapsedMs) });
|
||||
return entry.text ? `${head} — ${entry.text}` : head;
|
||||
}
|
||||
if (entry.kind === "tool") {
|
||||
const parts = [entry.step || entry.name || t("monitorKindTool")];
|
||||
const status = statusLabel(entry.status);
|
||||
if (status) parts.push(status);
|
||||
const time = shortDuration(entry.elapsedMs);
|
||||
if (time) parts.push(time);
|
||||
const detail = entry.snippet ? ` — ${entry.snippet}` : "";
|
||||
return `${parts.join(" · ")}${detail}`;
|
||||
}
|
||||
if (entry.kind === "retry") {
|
||||
const head = `${t("monitorRetryAttempt", { attempt: entry.attempt ?? 0 })}${entry.gaveUp ? ` · ${t("monitorGaveUp")}` : ""}`;
|
||||
return entry.error ? `${head} — ${entry.error}` : head;
|
||||
}
|
||||
return entry.text || t("monitorKindNotice");
|
||||
}
|
||||
|
||||
function clockOf(createdAt: string): string {
|
||||
const date = new Date(createdAt);
|
||||
if (Number.isNaN(date.getTime())) return "";
|
||||
const pad = (value: number) => String(value).padStart(2, "0");
|
||||
return `${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}`;
|
||||
}
|
||||
|
||||
function logDump(snapshot: RunActivity | null, entries: RunActivityEntry[]): string {
|
||||
const turn = snapshot?.turnLimit
|
||||
? t("monitorTurnOf", { turn: snapshot.turn ?? 0, limit: snapshot.turnLimit })
|
||||
: t("monitorTurn", { turn: snapshot?.turn ?? 0 });
|
||||
const head = [
|
||||
snapshot?.runId ?? "",
|
||||
snapshot?.status ?? "",
|
||||
snapshot?.turn != null ? turn : "",
|
||||
snapshot?.step ?? "",
|
||||
snapshot?.elapsedMs != null ? t("monitorElapsed", { time: formatElapsed(snapshot.elapsedMs) }) : "",
|
||||
].filter(Boolean).join(" · ");
|
||||
const trail = entries.map(entry => `${clockOf(entry.createdAt)} ${trailText(entry)}`);
|
||||
const error = snapshot?.error ? `\n[${snapshot.error.code}] ${snapshot.error.raw}` : "";
|
||||
return [head, ...trail].join("\n") + error;
|
||||
}
|
||||
|
||||
async function copyLog(text: string): Promise<boolean> {
|
||||
try {
|
||||
await navigator.clipboard.writeText(text);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps an avatar (or any small trigger) in a live readout of the run behind
|
||||
* it. Hover opens it and it closes ~300ms after the pointer leaves; a click
|
||||
* pins it so the panel survives reaching for the copy button; Esc lets go.
|
||||
*/
|
||||
export function RunProbe({ runId, align = "start", label, children }: { runId?: string | null; align?: "start" | "end"; label?: string; children: ReactNode }) {
|
||||
const [open, setOpen] = useState(false);
|
||||
const [pinned, setPinned] = useState(false);
|
||||
const [snapshot, setSnapshot] = useState<RunActivity | null>(null);
|
||||
const [entries, setEntries] = useState<RunActivityEntry[]>([]);
|
||||
const [stale, setStale] = useState(false);
|
||||
const [copied, setCopied] = useState<boolean | null>(null);
|
||||
const [box, setBox] = useState<{ left: number; bottom: number } | null>(null);
|
||||
const anchorRef = useRef<HTMLSpanElement | null>(null);
|
||||
const listRef = useRef<HTMLSpanElement | null>(null);
|
||||
const lastId = useRef(0);
|
||||
const followTail = useRef(true);
|
||||
const linger = useRef<number | null>(null);
|
||||
const pinnedRef = useRef(false);
|
||||
|
||||
useEffect(() => {
|
||||
pinnedRef.current = pinned;
|
||||
}, [pinned]);
|
||||
|
||||
useEffect(() => {
|
||||
lastId.current = 0;
|
||||
setSnapshot(null);
|
||||
setEntries([]);
|
||||
setStale(false);
|
||||
}, [runId]);
|
||||
|
||||
const place = useCallback(() => {
|
||||
const node = anchorRef.current;
|
||||
if (!node) return;
|
||||
const rect = node.getBoundingClientRect();
|
||||
const width = Math.min(PANEL_WIDTH, window.innerWidth - 24);
|
||||
const edge = align === "end" ? rect.right - width : rect.left;
|
||||
setBox({
|
||||
left: Math.max(12, Math.min(edge, window.innerWidth - 12 - width)),
|
||||
bottom: Math.max(12, window.innerHeight - rect.top + 8),
|
||||
});
|
||||
}, [align]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!open || !runId) return;
|
||||
let stopped = false;
|
||||
let timer = 0;
|
||||
const load = async () => {
|
||||
try {
|
||||
const after = lastId.current > 0 ? `?after=${lastId.current}` : "";
|
||||
const next = await api<RunActivity>(`/api/runs/${runId}/activity${after}`);
|
||||
if (stopped) return;
|
||||
setStale(false);
|
||||
setSnapshot(next);
|
||||
setEntries(current => {
|
||||
if (lastId.current <= 0) return next.activity.slice(-KEEP);
|
||||
const seen = new Set(current.map(entry => entry.id));
|
||||
return [...current, ...next.activity.filter(entry => !seen.has(entry.id))].slice(-KEEP);
|
||||
});
|
||||
for (const entry of next.activity) lastId.current = Math.max(lastId.current, entry.id);
|
||||
if (DONE.includes(next.status)) window.clearInterval(timer);
|
||||
} catch {
|
||||
if (!stopped) setStale(true);
|
||||
}
|
||||
};
|
||||
timer = window.setInterval(() => void load(), POLL_MS);
|
||||
void load();
|
||||
return () => {
|
||||
stopped = true;
|
||||
window.clearInterval(timer);
|
||||
};
|
||||
}, [open, runId]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) return;
|
||||
place();
|
||||
window.addEventListener("resize", place);
|
||||
return () => window.removeEventListener("resize", place);
|
||||
}, [open, place]);
|
||||
|
||||
useEffect(() => {
|
||||
const list = listRef.current;
|
||||
if (open && followTail.current && list) list.scrollTop = list.scrollHeight;
|
||||
}, [entries, open]);
|
||||
|
||||
const hoverIn = useCallback(() => {
|
||||
if (linger.current) window.clearTimeout(linger.current);
|
||||
linger.current = null;
|
||||
setOpen(true);
|
||||
}, []);
|
||||
|
||||
const hoverOut = useCallback(() => {
|
||||
if (pinnedRef.current) return;
|
||||
if (linger.current) window.clearTimeout(linger.current);
|
||||
linger.current = window.setTimeout(() => setOpen(false), CLOSE_MS);
|
||||
}, []);
|
||||
|
||||
useEffect(
|
||||
() => () => {
|
||||
if (linger.current) window.clearTimeout(linger.current);
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const onKeyDown = (event: ReactKeyboardEvent<HTMLSpanElement>) => {
|
||||
if (event.key === "Escape") {
|
||||
setPinned(false);
|
||||
setOpen(false);
|
||||
return;
|
||||
}
|
||||
if (event.key === "Enter" || event.key === " ") {
|
||||
event.preventDefault();
|
||||
setPinned(current => !current);
|
||||
setOpen(true);
|
||||
}
|
||||
};
|
||||
|
||||
const turnText =
|
||||
snapshot?.status === "queued"
|
||||
? t("monitorQueued")
|
||||
: snapshot?.turn != null
|
||||
? snapshot.turnLimit
|
||||
? t("monitorTurnOf", { turn: snapshot.turn, limit: snapshot.turnLimit })
|
||||
: t("monitorTurn", { turn: snapshot.turn })
|
||||
: "—";
|
||||
|
||||
return (
|
||||
<span
|
||||
className={`run-probe ${runId ? "" : "inactive"}`}
|
||||
ref={anchorRef}
|
||||
tabIndex={runId ? 0 : -1}
|
||||
aria-expanded={open}
|
||||
aria-label={label || t("monitorTitle")}
|
||||
onMouseEnter={runId ? hoverIn : undefined}
|
||||
onMouseLeave={runId ? hoverOut : undefined}
|
||||
onClick={
|
||||
runId
|
||||
? () => {
|
||||
setOpen(true);
|
||||
setPinned(current => !current);
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
onKeyDown={runId ? onKeyDown : undefined}
|
||||
>
|
||||
{children}
|
||||
{open ? (
|
||||
<span
|
||||
className={`run-monitor ${pinned ? "pinned" : ""}`}
|
||||
style={box ? { left: box.left, bottom: box.bottom, width: Math.min(PANEL_WIDTH, window.innerWidth - 24) } : { display: "none" }}
|
||||
role="dialog"
|
||||
aria-label={t("monitorTitle")}
|
||||
onMouseEnter={hoverIn}
|
||||
onMouseLeave={hoverOut}
|
||||
>
|
||||
<span className="run-monitor-head">
|
||||
<b>{t("monitorTitle")}</b>
|
||||
<span className="run-monitor-turn">
|
||||
{turnText}
|
||||
{snapshot?.elapsedMs != null ? ` · ${t("monitorElapsed", { time: formatElapsed(snapshot.elapsedMs) })}` : ""}
|
||||
</span>
|
||||
</span>
|
||||
<span className={`run-monitor-step ${snapshot?.error ? "bad" : ""}`}>
|
||||
{snapshot?.step ? t("monitorNow", { step: snapshot.step }) : snapshot?.error?.headline || t("monitorEmpty")}
|
||||
</span>
|
||||
<span
|
||||
className="run-monitor-list"
|
||||
ref={listRef}
|
||||
onScroll={event => {
|
||||
const node = event.currentTarget;
|
||||
followTail.current = node.scrollHeight - node.scrollTop - node.clientHeight < 32;
|
||||
}}
|
||||
>
|
||||
{entries.length === 0 ? <span className="run-monitor-empty">{stale ? t("monitorLoadFailed") : t("monitorEmpty")}</span> : null}
|
||||
{entries.map(entry => (
|
||||
<span className={`run-line kind-${entry.kind} ${entry.status && entry.status !== "ok" ? `is-${entry.status}` : ""}`} key={entry.id}>
|
||||
<span className="run-line-clock">{clockOf(entry.createdAt)}</span>
|
||||
<span className="run-line-kind">{kindLabel(entry.kind)}</span>
|
||||
<span className="run-line-text">{trailText(entry)}</span>
|
||||
</span>
|
||||
))}
|
||||
</span>
|
||||
{snapshot?.error ? (
|
||||
<span className="run-monitor-error">
|
||||
<b>{`[${snapshot.error.code}] ${errorTitle(snapshot.error.code)}`}</b>
|
||||
<code>{snapshot.error.raw}</code>
|
||||
</span>
|
||||
) : null}
|
||||
<span className="run-monitor-foot">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() =>
|
||||
void copyLog(logDump(snapshot, entries)).then(ok => {
|
||||
setCopied(ok);
|
||||
window.setTimeout(() => setCopied(null), 1800);
|
||||
})
|
||||
}
|
||||
>
|
||||
{copied === null ? t("monitorCopy") : copied ? t("monitorCopied") : t("clipboardWriteBlocked")}
|
||||
</button>
|
||||
<small>{pinned ? t("monitorPinned") : t("monitorPinHint")}</small>
|
||||
<button
|
||||
type="button"
|
||||
className="run-monitor-close"
|
||||
aria-label={t("monitorClose")}
|
||||
onClick={() => {
|
||||
setPinned(false);
|
||||
setOpen(false);
|
||||
}}
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</span>
|
||||
</span>
|
||||
) : null}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
.login-chip .login-label,.sched-chip .sched-label{color:var(--muted);font-size:12px}
|
||||
.login-chip .login-site{font-weight:600}
|
||||
.login-chip .login-why{color:var(--muted);font-size:13px}
|
||||
.resume-chip .resume-actions{display:flex;flex-wrap:wrap;gap:8px;margin-top:2px}
|
||||
.sched-list{display:grid;gap:8px;min-height:0;overflow:auto}
|
||||
.sched-list-head{display:flex;align-items:center;justify-content:space-between;color:var(--muted);font-size:13px}
|
||||
.sched-empty{margin:0;color:var(--faint);font-size:13px}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,13 @@ export interface Message { id:string; sessionId?:string; seq?:number; role:strin
|
|||
export interface MessageFile { kind:"image"|"file"; name:string; mimeType?:string; size?:number }
|
||||
export interface RoomMember { id:string; name:string; avatarColor:string; avatarShape:AvatarShape }
|
||||
export interface Room { id:string; name:string; members:RoomMember[]; lastMessageAt:string|null; lastPreview:string|null; unreadCount:number }
|
||||
export interface ComputerStatus { botId:string; mode:ComputerMode; state:ComputerState; controlHolder:"none"|"bot"|"user"; takeoverRequested:boolean; busyBotName:string|null; busySessionId:string|null; busyRunId:string|null; busyStep?:string|null; usingComputer?:boolean; waitingRunId?:string|null; waitingSessionId?:string|null; queuedRuns?:number; display:string|null; profileMode:string; screenAvailable:boolean }
|
||||
export interface ComputerStatus { botId:string; mode:ComputerMode; state:ComputerState; sharedInput?:boolean; controlHolder:"none"|"bot"|"user"; takeoverRequested:boolean; busyBotName:string|null; busySessionId:string|null; busyRunId:string|null; busyStep?:string|null; usingComputer?:boolean; waitingRunId?:string|null; waitingSessionId?:string|null; queuedRuns?:number; display:string|null; profileMode:string; screenAvailable:boolean }
|
||||
/** One line of the live trail a run writes while it works. */
|
||||
export type RunActivityKind = "run"|"model"|"tool"|"retry"|"notice";
|
||||
export interface RunActivityEntry { id:number; kind:RunActivityKind; createdAt:string; turn?:number|null; event?:string|null; task?:string|null; reason?:string|null; turns?:number|null; limit?:number|null; error?:string|null; name?:string|null; step?:string|null; status?:string|null; elapsedMs?:number|null; toolCalls?:number|null; text?:string|null; snippet?:string|null; attempt?:number|null; gaveUp?:boolean|null }
|
||||
export interface RunActivityError { code:string; headline:string; action?:string; raw:string }
|
||||
export interface RunActivity { runId:string; status:string; turn:number|null; turnLimit:number|null; step:string|null; stepAt?:string|null; elapsedMs:number|null; error:RunActivityError|null; activity:RunActivityEntry[] }
|
||||
|
||||
export interface PlaybookStep { do:string; expect?:string; note?:string }
|
||||
export interface PlaybookInput { name:string; description?:string; example?:string }
|
||||
export interface Playbook { name?:string; whenToUse?:string; intent?:string; inputs?:PlaybookInput[]; preconditions?:string[]; steps?:(PlaybookStep|string)[]; howToCheck?:string; whatToReturn?:string; cautions?:string[] }
|
||||
|
|
|
|||
|
|
@ -12,12 +12,28 @@
|
|||
background: #0f172a;
|
||||
overflow: hidden;
|
||||
}
|
||||
#status {
|
||||
display: none;
|
||||
}
|
||||
#status { flex:none; padding:6px 10px; color:#f8fafc; font:12px system-ui; }
|
||||
#status[hidden] { display:none; }
|
||||
body { display:flex; flex-direction:column; }
|
||||
#screen { flex:1; min-height:0; height:auto; }
|
||||
#screen canvas { cursor: default; }
|
||||
#trackpad { height:clamp(64px,18vh,150px); margin-top:6px; border:1px solid #64748b;
|
||||
border-radius:10px; background:#111b2e; touch-action:none; user-select:none;
|
||||
display:grid; place-items:center; color:#94a3b8; }
|
||||
#trackpad[hidden] { display:none; }
|
||||
#mobile-controls .shortcuts { margin-top:5px; }
|
||||
#mobile-controls .shortcuts[hidden] { display:none; }
|
||||
@media(orientation:landscape) {
|
||||
body.touch-ui {
|
||||
display:grid;
|
||||
grid-template:"status status" auto "screen controls" 1fr / minmax(0,1fr) 210px;
|
||||
}
|
||||
body.touch-ui #status { grid-area:status; }
|
||||
body.touch-ui #screen { grid-area:screen; width:auto; height:auto; min-width:0; }
|
||||
body.touch-ui #mobile-controls { grid-area:controls; width:auto; box-sizing:border-box; overflow:auto; }
|
||||
body.touch-ui #mobile-controls .buttons { flex-wrap:wrap; }
|
||||
body.touch-ui #trackpad { height:80px; }
|
||||
}
|
||||
#mobile-controls { display:none; flex:none; padding:6px 8px max(6px,env(safe-area-inset-bottom));
|
||||
background:#172033; color:#f8fafc; font:12px system-ui; border-top:1px solid #334155; }
|
||||
#mobile-controls .buttons { display:flex; gap:5px; overflow-x:auto; }
|
||||
|
|
@ -32,6 +48,7 @@
|
|||
#mobile-cursor::after { content:""; position:absolute; width:4px; height:4px; background:#fff;
|
||||
border-radius:50%; left:7px; top:7px; }
|
||||
@media(any-pointer:coarse) { #mobile-controls { display:block; } }
|
||||
body.touch-ui #mobile-controls { display:block; }
|
||||
/* Keep a real editable element on screen for iOS/Android keyboards. */
|
||||
#mobile-keyboard { position:fixed; bottom:0; left:0; width:1px; height:1px;
|
||||
padding:0; border:0; opacity:.01; font-size:16px; pointer-events:none; }
|
||||
|
|
@ -56,6 +73,7 @@
|
|||
const statusEl = document.getElementById("status");
|
||||
const setStatus = (text) => {
|
||||
statusEl.textContent = text;
|
||||
statusEl.hidden = !text;
|
||||
};
|
||||
|
||||
const prefix = window.location.pathname.replace(/[^/]+$/, "");
|
||||
|
|
@ -67,6 +85,20 @@
|
|||
|
||||
let rfb = null;
|
||||
let reconnectTimer = null;
|
||||
// A desktop that has never answered on this page is still coming up, and
|
||||
// x11vnc opens its port a heartbeat after the last attempt failed. Holding
|
||||
// every retry at the calm cadence leaves up to half a second of veil on top
|
||||
// of a desktop that is already there, so the first window of attempts comes
|
||||
// back eager. Once a session has been seen the calm cadence returns: a
|
||||
// genuinely dropped connection should not be hammered.
|
||||
const CALM_RETRY_MS = 1500;
|
||||
const EAGER_RETRY_MS = 350;
|
||||
const EAGER_ATTEMPTS = 40;
|
||||
let everConnected = false;
|
||||
let eagerAttempts = 0;
|
||||
const retryDelay = () => (!everConnected && eagerAttempts < EAGER_ATTEMPTS)
|
||||
? (eagerAttempts += 1, EAGER_RETRY_MS)
|
||||
: CALM_RETRY_MS;
|
||||
const parentOrigin = window.location.origin;
|
||||
const keyboard = document.getElementById("mobile-keyboard");
|
||||
const sentinel = "\u200b";
|
||||
|
|
@ -78,7 +110,13 @@
|
|||
const modeButton = document.getElementById("pointer-mode");
|
||||
const dragButton = document.getElementById("pointer-drag");
|
||||
const pointerHelp = document.getElementById("pointer-help");
|
||||
let trackpad = false;
|
||||
const touchUi = window.matchMedia?.("(any-pointer:coarse)");
|
||||
document.body?.classList?.toggle("touch-ui", Boolean(touchUi?.matches));
|
||||
touchUi?.addEventListener?.("change", event => document.body?.classList?.toggle("touch-ui", event.matches));
|
||||
let trackpad = Boolean(touchUi?.matches);
|
||||
const pad = document.getElementById("trackpad");
|
||||
const shortcuts = document.getElementById("keyboard-shortcuts");
|
||||
let modifier = null;
|
||||
let dragging = false;
|
||||
let gesture = null;
|
||||
let pointer = { x: .5, y: .5 };
|
||||
|
|
@ -126,11 +164,12 @@
|
|||
buttons:0, preventDefault(){}, stopPropagation(){} });
|
||||
}
|
||||
function updatePointerControls() {
|
||||
if (pad) pad.hidden = !trackpad;
|
||||
modeButton?.setAttribute?.("aria-pressed", String(trackpad));
|
||||
if (modeButton) modeButton.textContent = trackpad ? "觸控板" : "直接點選";
|
||||
if (pointerHelp) pointerHelp.textContent = trackpad
|
||||
? "滑動移游標・輕點左鍵・雙指捲動／輕點右鍵・拖曳按完再按一次放開"
|
||||
: "點畫面定位並開鍵盤・長按右鍵・精準操作可切換觸控板";
|
||||
: "點畫面定位・按「鍵盤」輸入・精準操作可切換觸控板";
|
||||
for (const button of controls?.querySelectorAll?.("button[data-action]") || []) {
|
||||
button.disabled = !rfb || rfb.viewOnly;
|
||||
}
|
||||
|
|
@ -139,6 +178,7 @@
|
|||
function showKeyboard() {
|
||||
if (!rfb || rfb.viewOnly) return;
|
||||
rfb.focusOnClick = false;
|
||||
if (shortcuts) shortcuts.hidden = false;
|
||||
keyboard.focus({ preventScroll: true });
|
||||
keyboard.setSelectionRange(keyboard.value.length, keyboard.value.length);
|
||||
}
|
||||
|
|
@ -152,6 +192,13 @@
|
|||
}
|
||||
if (!rfb || rfb.viewOnly) return;
|
||||
if (action === "keyboard") { releaseDrag(); showKeyboard(); return; }
|
||||
if (action === "hide-keyboard") { keyboard.blur(); if (shortcuts) shortcuts.hidden = true; return; }
|
||||
if (action === "modifier") {
|
||||
modifier = modifier === button.dataset.key ? null : button.dataset.key;
|
||||
for (const item of controls.querySelectorAll('[data-action="modifier"]')) item.setAttribute("aria-pressed", String(item.dataset.key === modifier));
|
||||
return;
|
||||
}
|
||||
if (action === "key") { mobileKey(button.dataset.key); return; }
|
||||
if (action === "left") pointerClick(1);
|
||||
if (action === "right") pointerClick(4);
|
||||
if (action === "up") { releaseDrag(); scrollPointer(0, -100); }
|
||||
|
|
@ -170,7 +217,7 @@
|
|||
y:points.reduce((sum,p)=>sum+p.clientY,0)/points.length };
|
||||
}
|
||||
function startPointerTouch(event) {
|
||||
if (!trackpad || !event.target.closest?.("#screen")) return false;
|
||||
if (!trackpad || !event.target.closest?.("#trackpad")) return false;
|
||||
stopTouch(event);
|
||||
if (!rfb || rfb.viewOnly) {
|
||||
window.parent.postMessage({ type:"lazyboy-request-control" }, parentOrigin);
|
||||
|
|
@ -222,7 +269,9 @@
|
|||
}
|
||||
function mobileKey(key) {
|
||||
if (!rfb || rfb.viewOnly) return;
|
||||
window.parent.postMessage({ type: "lazyboy-mobile-key", key }, parentOrigin);
|
||||
window.parent.postMessage({ type: "lazyboy-mobile-key", key: modifier ? `${modifier}+${key}` : key }, parentOrigin);
|
||||
modifier = null;
|
||||
for (const item of controls?.querySelectorAll?.('[data-action="modifier"]') || []) item.setAttribute("aria-pressed", "false");
|
||||
}
|
||||
function commitKeyboard() {
|
||||
if (composing) return;
|
||||
|
|
@ -255,9 +304,17 @@
|
|||
});
|
||||
// Focus synchronously in the user's tap, before iOS loses activation.
|
||||
// VNC pixels cannot reveal whether the remote target is a text field.
|
||||
function ignoreScreenTouch(event) {
|
||||
if (!trackpad || !event.target.closest?.("#screen")) return false;
|
||||
event.preventDefault();
|
||||
event.stopImmediatePropagation();
|
||||
touchStart = null;
|
||||
return true;
|
||||
}
|
||||
window.addEventListener("touchstart", event => {
|
||||
if (event.target === keyboard) return;
|
||||
if (startPointerTouch(event)) { touchStart = null; return; }
|
||||
if (ignoreScreenTouch(event)) return;
|
||||
const point = event.touches[0];
|
||||
const rect = pointerGeometry();
|
||||
if (point && rect && event.target.closest?.("#screen")) {
|
||||
|
|
@ -270,6 +327,7 @@
|
|||
}, { passive: false, capture: true });
|
||||
window.addEventListener("touchmove", event => {
|
||||
if (movePointerTouch(event)) return;
|
||||
if (ignoreScreenTouch(event)) return;
|
||||
const point = event.touches[0];
|
||||
if (touchStart && (!point || event.touches.length !== 1 ||
|
||||
Math.hypot(point.clientX - touchStart.x, point.clientY - touchStart.y) > 10)) touchStart = null;
|
||||
|
|
@ -277,11 +335,11 @@
|
|||
window.addEventListener("touchcancel", () => { touchStart = null; gesture = null; releaseDrag(); }, true);
|
||||
window.addEventListener("touchend", event => {
|
||||
if (endPointerTouch(event)) return;
|
||||
if (ignoreScreenTouch(event)) return;
|
||||
const tapped = touchStart && event.touches.length === 0;
|
||||
touchStart = null;
|
||||
if (!tapped || !rfb || rfb.viewOnly) return;
|
||||
keyboard.focus({ preventScroll: true });
|
||||
keyboard.setSelectionRange(keyboard.value.length, keyboard.value.length);
|
||||
// Direct taps position the pointer; the keyboard opens only on request.
|
||||
}, { passive: false, capture: true });
|
||||
|
||||
function pasteIntoDesktop(text) {
|
||||
|
|
@ -303,39 +361,58 @@
|
|||
}
|
||||
}
|
||||
|
||||
// The host flips this gate whenever control changes hands, including
|
||||
// while the frame is still dialling or between retries. Keep the last
|
||||
// thing it asked for and hand it to every new RFB, so a handoff never
|
||||
// depends on which side of a reconnect the message happened to land.
|
||||
let wantedViewOnly = null;
|
||||
function applyViewOnly(value) {
|
||||
wantedViewOnly = Boolean(value);
|
||||
if (!rfb) return;
|
||||
if (value) { releaseDrag(); gesture = null; }
|
||||
rfb.viewOnly = Boolean(value);
|
||||
if (wantedViewOnly) { releaseDrag(); gesture = null; modifier = null; if (shortcuts) shortcuts.hidden = true; }
|
||||
rfb.viewOnly = wantedViewOnly;
|
||||
updatePointerControls();
|
||||
if (rfb.viewOnly) { keyboard.blur(); resetKeyboard(); }
|
||||
// Taking over has to be usable, not just painted: focus so the very
|
||||
// first keystroke after the veil lands on the desktop.
|
||||
else { try { rfb.focus(); } catch (_) {} }
|
||||
}
|
||||
|
||||
function connect() {
|
||||
setStatus("Connecting to desktop…");
|
||||
window.parent.postMessage({ type: "lazyboy-desktop-lost" }, parentOrigin);
|
||||
rfb = new RFB(document.getElementById("screen"), url);
|
||||
rfb.viewOnly = flag("view_only", true);
|
||||
rfb.viewOnly = wantedViewOnly ?? flag("view_only", true);
|
||||
rfb.scaleViewport = true;
|
||||
rfb.qualityLevel = 6;
|
||||
rfb.compressionLevel = 2;
|
||||
rfb.clipViewport = false;
|
||||
rfb.background = "#0f172a";
|
||||
pinTaskbar();
|
||||
updatePointerControls();
|
||||
rfb.addEventListener("connect", () => {
|
||||
everConnected = true;
|
||||
eagerAttempts = 0;
|
||||
setStatus("");
|
||||
pinTaskbar();
|
||||
updatePointerControls();
|
||||
try { rfb.focus(); } catch (_) {}
|
||||
if (wantedViewOnly !== null) applyViewOnly(wantedViewOnly);
|
||||
window.parent.postMessage({ type: "lazyboy-desktop-ready" }, parentOrigin);
|
||||
});
|
||||
rfb.addEventListener("disconnect", (event) => {
|
||||
releaseDrag(); gesture = null;
|
||||
applyViewOnly(true);
|
||||
// Disable the disconnected instance without overwriting host intent.
|
||||
rfb.viewOnly = true;
|
||||
modifier = null;
|
||||
if (shortcuts) shortcuts.hidden = true;
|
||||
updatePointerControls();
|
||||
keyboard.blur(); resetKeyboard();
|
||||
window.parent.postMessage({ type: "lazyboy-desktop-lost" }, parentOrigin);
|
||||
const clean = event && event.detail && event.detail.clean;
|
||||
setStatus(clean ? "Disconnected — retrying" : "Desktop connection lost — retrying");
|
||||
if (reconnectTimer) clearTimeout(reconnectTimer);
|
||||
reconnectTimer = setTimeout(connect, 1500);
|
||||
reconnectTimer = setTimeout(connect, retryDelay());
|
||||
});
|
||||
rfb.addEventListener("clipboard", (event) => {
|
||||
const text = event && event.detail ? event.detail.text : "";
|
||||
|
|
@ -416,6 +493,18 @@
|
|||
<button data-action="down" aria-label="向下捲動">↓</button>
|
||||
<button data-action="keyboard">鍵盤</button>
|
||||
</div>
|
||||
<div class="buttons shortcuts" id="keyboard-shortcuts" hidden>
|
||||
<button data-action="modifier" data-key="ctrl" aria-pressed="false">Ctrl</button>
|
||||
<button data-action="modifier" data-key="alt" aria-pressed="false">Alt</button>
|
||||
<button data-action="key" data-key="a">A</button><button data-action="key" data-key="c">C</button>
|
||||
<button data-action="key" data-key="v">V</button><button data-action="key" data-key="z">Z</button>
|
||||
<button data-action="key" data-key="Tab">Tab</button><button data-action="key" data-key="Escape">Esc</button>
|
||||
<button data-action="key" data-key="Left">←</button><button data-action="key" data-key="Right">→</button>
|
||||
<button data-action="key" data-key="Up">↑</button><button data-action="key" data-key="Down">↓</button>
|
||||
<button data-action="key" data-key="Return">Enter</button><button data-action="key" data-key="BackSpace">⌫</button>
|
||||
<button data-action="hide-keyboard">收起鍵盤</button>
|
||||
</div>
|
||||
<div id="trackpad" role="group" aria-label="獨立觸控板" hidden>在這裡滑動控制滑鼠</div>
|
||||
<p id="pointer-help"></p>
|
||||
</nav>
|
||||
<textarea id="mobile-keyboard" aria-label="Remote desktop keyboard" tabindex="-1"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
# Clippy configuration for the LazyBoy workspace.
|
||||
#
|
||||
# cargo-clippy looks for this file next to the working directory it was started
|
||||
# in, so always run clippy from the workspace root (`make clippy`).
|
||||
#
|
||||
# Handlers and run/vault/voice helpers legitimately thread 8-10 parameters
|
||||
# (state + actor + ids). The default of 7 is too tight for this codebase, so we
|
||||
# keep the guard for genuinely oversized signatures instead of disabling it.
|
||||
too-many-arguments-threshold = 10
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
name = "lazyboy-api"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
publish.workspace = true
|
||||
|
||||
|
|
@ -14,23 +15,19 @@ axum.workspace = true
|
|||
tokio.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
sqlx.workspace = true
|
||||
uuid.workspace = true
|
||||
chrono.workspace = true
|
||||
tower-http.workspace = true
|
||||
async-trait = "0.1"
|
||||
rig-core.workspace = true
|
||||
base64.workspace = true
|
||||
hmac.workspace = true
|
||||
sha2.workspace = true
|
||||
hex.workspace = true
|
||||
reqwest.workspace = true
|
||||
tokio-tungstenite.workspace = true
|
||||
futures-util = "0.3"
|
||||
http-body-util = "0.1"
|
||||
dotenvy = "0.15"
|
||||
fastembed = { version = "6.0.2", default-features = false, features = ["hf-hub-rustls-tls", "ort-load-dynamic"] }
|
||||
rmcp = { version = "3.2", default-features = false, features = ["client", "transport-child-process", "transport-streamable-http-client-reqwest"] }
|
||||
|
|
@ -41,3 +38,6 @@ chrono-tz = "0.10"
|
|||
rand = "0.8"
|
||||
|
||||
cap-std = "3"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
|
|
|||
|
|
@ -185,14 +185,14 @@ pub async fn llm_parts(
|
|||
}
|
||||
}
|
||||
} else if is_text_mime(&stored.mime_type) {
|
||||
if let Some(bytes) = bytes {
|
||||
if let Some(text) = utf8_preview(&bytes) {
|
||||
if let Some(bytes) = bytes
|
||||
&& let Some(text) = utf8_preview(&bytes)
|
||||
{
|
||||
parts.push(UserContent::text(format!(
|
||||
"Contents of {}:\n```\n{text}\n```",
|
||||
stored.name
|
||||
)));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
notes.push(
|
||||
" Open this with open_path or the desktop if you need the original file.".into(),
|
||||
|
|
@ -220,8 +220,9 @@ pub async fn sweep_all_inboxes(data_dir: &str) {
|
|||
continue;
|
||||
};
|
||||
sweep_cap_inbox(&home, "inbox");
|
||||
if let Ok(bots) = home.open_dir("bots") {
|
||||
if let Ok(entries) = bots.entries() {
|
||||
if let Ok(bots) = home.open_dir("bots")
|
||||
&& let Ok(entries) = bots.entries()
|
||||
{
|
||||
for bot in entries.flatten() {
|
||||
if let Ok(dir) = bots.open_dir(bot.file_name()) {
|
||||
sweep_cap_inbox(&dir, "inbox");
|
||||
|
|
@ -229,7 +230,6 @@ pub async fn sweep_all_inboxes(data_dir: &str) {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
|
@ -351,9 +351,7 @@ fn parse_stored(value: &Value) -> Option<StoredAttachment> {
|
|||
}
|
||||
let path = value.get("path").and_then(Value::as_str).unwrap_or("");
|
||||
if !path.is_empty() {
|
||||
let Some(stored) = path.strip_prefix("inbox/") else {
|
||||
return None;
|
||||
};
|
||||
let stored = path.strip_prefix("inbox/")?;
|
||||
if stored.is_empty() || stored == "." || stored == ".." || stored.contains(['/', '\\']) {
|
||||
return None;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ impl AuthConfig {
|
|||
pub fn strong_enough_for_network(&self) -> bool {
|
||||
self.token
|
||||
.as_ref()
|
||||
.map(|token| token.as_bytes().len() >= 32 && token != "dev-token")
|
||||
.map(|token| token.len() >= 32 && token != "dev-token")
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
|
|
@ -83,15 +83,14 @@ impl AuthConfig {
|
|||
let key = hex::encode(Sha256::digest(value.as_bytes()));
|
||||
let mut sessions = self.sessions.lock().unwrap();
|
||||
sessions.retain(|_, expires| *expires > Instant::now());
|
||||
if sessions.len() >= 4096 {
|
||||
if let Some(oldest) = sessions
|
||||
if sessions.len() >= 4096
|
||||
&& let Some(oldest) = sessions
|
||||
.iter()
|
||||
.min_by_key(|(_, t)| **t)
|
||||
.map(|(k, _)| k.clone())
|
||||
{
|
||||
sessions.remove(&oldest);
|
||||
}
|
||||
}
|
||||
sessions.insert(key, Instant::now() + Duration::from_secs(604800));
|
||||
Some(format!(
|
||||
"{COOKIE_NAME}={value}; Path=/; HttpOnly; SameSite=Strict; Max-Age=604800{}",
|
||||
|
|
|
|||
|
|
@ -3,13 +3,13 @@ use std::time::Duration;
|
|||
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use lazyboy_contracts::{
|
||||
BrowserProfileMode, ComputerCapabilities, ComputerMode, ComputerState, ComputerStatus,
|
||||
ControlHolder, DEFAULT_SCREEN_HEIGHT, DEFAULT_SCREEN_WIDTH,
|
||||
BrowserProfileMode, ComputerCapabilities, ComputerMode, ComputerStatus, ControlHolder,
|
||||
DEFAULT_SCREEN_HEIGHT, DEFAULT_SCREEN_WIDTH,
|
||||
};
|
||||
use lazyboy_control::{
|
||||
AdapterContext, CommandRequest, EnsureScreenRequest, ProvisionRequest, admit_gui,
|
||||
admit_new_screen, browser_profile_path, execution_blocks_user_takeover, profile_lock_key,
|
||||
screen_layout, team_bot_workspace_directory, user_holds_control,
|
||||
screen_layout, team_bot_workspace_directory,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
|
|
@ -46,6 +46,7 @@ pub fn status_from(
|
|||
mode: parse_mode(&computer.scope),
|
||||
kind: parse_kind(&computer.kind),
|
||||
state: parse_state(&computer.state),
|
||||
shared_input: true,
|
||||
control_holder,
|
||||
control_bot_id,
|
||||
takeover_requested: false,
|
||||
|
|
@ -110,6 +111,71 @@ pub struct BoundScreen {
|
|||
pub gui_block: Option<String>,
|
||||
}
|
||||
|
||||
/// Refresh presentation on an existing running screen without opening apps,
|
||||
/// changing the Cua session, or booting a stopped computer.
|
||||
pub async fn refresh_cursor_color(
|
||||
state: &AppState,
|
||||
actor: &Actor,
|
||||
bot_id: &str,
|
||||
) -> Result<(), String> {
|
||||
let Some(bot) = state
|
||||
.db
|
||||
.get_bot(actor, bot_id)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(computer_id) = bot.computer_id else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(computer) = state
|
||||
.db
|
||||
.get_computer(&computer_id)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
if computer.state != "running" {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(target) = computer_ref(&computer) else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(screen) = state
|
||||
.db
|
||||
.get_screen(&computer_id, bot_id)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let result = state
|
||||
.sandbox
|
||||
.execute(
|
||||
&target,
|
||||
CommandRequest {
|
||||
argv: vec![
|
||||
"/usr/local/bin/lazyboy-screen".into(),
|
||||
"color".into(),
|
||||
screen.slot.to_string(),
|
||||
bot.avatar_color,
|
||||
],
|
||||
cwd: None,
|
||||
timeout_ms: Some(5_000),
|
||||
stdin: None,
|
||||
},
|
||||
&adapter_context_for(actor, bot_id, "cursor-color", Some(&screen), None),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
if result.code != 0 {
|
||||
return Err("cursor color refresh failed".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn ensure_bot_screen(
|
||||
state: &AppState,
|
||||
actor: &Actor,
|
||||
|
|
@ -209,10 +275,17 @@ pub async fn ensure_bot_screen(
|
|||
}
|
||||
};
|
||||
let ctx = adapter_context_for(actor, bot_id, "screen", Some(&row), run_id);
|
||||
let bot = state
|
||||
.db
|
||||
.get_bot(actor, bot_id)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let request = EnsureScreenRequest {
|
||||
slot: row.slot as u32,
|
||||
profile_path: row.profile_path.clone(),
|
||||
bot_id: bot_id.to_string(),
|
||||
bot_name: bot.as_ref().map(|bot| bot.name.clone()).unwrap_or_default(),
|
||||
bot_color: bot.map(|bot| bot.avatar_color).unwrap_or_default(),
|
||||
};
|
||||
let mut last_error = None;
|
||||
for attempt in 0..8 {
|
||||
|
|
@ -262,6 +335,7 @@ async fn restore_computer_screens(
|
|||
continue;
|
||||
}
|
||||
let ctx = adapter_context_for(actor, &screen.bot_id, "screen", Some(&screen), None);
|
||||
let bot = state.db.get_bot(actor, &screen.bot_id).await.ok().flatten();
|
||||
let _ = state
|
||||
.sandbox
|
||||
.ensure_screen(
|
||||
|
|
@ -270,6 +344,8 @@ async fn restore_computer_screens(
|
|||
slot: screen.slot as u32,
|
||||
profile_path: screen.profile_path.clone(),
|
||||
bot_id: screen.bot_id.clone(),
|
||||
bot_name: bot.as_ref().map(|bot| bot.name.clone()).unwrap_or_default(),
|
||||
bot_color: bot.map(|bot| bot.avatar_color).unwrap_or_default(),
|
||||
},
|
||||
&ctx,
|
||||
)
|
||||
|
|
@ -910,7 +986,7 @@ pub async fn takeover(
|
|||
&thread_id,
|
||||
&run_id,
|
||||
bot_id,
|
||||
"你已接手操作。完成後釋放控制權，我會從目前畫面繼續。",
|
||||
"我已暫停等你操作。完成後按「完成，繼續」，我會從目前畫面接著做。",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
|
@ -998,32 +1074,12 @@ pub async fn heartbeat(state: &AppState, actor: &Actor, bot_id: &str) -> Result<
|
|||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn user_has_control(computer: &ComputerRow, bot_id: &str) -> bool {
|
||||
user_has_screen_control(computer, None, bot_id)
|
||||
}
|
||||
|
||||
pub fn user_has_screen_control(
|
||||
computer: &ComputerRow,
|
||||
screen: Option<&ScreenRow>,
|
||||
bot_id: &str,
|
||||
) -> bool {
|
||||
if let Some(screen) = screen {
|
||||
return user_holds_control(
|
||||
parse_holder(&screen.control_holder),
|
||||
Some(screen.bot_id.as_str()),
|
||||
bot_id,
|
||||
screen.control_lease_expires_at,
|
||||
Utc::now(),
|
||||
);
|
||||
}
|
||||
user_holds_control(
|
||||
parse_holder(&computer.control_holder),
|
||||
computer.control_bot_id.as_deref(),
|
||||
bot_id,
|
||||
computer.control_lease_expires_at,
|
||||
Utc::now(),
|
||||
)
|
||||
/// Viewing/input does not acquire the human pause lease. Agent execution and
|
||||
/// explicit requests for human assistance retain their own pause/resume flow.
|
||||
pub fn user_can_interact(computer: &ComputerRow, screen: Option<&ScreenRow>, bot_id: &str) -> bool {
|
||||
computer.state == "running"
|
||||
&& computer.provider_ref.is_some()
|
||||
&& screen.is_some_and(|screen| screen.bot_id == bot_id && screen.computer_id == computer.id)
|
||||
}
|
||||
|
||||
pub async fn idle_loop(state: AppState) {
|
||||
|
|
@ -1076,8 +1132,8 @@ async fn pause_idle_computers(state: &AppState) {
|
|||
if computer_has_active_work(state, &computer.id).await {
|
||||
continue;
|
||||
}
|
||||
if let Some(computer_ref) = computer_ref(&computer) {
|
||||
if state
|
||||
if let Some(computer_ref) = computer_ref(&computer)
|
||||
&& state
|
||||
.sandbox
|
||||
.suspend(&computer_ref, &idle_adapter(&computer, "idle"))
|
||||
.await
|
||||
|
|
@ -1085,7 +1141,6 @@ async fn pause_idle_computers(state: &AppState) {
|
|||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let _ = sqlx::query(
|
||||
"UPDATE computers SET state = 'suspended', updated_at = now()
|
||||
WHERE id = $1 AND state = 'running'",
|
||||
|
|
@ -1234,6 +1289,77 @@ struct ActiveRunRow {
|
|||
step: Option<String>,
|
||||
}
|
||||
|
||||
pub fn _keep_state(state: ComputerState, holder: ControlHolder, mode: BrowserProfileMode) {
|
||||
let _ = (state, holder, mode);
|
||||
#[cfg(test)]
|
||||
mod shared_input_tests {
|
||||
use super::*;
|
||||
|
||||
fn desktop() -> (ComputerRow, ScreenRow) {
|
||||
let computer = ComputerRow {
|
||||
id: "computer".into(),
|
||||
space_id: "space".into(),
|
||||
user_id: "user".into(),
|
||||
scope: "team".into(),
|
||||
scope_key: "team:space".into(),
|
||||
home_key: "home".into(),
|
||||
home_revision: "1".into(),
|
||||
kind: "docker".into(),
|
||||
provider_ref: Some("container".into()),
|
||||
state: "running".into(),
|
||||
control_holder: "bot".into(),
|
||||
control_lease_id: None,
|
||||
control_lease_expires_at: None,
|
||||
control_bot_id: Some("bot".into()),
|
||||
control_run_id: None,
|
||||
execution_run_id: Some("run".into()),
|
||||
execution_bot_id: Some("bot".into()),
|
||||
execution_lease_expires_at: None,
|
||||
execution_fence: 1,
|
||||
browser_profile_mode: "per-bot".into(),
|
||||
};
|
||||
let screen = ScreenRow {
|
||||
id: "screen".into(),
|
||||
computer_id: computer.id.clone(),
|
||||
bot_id: "bot".into(),
|
||||
slot: 1,
|
||||
display: ":1".into(),
|
||||
view_port: 6080,
|
||||
profile_mode: "per-bot".into(),
|
||||
profile_path: "/tmp/profile".into(),
|
||||
control_holder: "bot".into(),
|
||||
control_lease_id: None,
|
||||
control_lease_expires_at: None,
|
||||
execution_run_id: Some("run".into()),
|
||||
execution_lease_expires_at: None,
|
||||
execution_fence: 1,
|
||||
};
|
||||
(computer, screen)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn human_input_does_not_require_or_change_the_agent_lease() {
|
||||
let (computer, mut screen) = desktop();
|
||||
for holder in ["bot", "none", "user"] {
|
||||
screen.control_holder = holder.into();
|
||||
assert!(user_can_interact(&computer, Some(&screen), "bot"));
|
||||
assert_eq!(screen.execution_run_id.as_deref(), Some("run"));
|
||||
assert!(status_from("bot", &computer, Some(&screen), None).shared_input);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shared_input_requires_the_bots_own_live_screen() {
|
||||
let (mut computer, mut screen) = desktop();
|
||||
assert!(!user_can_interact(&computer, None, "bot"));
|
||||
assert!(!user_can_interact(&computer, Some(&screen), "other-bot"));
|
||||
screen.computer_id = "other-computer".into();
|
||||
assert!(!user_can_interact(&computer, Some(&screen), "bot"));
|
||||
screen.computer_id = computer.id.clone();
|
||||
for state in ["stopped", "booting", "suspended", "error"] {
|
||||
computer.state = state.into();
|
||||
assert!(!user_can_interact(&computer, Some(&screen), "bot"));
|
||||
}
|
||||
computer.state = "running".into();
|
||||
computer.provider_ref = None;
|
||||
assert!(!user_can_interact(&computer, Some(&screen), "bot"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,9 @@ fn valid_name(name: &str) -> bool {
|
|||
&& name != "goal"
|
||||
&& name != "skills"
|
||||
&& name != "help"
|
||||
&& name.bytes().all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
|
||||
&& name
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
|
||||
}
|
||||
|
||||
fn parse_file(path: PathBuf, name: String) -> Option<FileSkill> {
|
||||
|
|
@ -33,30 +35,56 @@ fn parse_file(path: PathBuf, name: String) -> Option<FileSkill> {
|
|||
let (header, body) = rest.split_once("\n---")?;
|
||||
(
|
||||
header,
|
||||
body.trim_start_matches(|ch| ch == '\n' || ch == '\r')
|
||||
.trim()
|
||||
.to_string(),
|
||||
body.trim_start_matches(['\n', '\r']).trim().to_string(),
|
||||
)
|
||||
} else {
|
||||
("", text.trim().to_string())
|
||||
};
|
||||
let description = frontmatter.lines().find_map(|line| {
|
||||
let description = frontmatter
|
||||
.lines()
|
||||
.find_map(|line| {
|
||||
let (key, value) = line.split_once(':')?;
|
||||
(key.trim() == "description")
|
||||
.then(|| value.trim().trim_matches(|ch| ch == '"' || ch == '\'').to_string())
|
||||
}).unwrap_or_else(|| instructions.lines().next().unwrap_or("自訂技能").chars().take(120).collect());
|
||||
(!instructions.is_empty()).then_some(FileSkill { name, description, instructions })
|
||||
(key.trim() == "description").then(|| {
|
||||
value
|
||||
.trim()
|
||||
.trim_matches(|ch| ch == '"' || ch == '\'')
|
||||
.to_string()
|
||||
})
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
instructions
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap_or("自訂技能")
|
||||
.chars()
|
||||
.take(120)
|
||||
.collect()
|
||||
});
|
||||
(!instructions.is_empty()).then_some(FileSkill {
|
||||
name,
|
||||
description,
|
||||
instructions,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn list(data_dir: &str) -> Vec<FileSkill> {
|
||||
let root = Path::new(data_dir).join("skills");
|
||||
let Ok(entries) = std::fs::read_dir(&root) else { return Vec::new() };
|
||||
let mut skills = entries.filter_map(Result::ok).filter_map(|entry| {
|
||||
let Ok(entries) = std::fs::read_dir(&root) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut skills = entries
|
||||
.filter_map(Result::ok)
|
||||
.filter_map(|entry| {
|
||||
let file_type = entry.file_type().ok()?;
|
||||
if !file_type.is_dir() { return None; }
|
||||
if !file_type.is_dir() {
|
||||
return None;
|
||||
}
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
valid_name(&name).then(|| parse_file(entry.path().join("SKILL.md"), name)).flatten()
|
||||
}).collect::<Vec<_>>();
|
||||
valid_name(&name)
|
||||
.then(|| parse_file(entry.path().join("SKILL.md"), name))
|
||||
.flatten()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
skills.sort_by(|left, right| left.name.cmp(&right.name));
|
||||
skills
|
||||
}
|
||||
|
|
@ -64,15 +92,25 @@ pub fn list(data_dir: &str) -> Vec<FileSkill> {
|
|||
pub fn slash(prompt: &str, data_dir: &str) -> Option<(FileSkill, String)> {
|
||||
let mut words = prompt.trim().splitn(2, char::is_whitespace);
|
||||
let command = words.next()?.strip_prefix('/')?;
|
||||
if !valid_name(command) { return None; }
|
||||
let skill = list(data_dir).into_iter().find(|skill| skill.name == command)?;
|
||||
if !valid_name(command) {
|
||||
return None;
|
||||
}
|
||||
let skill = list(data_dir)
|
||||
.into_iter()
|
||||
.find(|skill| skill.name == command)?;
|
||||
Some((skill, words.next().unwrap_or("").trim().to_string()))
|
||||
}
|
||||
|
||||
pub fn index(data_dir: &str) -> String {
|
||||
let skills = list(data_dir);
|
||||
if skills.is_empty() { return String::new(); }
|
||||
let lines = skills.iter().map(|skill| format!("/{:<18} {}", skill.name, skill.description)).collect::<Vec<_>>().join("\n");
|
||||
if skills.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
let lines = skills
|
||||
.iter()
|
||||
.map(|skill| format!("/{:<18} {}", skill.name, skill.description))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
format!("可用的檔案技能（唯讀）：\n{lines}")
|
||||
}
|
||||
|
||||
|
|
@ -83,12 +121,20 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn loads_only_safe_skill_names_and_resolves_arguments() {
|
||||
let root = std::env::temp_dir().join(format!("lazyboy-file-skills-{}", uuid::Uuid::new_v4()));
|
||||
let root =
|
||||
std::env::temp_dir().join(format!("lazyboy-file-skills-{}", uuid::Uuid::new_v4()));
|
||||
fs::create_dir_all(root.join("skills/open-site")).unwrap();
|
||||
fs::write(root.join("skills/open-site/SKILL.md"), "---\ndescription: Open a site\n---\nUse the browser.\n").unwrap();
|
||||
fs::write(
|
||||
root.join("skills/open-site/SKILL.md"),
|
||||
"---\ndescription: Open a site\n---\nUse the browser.\n",
|
||||
)
|
||||
.unwrap();
|
||||
fs::create_dir_all(root.join("skills/goal")).unwrap();
|
||||
let root_text = root.to_str().unwrap();
|
||||
assert_eq!(slash("/open-site example.com", root_text).unwrap().1, "example.com");
|
||||
assert_eq!(
|
||||
slash("/open-site example.com", root_text).unwrap().1,
|
||||
"example.com"
|
||||
);
|
||||
assert!(slash("/goal do it", root_text).is_none());
|
||||
assert!(slash("/missing", root_text).is_none());
|
||||
let _ = fs::remove_dir_all(root);
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ mod file_skills;
|
|||
mod mcp;
|
||||
mod mcp_catalog;
|
||||
mod memory;
|
||||
mod monitor;
|
||||
mod retention;
|
||||
mod rooms;
|
||||
mod routes;
|
||||
|
|
@ -56,7 +57,9 @@ async fn main() {
|
|||
});
|
||||
|
||||
let retention_state = state.clone();
|
||||
tokio::spawn(async move { retention::retention_loop(retention_state).await; });
|
||||
tokio::spawn(async move {
|
||||
retention::retention_loop(retention_state).await;
|
||||
});
|
||||
|
||||
let worker_state = state.clone();
|
||||
tokio::spawn(async move {
|
||||
|
|
|
|||
|
|
@ -570,7 +570,7 @@ async fn create_server(
|
|||
internal(error.to_string())
|
||||
}
|
||||
})?;
|
||||
let mut row = load_row(state.pool(), &actor, &id)
|
||||
let row = load_row(state.pool(), &actor, &id)
|
||||
.await
|
||||
.map_err(|error| internal(error.to_string()))?
|
||||
.ok_or_else(|| internal("missing row".into()))?;
|
||||
|
|
|
|||
|
|
@ -28,7 +28,9 @@ pub struct MemoryService {
|
|||
|
||||
enum ModelState {
|
||||
Uninitialized,
|
||||
Ready(TextEmbedding),
|
||||
// Boxed: the embedding model is far larger than the other two variants and
|
||||
// this enum lives inside an Arc<Mutex<..>> shared by every request.
|
||||
Ready(Box<TextEmbedding>),
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
|
|
@ -102,7 +104,7 @@ impl MemoryService {
|
|||
.with_cache_dir(cache_dir)
|
||||
.with_show_download_progress(false);
|
||||
match TextEmbedding::try_new(options) {
|
||||
Ok(embedding) => *state = ModelState::Ready(embedding),
|
||||
Ok(embedding) => *state = ModelState::Ready(Box::new(embedding)),
|
||||
Err(error) => {
|
||||
*state = ModelState::Unavailable;
|
||||
return Err(format!("FastEmbed unavailable: {error}"));
|
||||
|
|
@ -151,7 +153,7 @@ impl MemoryService {
|
|||
let content = validate_content(&input.content)?;
|
||||
validate_importance(input.importance)?;
|
||||
let embedding = self.embed(content.clone()).await;
|
||||
let vector = embedding.as_ref().map(vector_literal);
|
||||
let vector = embedding.as_deref().map(vector_literal);
|
||||
let id = Uuid::new_v4();
|
||||
let mut tx = pool.begin().await.map_err(|error| error.to_string())?;
|
||||
let item: MemoryItem = sqlx::query_as(
|
||||
|
|
@ -212,7 +214,7 @@ impl MemoryService {
|
|||
}
|
||||
let limit = limit.unwrap_or(self.top_k).clamp(1, 50);
|
||||
let embedding = self.embed(query.to_string()).await;
|
||||
let rows = if let Some(vector) = embedding.as_ref().map(vector_literal) {
|
||||
let rows = if let Some(vector) = embedding.as_deref().map(vector_literal) {
|
||||
sqlx::query_as(
|
||||
"SELECT id,session_id,source_run_id,source_message_id,content,importance,revision,
|
||||
created_at,updated_at,deleted_at
|
||||
|
|
@ -269,7 +271,7 @@ impl MemoryService {
|
|||
let vector = self
|
||||
.embed(content.clone())
|
||||
.await
|
||||
.as_ref()
|
||||
.as_deref()
|
||||
.map(vector_literal);
|
||||
let mut tx = pool.begin().await.map_err(|error| error.to_string())?;
|
||||
let item: Option<MemoryItem> = sqlx::query_as(
|
||||
|
|
@ -377,7 +379,7 @@ async fn insert_revision(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn vector_literal(vector: &Vec<f32>) -> String {
|
||||
fn vector_literal(vector: &[f32]) -> String {
|
||||
format!(
|
||||
"[{}]",
|
||||
vector
|
||||
|
|
|
|||
|
|
@ -0,0 +1,540 @@
|
|||
//! Observability surface for a run: the activity trail the chat bubble reads,
|
||||
//! and the plain-language reading of a failure that turns into an action item.
|
||||
use axum::extract::{Path, Query, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Json, Router};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::db::Actor;
|
||||
use crate::state::AppState;
|
||||
|
||||
type ApiError = (StatusCode, Json<Value>);
|
||||
|
||||
/// Longest a single string inside an activity payload may get. The trail is a
|
||||
/// glanceable summary, not a second copy of the transcript.
|
||||
const MAX_STRING_CHARS: usize = 512;
|
||||
const DEFAULT_ACTIVITY_LIMIT: i32 = 60;
|
||||
const MAX_ACTIVITY_LIMIT: i32 = 200;
|
||||
|
||||
pub fn router() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/api/runs/{id}/activity", get(activity))
|
||||
.route("/api/runs/{id}/retry", post(retry))
|
||||
}
|
||||
|
||||
/// Append one line to the run's trail. Diagnostics never fail a run: a write
|
||||
/// that cannot land is logged and dropped.
|
||||
pub async fn record(state: &AppState, run_id: &str, kind: &str, payload: Value) {
|
||||
let result = sqlx::query("INSERT INTO run_activity (run_id,kind,payload) VALUES ($1,$2,$3)")
|
||||
.bind(run_id)
|
||||
.bind(kind)
|
||||
.bind(clamp_strings(&payload, MAX_STRING_CHARS))
|
||||
.execute(state.pool())
|
||||
.await;
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(run_id, kind, "failed to record run activity: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
/// One line of text for the trail: single line, bounded, never an image.
|
||||
pub fn snippet(text: &str, max_chars: usize) -> String {
|
||||
let flat = text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if flat.chars().count() <= max_chars {
|
||||
return flat;
|
||||
}
|
||||
let mut head: String = flat.chars().take(max_chars).collect();
|
||||
head.push('…');
|
||||
head
|
||||
}
|
||||
|
||||
/// Bound every string leaf so one chatty tool result cannot bloat the trail.
|
||||
pub fn clamp_strings(value: &Value, max_chars: usize) -> Value {
|
||||
match value {
|
||||
Value::String(text) if text.chars().count() > max_chars => {
|
||||
let mut head: String = text.chars().take(max_chars).collect();
|
||||
head.push('…');
|
||||
Value::String(head)
|
||||
}
|
||||
Value::Array(items) => Value::Array(
|
||||
items
|
||||
.iter()
|
||||
.map(|item| clamp_strings(item, max_chars))
|
||||
.collect(),
|
||||
),
|
||||
Value::Object(map) => Value::Object(
|
||||
map.iter()
|
||||
.map(|(key, item)| (key.clone(), clamp_strings(item, max_chars)))
|
||||
.collect(),
|
||||
),
|
||||
other => other.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// What the user is told, and what they can do about it. `code` is the stable
|
||||
/// contract the chat card renders from; the Chinese strings are the fallback
|
||||
/// body that also lives in the message history.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RunFailure {
|
||||
pub code: &'static str,
|
||||
pub headline: &'static str,
|
||||
pub action: &'static str,
|
||||
pub retryable: bool,
|
||||
pub needs_settings: bool,
|
||||
pub needs_computer: bool,
|
||||
}
|
||||
|
||||
impl RunFailure {
|
||||
fn new(
|
||||
code: &'static str,
|
||||
headline: &'static str,
|
||||
action: &'static str,
|
||||
retryable: bool,
|
||||
needs_settings: bool,
|
||||
needs_computer: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
code,
|
||||
headline,
|
||||
action,
|
||||
retryable,
|
||||
needs_settings,
|
||||
needs_computer,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Turn a raw run error into a classified, human-readable failure. Ordering is
|
||||
/// the policy: the most specific and most actionable cause wins, so a 429 that
|
||||
/// also mentions a timeout is reported as a quota problem, not a slow network.
|
||||
pub fn classify_run_error(error: &str) -> RunFailure {
|
||||
let text = error.to_lowercase();
|
||||
let has = |needles: &[&str]| needles.iter().any(|needle| text.contains(needle));
|
||||
if has(&["worker interrupted"]) {
|
||||
return RunFailure::new(
|
||||
"interrupted",
|
||||
"我剛剛被打斷了，最後那個動作可能已經做下去了。",
|
||||
"先按重試；不確定的話，打開它的畫面確認現狀再繼續。",
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
}
|
||||
if text.contains("tool ") && text.contains("timed out") {
|
||||
return RunFailure::new(
|
||||
"tool_timeout",
|
||||
"有一個動作卡超過 150 秒，我無法確定它有沒有做完。",
|
||||
"先打開它的畫面看一眼，再按重試；我不會重複已經成功的步驟。",
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
}
|
||||
if has(&[
|
||||
"invalid api key",
|
||||
"incorrect api key",
|
||||
"unauthorized",
|
||||
"authentication",
|
||||
"forbidden",
|
||||
"401",
|
||||
"403",
|
||||
]) {
|
||||
return RunFailure::new(
|
||||
"model_key",
|
||||
"模型不認得這個 API 金鑰。",
|
||||
"到「設定 → 模型」重新貼一次金鑰，再按重試。",
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
);
|
||||
}
|
||||
if has(&[
|
||||
"429",
|
||||
"rate limit",
|
||||
"ratelimit",
|
||||
"too many requests",
|
||||
"insufficient",
|
||||
"quota",
|
||||
"credit",
|
||||
"billing",
|
||||
]) {
|
||||
return RunFailure::new(
|
||||
"model_quota",
|
||||
"模型那邊限流了，或是額度已經用完。",
|
||||
"等一下再按重試；如果是額度用完，到「設定 → 模型」換一個或加值。",
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
);
|
||||
}
|
||||
if has(&[
|
||||
"model not found",
|
||||
"unknown model",
|
||||
"invalid model",
|
||||
"does not exist",
|
||||
"404",
|
||||
]) {
|
||||
return RunFailure::new(
|
||||
"model_unknown",
|
||||
"這個模型名稱找不到，可能已下架或打錯了。",
|
||||
"到「設定 → 模型」選一個存在的模型，再按重試。",
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
);
|
||||
}
|
||||
if has(&["timed out", "timeout", "逾時"]) {
|
||||
return RunFailure::new(
|
||||
"model_timeout",
|
||||
"模型太久沒有回話（超過 165 秒）。",
|
||||
"先按重試；每次都發生的話，到「設定 → 模型」換一個快一點的模型。",
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
);
|
||||
}
|
||||
if has(&[
|
||||
"error sending request",
|
||||
"failed to connect",
|
||||
"connection refused",
|
||||
"dns",
|
||||
"host not found",
|
||||
"certificate",
|
||||
"tls",
|
||||
"network",
|
||||
]) {
|
||||
return RunFailure::new(
|
||||
"network",
|
||||
"連不到模型服務，網路或位址不對。",
|
||||
"確認網路與「設定 → 模型」的 Base URL，再按重試。",
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
);
|
||||
}
|
||||
if has(&[
|
||||
"no such container",
|
||||
"container",
|
||||
"docker",
|
||||
"computer not found",
|
||||
"computer is not running",
|
||||
]) {
|
||||
return RunFailure::new(
|
||||
"computer_gone",
|
||||
"它的電腦不見了或已經停止。",
|
||||
"先把電腦啟動，再按重試。",
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
}
|
||||
if text.contains("lease") {
|
||||
return RunFailure::new(
|
||||
"lease_lost",
|
||||
"這份工作被另一邊接手過，我手上這份已經失效。",
|
||||
"按重試重新開始這一步就好。",
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
}
|
||||
RunFailure::new(
|
||||
"unknown",
|
||||
"我卡住了，這一輪沒有完成。",
|
||||
"按重試看看；需要的話把下面的記錄複製起來給我。",
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
/// The sentence that goes into the chat: what happened, how far the run got,
|
||||
/// and the one thing the human can do next.
|
||||
pub fn failure_message(failure: &RunFailure, last_step: Option<&str>, turn: Option<i64>) -> String {
|
||||
let progress = match (last_step, turn) {
|
||||
(Some(step), Some(n)) if n > 0 && !step.trim().is_empty() => {
|
||||
format!("我最後在做：{}（第 {n} 輪）。", snippet(step, 80))
|
||||
}
|
||||
(Some(step), _) if !step.trim().is_empty() => {
|
||||
format!("我最後在做：{}。", snippet(step, 80))
|
||||
}
|
||||
_ => "我還沒有開始動手。".to_string(),
|
||||
};
|
||||
format!("{} {} {}", failure.headline, progress, failure.action)
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
struct ActivityQuery {
|
||||
after: Option<i64>,
|
||||
limit: Option<i32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
struct RunRow {
|
||||
status: String,
|
||||
step: Option<String>,
|
||||
turn: Option<i64>,
|
||||
turn_limit: Option<i64>,
|
||||
error: Option<String>,
|
||||
step_at: Option<DateTime<Utc>>,
|
||||
started_at: Option<DateTime<Utc>>,
|
||||
completed_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
async fn actor(state: &AppState) -> Result<Actor, ApiError> {
|
||||
state.bootstrap().await.map_err(|error| {
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(json!({"message": error.to_string()})),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// The bubble's whole payload: where the run stands right now plus its trail.
|
||||
/// `after` pages forward; the first read returns the newest window, oldest
|
||||
/// first, so the UI renders it and then tails from the last id.
|
||||
async fn activity(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<String>,
|
||||
Query(query): Query<ActivityQuery>,
|
||||
) -> Result<Json<Value>, ApiError> {
|
||||
let actor = actor(&state).await?;
|
||||
let run = sqlx::query_as::<_, RunRow>(
|
||||
"SELECT status, checkpoint->>'step' AS step, (checkpoint->>'turn')::bigint AS turn,
|
||||
(checkpoint->>'turnLimit')::bigint AS turn_limit, error, (checkpoint->>'stepAt')::timestamptz AS step_at, started_at, completed_at
|
||||
FROM runs WHERE id=$1 AND space_id=$2 AND user_id=$3",
|
||||
)
|
||||
.bind(&id)
|
||||
.bind(&actor.space_id)
|
||||
.bind(&actor.user_id)
|
||||
.fetch_optional(state.pool())
|
||||
.await
|
||||
.map_err(internal)?;
|
||||
let Some(run) = run else {
|
||||
return Err(not_found("run not found"));
|
||||
};
|
||||
let limit = query
|
||||
.limit
|
||||
.unwrap_or(DEFAULT_ACTIVITY_LIMIT)
|
||||
.clamp(1, MAX_ACTIVITY_LIMIT);
|
||||
let rows = sqlx::query_as::<_, (i64, String, Value, DateTime<Utc>)>(
|
||||
"SELECT id, kind, payload, created_at FROM (
|
||||
SELECT id, kind, payload, created_at FROM run_activity
|
||||
WHERE run_id=$1 AND ($2::bigint IS NULL OR id>$2)
|
||||
ORDER BY id DESC LIMIT $3
|
||||
) recent ORDER BY id ASC",
|
||||
)
|
||||
.bind(&id)
|
||||
.bind(query.after)
|
||||
.bind(limit)
|
||||
.fetch_all(state.pool())
|
||||
.await
|
||||
.map_err(internal)?;
|
||||
let error = run
|
||||
.error
|
||||
.as_deref()
|
||||
.filter(|raw| !raw.trim().is_empty())
|
||||
.map(|raw| {
|
||||
let failure = classify_run_error(raw);
|
||||
json!({"code": failure.code, "headline": failure.headline, "action": failure.action, "raw": raw})
|
||||
});
|
||||
let elapsed = run.started_at.map(|started| {
|
||||
(run.completed_at.unwrap_or_else(Utc::now) - started)
|
||||
.num_milliseconds()
|
||||
.max(0) as u64
|
||||
});
|
||||
Ok(Json(json!({
|
||||
"runId": id,
|
||||
"status": run.status,
|
||||
"turn": run.turn,
|
||||
"turnLimit": run.turn_limit,
|
||||
"step": run.step,
|
||||
"elapsedMs": elapsed,
|
||||
"stepAt": run.step_at,
|
||||
"error": error,
|
||||
"activity": rows.into_iter().map(|(row_id, kind, payload, created_at)| {
|
||||
let mut entry = payload;
|
||||
if let Some(map) = entry.as_object_mut() {
|
||||
map.insert("id".to_string(), json!(row_id));
|
||||
map.insert("kind".to_string(), json!(kind));
|
||||
map.insert("createdAt".to_string(), json!(created_at.to_rfc3339()));
|
||||
}
|
||||
entry
|
||||
}).collect::<Vec<_>>(),
|
||||
})))
|
||||
}
|
||||
|
||||
/// Re-queue a run that stopped on an error, keeping its harness checkpoint so
|
||||
/// it continues where it stopped instead of starting the task over.
|
||||
async fn retry(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<String>,
|
||||
) -> Result<Json<Value>, ApiError> {
|
||||
let actor = actor(&state).await?;
|
||||
let exists: Option<String> =
|
||||
sqlx::query_scalar("SELECT status FROM runs WHERE id=$1 AND space_id=$2 AND user_id=$3")
|
||||
.bind(&id)
|
||||
.bind(&actor.space_id)
|
||||
.bind(&actor.user_id)
|
||||
.fetch_optional(state.pool())
|
||||
.await
|
||||
.map_err(internal)?;
|
||||
if exists.is_none() {
|
||||
return Err(not_found("run not found"));
|
||||
}
|
||||
let queued: Option<String> = sqlx::query_scalar(
|
||||
"UPDATE runs SET status='queued', retry_count=0, error=NULL, completed_at=NULL,
|
||||
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||||
WHERE id=$1 AND space_id=$2 AND user_id=$3
|
||||
AND status IN ('failed','cancelled')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM runs a
|
||||
WHERE a.bot_id=runs.bot_id AND a.id<>runs.id
|
||||
AND a.status IN ('leased','running','waiting_input','waiting_takeover')
|
||||
AND (a.lease_expires_at IS NULL OR a.lease_expires_at >= now())
|
||||
)
|
||||
RETURNING thread_id",
|
||||
)
|
||||
.bind(&id)
|
||||
.bind(&actor.space_id)
|
||||
.bind(&actor.user_id)
|
||||
.fetch_optional(state.pool())
|
||||
.await
|
||||
.map_err(internal)?;
|
||||
if queued.is_none() {
|
||||
return Err((
|
||||
StatusCode::CONFLICT,
|
||||
Json(json!({"message": "run is not retryable"})),
|
||||
));
|
||||
}
|
||||
record(&state, &id, "run", json!({"event": "retry"})).await;
|
||||
tracing::info!(run_id = id.as_str(), "run re-queued from the chat");
|
||||
Ok(Json(json!({"ok": true, "runId": id})))
|
||||
}
|
||||
|
||||
fn not_found(message: &str) -> ApiError {
|
||||
(StatusCode::NOT_FOUND, Json(json!({"message": message})))
|
||||
}
|
||||
|
||||
fn internal(error: sqlx::Error) -> ApiError {
|
||||
tracing::error!("monitor: {error}");
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(json!({"message": "internal error"})),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{RunFailure, clamp_strings, classify_run_error, failure_message, snippet};
|
||||
use serde_json::json;
|
||||
|
||||
fn code(error: &str) -> String {
|
||||
classify_run_error(error).code.to_string()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_failure_offers_an_action_and_a_code() {
|
||||
for error in [
|
||||
"boom",
|
||||
"HTTP 401",
|
||||
"model request timed out after 165 seconds",
|
||||
"no such container",
|
||||
] {
|
||||
let failure = classify_run_error(error);
|
||||
assert!(!failure.code.is_empty(), "{error}");
|
||||
assert!(!failure.action.is_empty(), "{error}");
|
||||
assert!(!failure.headline.is_empty(), "{error}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auth_quota_and_unknown_model_are_told_apart() {
|
||||
assert_eq!(
|
||||
code("Api error 401 Unauthorized: invalid api key"),
|
||||
"model_key"
|
||||
);
|
||||
assert_eq!(code("HTTP status 403 Forbidden"), "model_key");
|
||||
assert_eq!(code("status 429 Too Many Requests"), "model_quota");
|
||||
assert_eq!(code("insufficient quota for this project"), "model_quota");
|
||||
assert_eq!(code("The model gpt-9 does not exist"), "model_unknown");
|
||||
assert_eq!(code("HTTP status 404 Not Found"), "model_unknown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_specific_timeouts_win_over_the_generic_one() {
|
||||
assert_eq!(
|
||||
code("tool browser timed out after 150 seconds"),
|
||||
"tool_timeout"
|
||||
);
|
||||
assert_eq!(
|
||||
code("model request timed out after 165 seconds"),
|
||||
"model_timeout"
|
||||
);
|
||||
assert_eq!(code("AI 回應逾時（150 秒）"), "model_timeout");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn infrastructure_and_network_failures_are_separate_from_the_model() {
|
||||
assert_eq!(code("no such container: lazyboy-bot-1"), "computer_gone");
|
||||
assert_eq!(code("computer not found"), "computer_gone");
|
||||
assert_eq!(code("error sending request: dns failure"), "network");
|
||||
assert_eq!(code("run lease was lost"), "lease_lost");
|
||||
assert_eq!(
|
||||
code("Worker interrupted after tool execution; inspect current state"),
|
||||
"interrupted"
|
||||
);
|
||||
assert_eq!(code("something odd happened"), "unknown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_retryable_answer_never_promises_a_setting_that_cannot_help() {
|
||||
assert!(!classify_run_error("401 invalid api key").retryable);
|
||||
assert!(classify_run_error("401 invalid api key").needs_settings);
|
||||
assert!(classify_run_error("no such container").needs_computer);
|
||||
assert!(!classify_run_error("no such container").needs_settings);
|
||||
assert!(classify_run_error("status 429 rate limit").retryable);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failure_sentence_names_the_last_step_and_turn() {
|
||||
let failure = RunFailure::new("tool_timeout", "卡住了。", "按重試。", true, false, false);
|
||||
let said = failure_message(&failure, Some("browser: click #12"), Some(7));
|
||||
assert!(said.contains("卡住了。"), "{said}");
|
||||
assert!(said.contains("browser: click #12"), "{said}");
|
||||
assert!(said.contains("第 7 輪"), "{said}");
|
||||
assert!(said.contains("按重試。"), "{said}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failure_before_any_work_says_so_instead_of_inventing_a_step() {
|
||||
let failure = RunFailure::new("model_key", "金鑰不對。", "改金鑰。", false, true, false);
|
||||
assert!(failure_message(&failure, None, None).contains("我還沒有開始動手。"));
|
||||
assert!(failure_message(&failure, Some(" "), Some(0)).contains("我還沒有開始動手。"));
|
||||
assert!(failure_message(&failure, Some("思考中"), None).contains("思考中"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snippets_stay_single_line_and_bounded() {
|
||||
assert_eq!(snippet(" 多行\n文字\t在這裡 ", 40), "多行 文字 在這裡");
|
||||
let long = "字".repeat(500);
|
||||
let cut = snippet(&long, 200);
|
||||
assert_eq!(cut.chars().count(), 201);
|
||||
assert!(cut.ends_with('…'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn payloads_are_clamped_at_every_string_leaf() {
|
||||
let clamped = clamp_strings(
|
||||
&json!({"error": "x".repeat(900), "nested": {"raw": "y".repeat(900)}, "n": 7}),
|
||||
50,
|
||||
);
|
||||
assert_eq!(clamped["error"].as_str().unwrap().chars().count(), 51);
|
||||
assert_eq!(
|
||||
clamped["nested"]["raw"].as_str().unwrap().chars().count(),
|
||||
51
|
||||
);
|
||||
assert_eq!(clamped["n"].as_i64(), Some(7));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,23 +1,60 @@
|
|||
//! Bounded maintenance of expendable diagnostics, never user-authored content.
|
||||
use std::time::{Duration};
|
||||
use sqlx::PgPool;
|
||||
use crate::state::AppState;
|
||||
use sqlx::PgPool;
|
||||
use std::time::Duration;
|
||||
|
||||
fn days(name: &str, default: i32) -> i32 {
|
||||
std::env::var(name).ok().and_then(|v| v.parse::<i32>().ok())
|
||||
.filter(|v| (1..=3650).contains(v)).unwrap_or(default)
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<i32>().ok())
|
||||
.filter(|v| (1..=3650).contains(v))
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
pub async fn retention_loop(state: AppState) {
|
||||
let recording_days = days("LAZYBOY_RECORDING_RETENTION_DAYS", 30);
|
||||
let rules = [
|
||||
("events", include_str!("retention/events.sql"), days("LAZYBOY_EVENT_RETENTION_DAYS", 30)),
|
||||
("checkpoints", include_str!("retention/checkpoints.sql"), days("LAZYBOY_CHECKPOINT_RETENTION_DAYS", 7)),
|
||||
("runs", include_str!("retention/runs.sql"), days("LAZYBOY_RUN_RETENTION_DAYS", 90)),
|
||||
("recordings", include_str!("retention/recordings.sql"), days("LAZYBOY_RECORDING_RETENTION_DAYS", 30)),
|
||||
("revisions", include_str!("retention/revisions.sql"), days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90)),
|
||||
("deleted_memories", include_str!("retention/deleted_memories.sql"), days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90)),
|
||||
(
|
||||
"events",
|
||||
include_str!("retention/events.sql"),
|
||||
days("LAZYBOY_EVENT_RETENTION_DAYS", 30),
|
||||
),
|
||||
(
|
||||
"checkpoints",
|
||||
include_str!("retention/checkpoints.sql"),
|
||||
days("LAZYBOY_CHECKPOINT_RETENTION_DAYS", 7),
|
||||
),
|
||||
(
|
||||
"runs",
|
||||
include_str!("retention/runs.sql"),
|
||||
days("LAZYBOY_RUN_RETENTION_DAYS", 90),
|
||||
),
|
||||
(
|
||||
"run_activity",
|
||||
include_str!("retention/run_activity.sql"),
|
||||
days("LAZYBOY_RUN_ACTIVITY_RETENTION_DAYS", 7),
|
||||
),
|
||||
(
|
||||
"recordings",
|
||||
include_str!("retention/recordings.sql"),
|
||||
recording_days,
|
||||
),
|
||||
(
|
||||
"revisions",
|
||||
include_str!("retention/revisions.sql"),
|
||||
days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90),
|
||||
),
|
||||
(
|
||||
"deleted_memories",
|
||||
include_str!("retention/deleted_memories.sql"),
|
||||
days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90),
|
||||
),
|
||||
("leases", include_str!("retention/leases.sql"), 7),
|
||||
("profile_locks", include_str!("retention/profile_locks.sql"), 7),
|
||||
(
|
||||
"profile_locks",
|
||||
include_str!("retention/profile_locks.sql"),
|
||||
7,
|
||||
),
|
||||
];
|
||||
loop {
|
||||
for (name, query, age) in rules {
|
||||
|
|
@ -25,20 +62,48 @@ pub async fn retention_loop(state: AppState) {
|
|||
// Limit both transaction size and work per hour; defer excess backlog.
|
||||
for _ in 0..20 {
|
||||
match batch(state.pool(), query, age).await {
|
||||
Ok(count) => { removed += count; if count < 1000 { break; } }
|
||||
Err(error) => { tracing::warn!(name, %error, "retention batch failed; retry next hour"); break; }
|
||||
Ok(count) => {
|
||||
removed += count;
|
||||
if count < 1000 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!(name, %error, "retention batch failed; retry next hour");
|
||||
break;
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
if removed > 0 { tracing::info!(name, rows=removed, "retention cleaned expired diagnostics"); }
|
||||
if removed > 0 {
|
||||
tracing::info!(
|
||||
name,
|
||||
rows = removed,
|
||||
"retention cleaned expired diagnostics"
|
||||
);
|
||||
}
|
||||
if let Err(error) = clean_frames(&state, rules[3].2).await {
|
||||
}
|
||||
if let Err(error) = clean_frames(&state, recording_days).await {
|
||||
tracing::warn!(%error, "recording file retention failed; retry next hour");
|
||||
}
|
||||
if let Ok(bytes) = sqlx::query_scalar::<_, i64>("SELECT pg_database_size(current_database())").fetch_one(state.pool()).await {
|
||||
let warn_mb = std::env::var("LAZYBOY_DB_WARN_MB").ok().and_then(|v| v.parse::<i64>().ok()).filter(|v| *v > 0 && *v < 1_000_000).unwrap_or(1024);
|
||||
if let Ok(bytes) =
|
||||
sqlx::query_scalar::<_, i64>("SELECT pg_database_size(current_database())")
|
||||
.fetch_one(state.pool())
|
||||
.await
|
||||
{
|
||||
let warn_mb = std::env::var("LAZYBOY_DB_WARN_MB")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<i64>().ok())
|
||||
.filter(|v| *v > 0 && *v < 1_000_000)
|
||||
.unwrap_or(1024);
|
||||
tracing::info!(bytes, "database size after retention");
|
||||
if bytes > warn_mb * 1024 * 1024 { tracing::warn!(bytes, warn_mb, "database exceeds configured size warning; review retained conversations and memories"); }
|
||||
if bytes > warn_mb * 1024 * 1024 {
|
||||
tracing::warn!(
|
||||
bytes,
|
||||
warn_mb,
|
||||
"database exceeds configured size warning; review retained conversations and memories"
|
||||
);
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(3600)).await;
|
||||
}
|
||||
|
|
@ -47,16 +112,31 @@ pub async fn retention_loop(state: AppState) {
|
|||
async fn batch(pool: &PgPool, query: &str, age: i32) -> Result<u64, sqlx::Error> {
|
||||
let mut tx = pool.begin().await?;
|
||||
// One maintenance writer, even when multiple API processes start together.
|
||||
let acquired: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(72189431)").fetch_one(&mut *tx).await?;
|
||||
if !acquired { return Ok(0); }
|
||||
sqlx::query("SET LOCAL statement_timeout = '10s'").execute(&mut *tx).await?;
|
||||
sqlx::query("SET LOCAL lock_timeout = '1s'").execute(&mut *tx).await?;
|
||||
let result = sqlx::query(query).bind(age).bind(1000_i64).execute(&mut *tx).await?;
|
||||
let acquired: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(72189431)")
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
if !acquired {
|
||||
return Ok(0);
|
||||
}
|
||||
sqlx::query("SET LOCAL statement_timeout = '10s'")
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query("SET LOCAL lock_timeout = '1s'")
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let result = sqlx::query(query)
|
||||
.bind(age)
|
||||
.bind(1000_i64)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
async fn clean_frames(state: &AppState, age: i32) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
async fn clean_frames(
|
||||
state: &AppState,
|
||||
age: i32,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let root = std::path::Path::new(&state.data_dir).join("teach");
|
||||
let mut dirs = match tokio::fs::read_dir(&root).await {
|
||||
Ok(dirs) => dirs,
|
||||
|
|
@ -65,20 +145,35 @@ async fn clean_frames(state: &AppState, age: i32) -> Result<(), Box<dyn std::err
|
|||
};
|
||||
let mut cleaned = 0;
|
||||
while let Some(entry) = dirs.next_entry().await? {
|
||||
if cleaned >= 1000 { break; }
|
||||
if cleaned >= 1000 {
|
||||
break;
|
||||
}
|
||||
// Ignore symlinks and unexpected names; never traverse a user's home.
|
||||
if !entry.file_type().await?.is_dir() { continue; }
|
||||
if !entry.file_type().await?.is_dir() {
|
||||
continue;
|
||||
}
|
||||
let id = entry.file_name().to_string_lossy().into_owned();
|
||||
if uuid::Uuid::parse_str(&id).is_err() { continue; }
|
||||
if uuid::Uuid::parse_str(&id).is_err() {
|
||||
continue;
|
||||
}
|
||||
let eligible: Option<bool> = sqlx::query_scalar(
|
||||
"SELECT status IN ('saved','failed','draft') AND updated_at < now() - make_interval(days => $2) FROM taught_skills WHERE id=$1"
|
||||
).bind(&id).bind(age).fetch_optional(state.pool()).await?;
|
||||
let old_orphan = eligible.is_none() && entry.metadata().await?.modified()?.elapsed().unwrap_or(Duration::ZERO) > Duration::from_secs(age as u64 * 86400);
|
||||
let old_orphan = eligible.is_none()
|
||||
&& entry
|
||||
.metadata()
|
||||
.await?
|
||||
.modified()?
|
||||
.elapsed()
|
||||
.unwrap_or(Duration::ZERO)
|
||||
> Duration::from_secs(age as u64 * 86400);
|
||||
if eligible == Some(true) || old_orphan {
|
||||
tokio::fs::remove_dir_all(entry.path()).await?;
|
||||
cleaned += 1;
|
||||
}
|
||||
}
|
||||
if cleaned > 0 { tracing::info!(cleaned, "removed expired teaching frame directories"); }
|
||||
if cleaned > 0 {
|
||||
tracing::info!(cleaned, "removed expired teaching frame directories");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
DELETE FROM run_activity WHERE id IN (
|
||||
SELECT id FROM run_activity WHERE created_at < now() - make_interval(days => $1)
|
||||
ORDER BY created_at LIMIT $2 FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
|
|
@ -64,13 +64,21 @@ async fn members_for(state: &AppState, room_id: &str) -> Result<Vec<RoomMember>,
|
|||
Ok(rows.into_iter().map(member_from_row).collect())
|
||||
}
|
||||
|
||||
/// `room_from_id` projection: id, name, last message time, preview, unread count.
|
||||
type RoomSummaryRow = (
|
||||
String,
|
||||
String,
|
||||
Option<chrono::DateTime<chrono::Utc>>,
|
||||
Option<String>,
|
||||
i64,
|
||||
);
|
||||
|
||||
async fn room_from_id(
|
||||
state: &AppState,
|
||||
actor: &Actor,
|
||||
id: &str,
|
||||
) -> Result<Option<Room>, sqlx::Error> {
|
||||
let row: Option<(String, String, Option<chrono::DateTime<chrono::Utc>>, Option<String>, i64)> =
|
||||
sqlx::query_as(
|
||||
let row: Option<RoomSummaryRow> = sqlx::query_as(
|
||||
"SELECT r.id, r.name,
|
||||
(SELECT MAX(m.created_at) FROM messages m JOIN threads t ON t.id=m.thread_id WHERE t.room_id=r.id),
|
||||
(SELECT m.body FROM messages m JOIN threads t ON t.id=m.thread_id
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ pub fn router(state: AppState) -> Router {
|
|||
Router::new()
|
||||
.merge(crate::sessions::router())
|
||||
.merge(crate::memory::router())
|
||||
.merge(crate::monitor::router())
|
||||
.merge(crate::rooms::router())
|
||||
.merge(crate::mcp::router())
|
||||
.merge(crate::workspace::router())
|
||||
|
|
@ -313,6 +314,11 @@ async fn update_bot(
|
|||
Json(json!({"message":"bot not found"})),
|
||||
));
|
||||
}
|
||||
// The saved setting remains authoritative if the desktop is disconnected;
|
||||
// ensure/restore will apply it again before the next run.
|
||||
if let Err(error) = computer::refresh_cursor_color(&state, &actor, &id).await {
|
||||
tracing::warn!(bot_id = %id, %error, "could not refresh cursor color");
|
||||
}
|
||||
Ok(Json(json!({"ok":true})))
|
||||
}
|
||||
|
||||
|
|
@ -347,8 +353,8 @@ async fn delete_bot(
|
|||
if let Some(computer) = computer
|
||||
.as_ref()
|
||||
.filter(|row| parse_mode(&row.scope) == ComputerMode::Dedicated)
|
||||
&& let Some(computer_ref) = computer::computer_ref(computer)
|
||||
{
|
||||
if let Some(computer_ref) = computer::computer_ref(computer) {
|
||||
state
|
||||
.sandbox
|
||||
.destroy(
|
||||
|
|
@ -358,7 +364,6 @@ async fn delete_bot(
|
|||
.await
|
||||
.map_err(|error| bad_gateway(error.to_string()))?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut tx = state.pool().begin().await.map_err(internal_error)?;
|
||||
sqlx::query("DELETE FROM computer_screens WHERE bot_id = $1")
|
||||
|
|
@ -609,7 +614,9 @@ async fn stop(
|
|||
State(state): State<AppState>,
|
||||
Path(id): Path<String>,
|
||||
) -> Result<Json<Value>, (StatusCode, Json<Value>)> {
|
||||
let actor = actor(&state).await.map_err(|status| (status, Json(json!({"message":"無法取得工作區"}))))?;
|
||||
let actor = actor(&state)
|
||||
.await
|
||||
.map_err(|status| (status, Json(json!({"message":"無法取得工作區"}))))?;
|
||||
computer::stop(&state, &actor, &id)
|
||||
.await
|
||||
.map(|status| Json(serde_json::to_value(status).unwrap()))
|
||||
|
|
@ -646,7 +653,7 @@ async fn screen_url(
|
|||
state.db.get_screen(&computer.id, &id).await.ok().flatten()
|
||||
}
|
||||
};
|
||||
let interactive = computer::user_has_screen_control(&computer, screen.as_ref(), &id);
|
||||
let interactive = computer::user_can_interact(&computer, screen.as_ref(), &id);
|
||||
let _ = state
|
||||
.sandbox
|
||||
.connect_screen(
|
||||
|
|
@ -726,44 +733,19 @@ async fn input(
|
|||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
|
||||
.ok_or(StatusCode::NOT_FOUND)?;
|
||||
let screen = state.db.get_screen(&computer.id, &id).await.ok().flatten();
|
||||
if !computer::user_has_screen_control(&computer, screen.as_ref(), &id) {
|
||||
if !computer::user_can_interact(&computer, screen.as_ref(), &id) {
|
||||
return Err(StatusCode::CONFLICT);
|
||||
}
|
||||
let computer_ref = computer::computer_ref(&computer).ok_or(StatusCode::BAD_REQUEST)?;
|
||||
if body.kind == "clipboard" || body.kind == "copy" {
|
||||
let text = body.text.unwrap_or_default();
|
||||
if text.len() > 1024 * 1024 {
|
||||
if body
|
||||
.text
|
||||
.as_ref()
|
||||
.is_some_and(|text| text.len() > 1024 * 1024)
|
||||
{
|
||||
return Err(StatusCode::PAYLOAD_TOO_LARGE);
|
||||
}
|
||||
let context =
|
||||
computer::adapter_context_for(&actor, &id, "clipboard", screen.as_ref(), None);
|
||||
let mut argv =
|
||||
lazyboy_control::paste_command_on(context.display.as_deref().unwrap_or(":1"));
|
||||
if body.kind == "copy" {
|
||||
argv.push("copy".into());
|
||||
}
|
||||
let result = state
|
||||
.sandbox
|
||||
.execute(
|
||||
&computer_ref,
|
||||
lazyboy_control::CommandRequest {
|
||||
argv,
|
||||
cwd: None,
|
||||
timeout_ms: Some(10_000),
|
||||
stdin: Some(text),
|
||||
},
|
||||
&context,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| StatusCode::BAD_GATEWAY)?;
|
||||
if result.code != 0 {
|
||||
return Err(StatusCode::BAD_GATEWAY);
|
||||
}
|
||||
return Ok(Json(
|
||||
json!({"ok":true,"text":if body.kind=="copy" {Some(result.stdout)} else {None}}),
|
||||
));
|
||||
}
|
||||
let action = match body.kind.as_str() {
|
||||
"copy" => lazyboy_contracts::ComputerAction::CopySelection,
|
||||
"key" => lazyboy_contracts::ComputerAction::Key {
|
||||
key: body.key.unwrap_or_default(),
|
||||
modifiers: None,
|
||||
|
|
@ -778,7 +760,7 @@ async fn input(
|
|||
button: Some(lazyboy_contracts::PointerButton::Left),
|
||||
},
|
||||
};
|
||||
state
|
||||
let result = state
|
||||
.sandbox
|
||||
.act(
|
||||
&computer_ref,
|
||||
|
|
@ -793,7 +775,7 @@ async fn input(
|
|||
)
|
||||
.await
|
||||
.map_err(|_| StatusCode::BAD_GATEWAY)?;
|
||||
Ok(Json(json!({ "ok": true })))
|
||||
Ok(Json(json!({ "ok": true, "text": result.clipboard_text })))
|
||||
}
|
||||
|
||||
fn _mode(mode: ComputerMode) {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -457,6 +457,16 @@ async fn enqueue(state: &AppState, row: &ScheduleRow, from_tick: bool) -> Result
|
|||
.map_err(|e| e.to_string())?;
|
||||
}
|
||||
tx.commit().await.map_err(|error| error.to_string())?;
|
||||
// A scheduled run starts with nobody pressing send, so the transcript line
|
||||
// has to announce itself: without this an open chat stays blind to the run
|
||||
// until it happens to poll.
|
||||
let _ = crate::sessions::append_event(
|
||||
state,
|
||||
&thread_id,
|
||||
"message.created",
|
||||
json!({"id":message_id,"seq":seq,"role":"user","body":body,"runId":run_id}),
|
||||
)
|
||||
.await;
|
||||
Ok(run_id)
|
||||
}
|
||||
|
||||
|
|
@ -617,7 +627,7 @@ pub fn describe_cron(expr: &str) -> String {
|
|||
if let Ok(Some(seconds)) = interval_seconds(expr) {
|
||||
return format!("每隔 {} 分鐘（固定間隔）", seconds / 60);
|
||||
}
|
||||
let parts: Vec<&str> = expr.trim().split_whitespace().collect();
|
||||
let parts: Vec<&str> = expr.split_whitespace().collect();
|
||||
if parts.len() != 5 {
|
||||
return expr.to_string();
|
||||
}
|
||||
|
|
@ -625,8 +635,12 @@ pub fn describe_cron(expr: &str) -> String {
|
|||
if expr.trim() == "* * * * *" {
|
||||
return "每分鐘".into();
|
||||
}
|
||||
if let Some(rest) = min.strip_prefix("*/") {
|
||||
if hour == "*" && dom == "*" && month == "*" && dow == "*" {
|
||||
if let Some(rest) = min.strip_prefix("*/")
|
||||
&& hour == "*"
|
||||
&& dom == "*"
|
||||
&& month == "*"
|
||||
&& dow == "*"
|
||||
{
|
||||
return if rest
|
||||
.parse::<u32>()
|
||||
.ok()
|
||||
|
|
@ -637,13 +651,15 @@ pub fn describe_cron(expr: &str) -> String {
|
|||
format!("日曆排程：{expr}")
|
||||
};
|
||||
}
|
||||
}
|
||||
if min == "0" && hour == "*" && dom == "*" && month == "*" && dow == "*" {
|
||||
return "每小時".into();
|
||||
}
|
||||
if min == "0" {
|
||||
if let Some(rest) = hour.strip_prefix("*/") {
|
||||
if dom == "*" && month == "*" && dow == "*" {
|
||||
if min == "0"
|
||||
&& let Some(rest) = hour.strip_prefix("*/")
|
||||
&& dom == "*"
|
||||
&& month == "*"
|
||||
&& dow == "*"
|
||||
{
|
||||
return if rest
|
||||
.parse::<u32>()
|
||||
.ok()
|
||||
|
|
@ -654,8 +670,6 @@ pub fn describe_cron(expr: &str) -> String {
|
|||
format!("日曆排程：{expr}")
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
if min.parse::<u32>().is_ok() && hour.parse::<u32>().is_ok() && month == "*" {
|
||||
let at = format!("{hour:0>2}:{min:0>2}");
|
||||
if dom == "*" && dow == "*" {
|
||||
|
|
|
|||
|
|
@ -44,14 +44,14 @@ async fn proxy(state: AppState, bot_id: String, rest: String, req: Request) -> R
|
|||
return StatusCode::NOT_FOUND.into_response();
|
||||
}
|
||||
let ensure = upgrade || is_viewer_page(&rest) || rest.contains("websockify");
|
||||
let port = match upstream_port(&state, &bot_id, ensure).await {
|
||||
Ok(port) => port,
|
||||
let (host, port) = match upstream_target(&state, &bot_id, ensure).await {
|
||||
Ok(target) => target,
|
||||
Err(status) => return status.into_response(),
|
||||
};
|
||||
if upgrade {
|
||||
return match WebSocketUpgrade::from_request(req, &state).await {
|
||||
Ok(ws) => ws
|
||||
.on_upgrade(move |socket| proxy_socket(socket, port, rest))
|
||||
.on_upgrade(move |socket| proxy_socket(socket, host, port, rest))
|
||||
.into_response(),
|
||||
Err(error) => error.into_response(),
|
||||
};
|
||||
|
|
@ -86,7 +86,11 @@ async fn viewer_page() -> Response {
|
|||
(StatusCode::OK, headers, html).into_response()
|
||||
}
|
||||
|
||||
async fn upstream_port(state: &AppState, bot_id: &str, ensure: bool) -> Result<u16, StatusCode> {
|
||||
async fn upstream_target(
|
||||
state: &AppState,
|
||||
bot_id: &str,
|
||||
ensure: bool,
|
||||
) -> Result<(String, u16), StatusCode> {
|
||||
let actor = state
|
||||
.bootstrap()
|
||||
.await
|
||||
|
|
@ -124,20 +128,43 @@ async fn upstream_port(state: &AppState, bot_id: &str, ensure: bool) -> Result<u
|
|||
.sandbox
|
||||
.connect_screen(
|
||||
&computer_ref,
|
||||
computer::user_has_screen_control(&computer, screen.as_ref(), bot_id),
|
||||
computer::user_can_interact(&computer, screen.as_ref(), bot_id),
|
||||
&computer::adapter_context_for(&actor, bot_id, "view", screen.as_ref(), None),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| StatusCode::BAD_GATEWAY)?;
|
||||
let url = session.url.ok_or(StatusCode::NOT_FOUND)?;
|
||||
let rewritten = rewrite_upstream(&url);
|
||||
let uri: Uri = rewritten.parse().map_err(|_| StatusCode::BAD_GATEWAY)?;
|
||||
uri.port_u16().ok_or(StatusCode::BAD_GATEWAY)
|
||||
upstream_authority(&url)
|
||||
}
|
||||
|
||||
fn rewrite_upstream(url: &str) -> String {
|
||||
let host = std::env::var("LAZYBOY_SCREEN_UPSTREAM").unwrap_or_else(|_| "127.0.0.1".into());
|
||||
url.replace("127.0.0.1", &host).replace("localhost", &host)
|
||||
/// Splits the supervisor's noVNC URL into the host the API must dial and its
|
||||
/// port. The host is a container name on the shared screen network, or loopback
|
||||
/// plus a published host port when the API runs outside Docker.
|
||||
fn upstream_authority(url: &str) -> Result<(String, u16), StatusCode> {
|
||||
let uri: Uri = url.parse().map_err(|_| StatusCode::BAD_GATEWAY)?;
|
||||
let host = uri
|
||||
.host()
|
||||
.map(str::to_string)
|
||||
.ok_or(StatusCode::BAD_GATEWAY)?;
|
||||
let port = uri.port_u16().ok_or(StatusCode::BAD_GATEWAY)?;
|
||||
Ok((dial_host(&host), port))
|
||||
}
|
||||
|
||||
/// Maps a loopback authority onto the host the API can actually reach, which
|
||||
/// matters when the API itself runs inside a container. Container names on the
|
||||
/// shared screen network are dialled verbatim.
|
||||
fn dial_host(authority: &str) -> String {
|
||||
if !is_loopback(authority) {
|
||||
return authority.to_string();
|
||||
}
|
||||
match std::env::var("LAZYBOY_SCREEN_UPSTREAM") {
|
||||
Ok(host) if !host.is_empty() && host != authority => host,
|
||||
_ => authority.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_loopback(host: &str) -> bool {
|
||||
matches!(host, "127.0.0.1" | "localhost" | "::1" | "[::1]")
|
||||
}
|
||||
|
||||
fn safe_asset(rest: &str) -> bool {
|
||||
|
|
@ -173,8 +200,7 @@ async fn trusted_asset(rest: &str) -> Response {
|
|||
}
|
||||
}
|
||||
|
||||
async fn proxy_socket(mut client: WebSocket, port: u16, rest: String) {
|
||||
let host = std::env::var("LAZYBOY_SCREEN_UPSTREAM").unwrap_or_else(|_| "127.0.0.1".into());
|
||||
async fn proxy_socket(mut client: WebSocket, host: String, port: u16, rest: String) {
|
||||
let path = if rest.is_empty() {
|
||||
"websockify".into()
|
||||
} else {
|
||||
|
|
@ -215,10 +241,10 @@ async fn proxy_socket(mut client: WebSocket, port: u16, rest: String) {
|
|||
if client.send(AxumMessage::Text(text.to_string().into())).await.is_err() { break; }
|
||||
}
|
||||
Some(Ok(WsMessage::Ping(data))) => {
|
||||
if client.send(AxumMessage::Ping(data.into())).await.is_err() { break; }
|
||||
if client.send(AxumMessage::Ping(data)).await.is_err() { break; }
|
||||
}
|
||||
Some(Ok(WsMessage::Pong(data))) => {
|
||||
if client.send(AxumMessage::Pong(data.into())).await.is_err() { break; }
|
||||
if client.send(AxumMessage::Pong(data)).await.is_err() { break; }
|
||||
}
|
||||
Some(Ok(WsMessage::Close(_))) | Some(Ok(WsMessage::Frame(_))) | None => break,
|
||||
Some(Err(_)) => break,
|
||||
|
|
@ -245,3 +271,29 @@ mod asset_tests {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod upstream_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn upstream_authority_follows_the_supervisor_url() {
|
||||
assert_eq!(
|
||||
upstream_authority("http://lb-team-local-space:6081/vnc_lite.html?view_only=true")
|
||||
.unwrap(),
|
||||
("lb-team-local-space".to_string(), 6081)
|
||||
);
|
||||
assert_eq!(
|
||||
upstream_authority("http://127.0.0.1:32905/").unwrap(),
|
||||
("127.0.0.1".to_string(), 32905)
|
||||
);
|
||||
assert_eq!(
|
||||
upstream_authority("http://lb-localhost:6082/vnc_lite.html").unwrap(),
|
||||
("lb-localhost".to_string(), 6082)
|
||||
);
|
||||
assert_eq!(
|
||||
upstream_authority("http://127.0.0.1/vnc_lite.html").err(),
|
||||
Some(StatusCode::BAD_GATEWAY)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,11 @@ use crate::state::AppState;
|
|||
|
||||
type ApiError = (StatusCode, Json<Value>);
|
||||
|
||||
/// Safety net for the event stream. A wake normally lands within milliseconds,
|
||||
/// so this poll only exists for the cases a wake cannot cover: a reader that
|
||||
/// lagged behind the channel, or a row written outside this process.
|
||||
const EVENT_FALLBACK_POLL: Duration = Duration::from_secs(5);
|
||||
|
||||
pub fn router() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route(
|
||||
|
|
@ -220,9 +225,7 @@ async fn delete_session(
|
|||
Json(json!({"message":"session not found"})),
|
||||
));
|
||||
}
|
||||
cancel_session_runs(&state, &id)
|
||||
.await
|
||||
.map_err(|error| internal(error))?;
|
||||
cancel_session_runs(&state, &id).await.map_err(internal)?;
|
||||
let mut tx = state
|
||||
.pool()
|
||||
.begin()
|
||||
|
|
@ -286,9 +289,7 @@ async fn clear_messages(
|
|||
Json(json!({"message":"session not found"})),
|
||||
));
|
||||
}
|
||||
cancel_session_runs(&state, &id)
|
||||
.await
|
||||
.map_err(|error| internal(error))?;
|
||||
cancel_session_runs(&state, &id).await.map_err(internal)?;
|
||||
let mut tx = state
|
||||
.pool()
|
||||
.begin()
|
||||
|
|
@ -364,9 +365,7 @@ async fn stop_session(
|
|||
Json(json!({"message":"session not found"})),
|
||||
));
|
||||
}
|
||||
cancel_session_runs(&state, &id)
|
||||
.await
|
||||
.map_err(|error| internal(error))?;
|
||||
cancel_session_runs(&state, &id).await.map_err(internal)?;
|
||||
Ok(Json(json!({"ok":true})))
|
||||
}
|
||||
|
||||
|
|
@ -387,10 +386,21 @@ async fn events(
|
|||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse::<i32>().ok())
|
||||
.unwrap_or(0);
|
||||
let stream_state = (state, id, actor, after, Vec::<(i32, String, Value)>::new());
|
||||
// Subscribe before the state moves into the stream: taking the subscription
|
||||
// afterwards would open a window in which a commit could knock on a channel
|
||||
// this reader is not listening to yet.
|
||||
let wakes = state.wakes.subscribe();
|
||||
let stream_state = (
|
||||
state,
|
||||
id,
|
||||
actor,
|
||||
after,
|
||||
Vec::<(i32, String, Value)>::new(),
|
||||
wakes,
|
||||
);
|
||||
let output = stream::unfold(
|
||||
stream_state,
|
||||
|(state, id, actor, mut after, mut pending)| async move {
|
||||
|(state, id, actor, mut after, mut pending, mut wakes)| async move {
|
||||
loop {
|
||||
if let Some((seq, kind, payload)) = pending.pop() {
|
||||
after = seq;
|
||||
|
|
@ -399,7 +409,7 @@ async fn events(
|
|||
.event(kind)
|
||||
.json_data(payload)
|
||||
.unwrap_or_else(|_| Event::default().event("error").data("{}"));
|
||||
return Some((Ok(event), (state, id, actor, after, pending)));
|
||||
return Some((Ok(event), (state, id, actor, after, pending, wakes)));
|
||||
}
|
||||
match sqlx::query_as::<_, (i32, String, Value)>(
|
||||
"SELECT e.seq,e.type,e.payload FROM events e
|
||||
|
|
@ -418,7 +428,15 @@ async fn events(
|
|||
rows.reverse();
|
||||
pending = rows;
|
||||
}
|
||||
Ok(_) | Err(_) => tokio::time::sleep(Duration::from_millis(750)).await,
|
||||
// Idle means "wait to be knocked", not "sleep then guess":
|
||||
// the reader blocks on the wake channel and re-reads the
|
||||
// cursor the moment a writer commits.
|
||||
Ok(_) | Err(_) => {
|
||||
tokio::select! {
|
||||
_ = wakes.wait(&id) => {}
|
||||
_ = tokio::time::sleep(EVENT_FALLBACK_POLL) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -447,12 +465,8 @@ pub async fn default_session_for_bot(
|
|||
.await
|
||||
}
|
||||
|
||||
pub async fn messages_for_session(
|
||||
state: &AppState,
|
||||
actor: &Actor,
|
||||
id: &str,
|
||||
) -> Result<Vec<SessionMessage>, ApiError> {
|
||||
let rows: Vec<(
|
||||
/// `messages_for_session` projection: message columns joined with the speaking bot.
|
||||
type MessageWithSpeakerRow = (
|
||||
String,
|
||||
String,
|
||||
i32,
|
||||
|
|
@ -466,7 +480,14 @@ pub async fn messages_for_session(
|
|||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
)> = sqlx::query_as(
|
||||
);
|
||||
|
||||
pub async fn messages_for_session(
|
||||
state: &AppState,
|
||||
actor: &Actor,
|
||||
id: &str,
|
||||
) -> Result<Vec<SessionMessage>, ApiError> {
|
||||
let rows: Vec<MessageWithSpeakerRow> = sqlx::query_as(
|
||||
"SELECT m.id, m.thread_id, m.seq, m.role, m.body, m.blocks, m.run_id,
|
||||
m.client_nonce, m.created_at, m.speaker_bot_id, b.name, b.avatar_color, b.avatar_shape
|
||||
FROM messages m
|
||||
|
|
@ -555,6 +576,10 @@ pub async fn append_event(
|
|||
.execute(&mut *tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
// Every event writer funnels through here, so one knock after the commit
|
||||
// covers messages, run state, and metrics without each call site having to
|
||||
// remember it.
|
||||
state.wakes.wake(thread_id);
|
||||
Ok(seq)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,10 @@
|
|||
//! Skills taught by demonstration.
|
||||
//!
|
||||
//! Flow: the human presses "teach it a task", takes control of the bot's
|
||||
//! desktop and does the task once. Meanwhile a CDP recorder inside the
|
||||
//! desktop logs *semantic* browser events (which control was clicked, what
|
||||
//! text went into which field, which URL loaded) and a poller keeps window
|
||||
//! titles plus a few keyframes. When the human stops, a model distils the
|
||||
//! trace into an intent-level playbook (goal, inputs, steps described by
|
||||
//! meaning, how to verify). Later runs get the playbook and execute it with the
|
||||
//! ordinary tools, locating controls on the live screen instead of replaying
|
||||
//! coordinates.
|
||||
//! The human demonstrates on the shared desktop. Cua observations supply
|
||||
//! window changes and keyframes; Cua trajectory recording adds actions invoked
|
||||
//! through the driver. Trajectories do not record raw human VNC input. A model
|
||||
//! distils the available visual evidence into an intent-level playbook, which
|
||||
//! later runs execute using controls located on the current screen.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
|
|
@ -19,8 +15,7 @@ use axum::{Json, Router};
|
|||
use base64::Engine;
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use lazyboy_control::{
|
||||
AdapterContext, CommandRequest, ComputerRef, cdp_record_command_on, cdp_record_stop_command,
|
||||
frame_signature, signatures_similar, teach_recorder_output,
|
||||
AdapterContext, ComputerRef, RecordingRequest, frame_signature, signatures_similar,
|
||||
};
|
||||
use rig_core::completion::message::{
|
||||
AssistantContent, ImageDetail, ImageMediaType, Message, UserContent,
|
||||
|
|
@ -282,17 +277,14 @@ async fn start_skill(
|
|||
})?;
|
||||
|
||||
let ctx = computer::adapter_context_for(&actor, &bot_id, "teach", screen.as_ref(), None);
|
||||
let display = ctx.display.clone().unwrap_or_else(|| ":1".into());
|
||||
let argv = cdp_record_command_on(&display, ctx.profile_path.as_deref(), &skill_id);
|
||||
if let Err(error) = state
|
||||
.sandbox
|
||||
.execute(
|
||||
.start_recording(
|
||||
&computer_ref,
|
||||
CommandRequest {
|
||||
argv,
|
||||
cwd: None,
|
||||
timeout_ms: Some(10_000),
|
||||
stdin: None,
|
||||
RecordingRequest {
|
||||
skill_id: skill_id.clone(),
|
||||
display: ctx.display.clone(),
|
||||
profile_path: ctx.profile_path.clone(),
|
||||
},
|
||||
&ctx,
|
||||
)
|
||||
|
|
@ -308,7 +300,7 @@ async fn start_skill(
|
|||
&skill_id,
|
||||
&bot_id,
|
||||
&format!(
|
||||
"開始學習：{goal}\n畫面交給你了，請直接在上面示範一次。我會記錄你點了哪些控制項、輸入了什麼、去了哪些頁面（密碼欄位不會記錄）。做完請按「完成示範」。"
|
||||
"開始學習：{goal}\n畫面交給你了，請直接在上面示範一次。我會擷取示範中的畫面與視窗變化，整理成可供你確認的流程草稿。每個步驟完成後請稍停一下，做完請按「完成示範」。"
|
||||
),
|
||||
)
|
||||
.await;
|
||||
|
|
@ -889,6 +881,14 @@ async fn record_loop(
|
|||
}
|
||||
}
|
||||
|
||||
fn recording_request(skill_id: &str, ctx: &AdapterContext) -> RecordingRequest {
|
||||
RecordingRequest {
|
||||
skill_id: skill_id.to_string(),
|
||||
display: ctx.display.clone(),
|
||||
profile_path: ctx.profile_path.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn stop_recorder(
|
||||
state: &AppState,
|
||||
computer_ref: &ComputerRef,
|
||||
|
|
@ -897,16 +897,7 @@ async fn stop_recorder(
|
|||
) {
|
||||
let _ = state
|
||||
.sandbox
|
||||
.execute(
|
||||
computer_ref,
|
||||
CommandRequest {
|
||||
argv: cdp_record_stop_command(skill_id),
|
||||
cwd: None,
|
||||
timeout_ms: Some(5_000),
|
||||
stdin: None,
|
||||
},
|
||||
ctx,
|
||||
)
|
||||
.stop_recording(computer_ref, recording_request(skill_id, ctx), ctx)
|
||||
.await;
|
||||
}
|
||||
|
||||
|
|
@ -916,34 +907,17 @@ async fn collect_browser_events(
|
|||
ctx: &AdapterContext,
|
||||
skill_id: &str,
|
||||
) -> Vec<Value> {
|
||||
let out = teach_recorder_output(skill_id);
|
||||
let result = state
|
||||
match state
|
||||
.sandbox
|
||||
.execute(
|
||||
computer_ref,
|
||||
CommandRequest {
|
||||
argv: vec![
|
||||
"sh".into(),
|
||||
"-c".into(),
|
||||
"cat \"$0\" 2>/dev/null; rm -f \"$0\"".into(),
|
||||
out,
|
||||
],
|
||||
cwd: None,
|
||||
timeout_ms: Some(10_000),
|
||||
stdin: None,
|
||||
},
|
||||
ctx,
|
||||
)
|
||||
.await;
|
||||
let Ok(result) = result else {
|
||||
return Vec::new();
|
||||
};
|
||||
result
|
||||
.stdout
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
|
||||
.filter(|event| event.get("t").and_then(Value::as_str) != Some("recorder"))
|
||||
.collect()
|
||||
.collect_recording(computer_ref, recording_request(skill_id, ctx), ctx)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result.events,
|
||||
Err(error) => {
|
||||
tracing::warn!("teach {skill_id}: collect recording failed: {error}");
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop recording, gather the trace, hand the desktop back and distil the
|
||||
|
|
@ -1073,10 +1047,17 @@ async fn finalize(
|
|||
.await;
|
||||
|
||||
if let Some(thread_id) = &row.thread_id {
|
||||
let mut body = format!("我學會了「{name}」。\n{}", summarize_playbook(&playbook));
|
||||
if error.is_some() {
|
||||
body.push_str("\n\n（模型整理失敗，這是依事件直接列出的版本，建議先修改再儲存。）");
|
||||
}
|
||||
let mut body = if error.is_some() {
|
||||
format!(
|
||||
"已保留「{name}」的示範，但模型整理失敗，流程草稿尚未完成。請補齊並確認步驟後再儲存。\n{}",
|
||||
summarize_playbook(&playbook)
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"已整理「{name}」的流程草稿，請先確認內容。\n{}",
|
||||
summarize_playbook(&playbook)
|
||||
)
|
||||
};
|
||||
body.push_str("\n\n確認名稱後按「儲存」，之後跟我說「執行");
|
||||
body.push_str(&name);
|
||||
body.push_str("」就會照這個流程做；或先「試跑」看看。");
|
||||
|
|
@ -1259,9 +1240,9 @@ fn pick_frames(frames: &[Value]) -> Vec<Value> {
|
|||
picked
|
||||
}
|
||||
|
||||
const DISTILL_SYSTEM: &str = "You turn a human's one-time screen demonstration into a reusable skill for a computer-use agent that controls the same Linux desktop (Chromium via a DOM snapshot/click/type tool, plus screenshots and xdotool for native windows).
|
||||
const DISTILL_SYSTEM: &str = "You turn a human's one-time screen demonstration into a reusable skill for a computer-use agent that controls the same Linux desktop (Cua browser and native element tools, plus shared-desktop screenshots).
|
||||
|
||||
You receive: the human's stated goal, a timeline of what they did (semantic browser events: which control was clicked by its label/text, what text was typed into which field, which URLs loaded, active window titles) and a few screenshots taken along the way. The trace is noisy: ignore mis-clicks, corrections, tab switches and anything unrelated to the goal.
|
||||
You receive the human's stated goal, window changes and screenshots captured during the demonstration. Driver-invoked actions may also appear, but raw human clicks and keystrokes are not recorded as semantic events. Infer steps only when the visual evidence supports them; mark missing or ambiguous steps for user review instead of inventing an action. Ignore corrections, tab switches and anything unrelated to the goal.
|
||||
|
||||
Produce a playbook that captures INTENT and PROCESS, never pixel positions:
|
||||
- Describe each step by what it achieves and which control to use, named by its visible label/role/page (e.g. \"在 Wikipedia 首頁的搜尋框輸入 <主題> 並按 Enter\"), so the agent can find it on a slightly different layout.
|
||||
|
|
@ -1394,7 +1375,11 @@ pub fn fallback_playbook(goal: &str, events: &[Value]) -> Value {
|
|||
})
|
||||
.filter_map(|event| describe_event(event, t0))
|
||||
.map(|line| {
|
||||
let text = line.splitn(2, ' ').nth(1).unwrap_or(&line).to_string();
|
||||
let text = line
|
||||
.split_once(' ')
|
||||
.map(|x| x.1)
|
||||
.unwrap_or(&line)
|
||||
.to_string();
|
||||
json!({ "do": text, "expect": "", "note": "" })
|
||||
})
|
||||
.take(40)
|
||||
|
|
|
|||
|
|
@ -47,12 +47,96 @@ pub struct CallLease {
|
|||
impl Drop for CallLease {
|
||||
fn drop(&mut self) {
|
||||
let mut map = self.registry.lock();
|
||||
if map.get(&self.bot_id).is_some_and(|held| *held == self.call_id) {
|
||||
if map
|
||||
.get(&self.bot_id)
|
||||
.is_some_and(|held| *held == self.call_id)
|
||||
{
|
||||
map.remove(&self.bot_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Capacity of the wake channel. A wake carries only a thread id, so a full
|
||||
/// channel means a reader stopped draining: it degrades to that reader's
|
||||
/// fallback poll rather than dropping a message, because the `events` table is
|
||||
/// still the source of truth for order and replay.
|
||||
const WAKE_CAPACITY: usize = 512;
|
||||
|
||||
/// Signals the session event stream that a thread's cursor moved. A wake only
|
||||
/// says "read it now"; it is what turns the browser's event feed from a poll
|
||||
/// into a push, which is what makes a chat reply feel instant.
|
||||
#[derive(Clone)]
|
||||
pub struct WakeBus {
|
||||
sender: tokio::sync::broadcast::Sender<String>,
|
||||
}
|
||||
|
||||
impl WakeBus {
|
||||
fn with_capacity(capacity: usize) -> Self {
|
||||
let (sender, _) = tokio::sync::broadcast::channel(capacity);
|
||||
Self { sender }
|
||||
}
|
||||
|
||||
/// Never blocks and never fails a request: nobody listening, or a reader too
|
||||
/// slow to keep up, is a latency concern only.
|
||||
pub fn wake(&self, thread_id: &str) {
|
||||
let _ = self.sender.send(thread_id.to_string());
|
||||
}
|
||||
|
||||
pub fn subscribe(&self) -> WakeSubscription {
|
||||
WakeSubscription {
|
||||
receiver: self.sender.subscribe(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for WakeBus {
|
||||
fn default() -> Self {
|
||||
Self::with_capacity(WAKE_CAPACITY)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct WakeSubscription {
|
||||
receiver: tokio::sync::broadcast::Receiver<String>,
|
||||
}
|
||||
|
||||
impl WakeSubscription {
|
||||
/// Resolves when `thread_id` moves. Every branch either returns or waits,
|
||||
/// and `recv` is cancel safe, so a `select!` that drops this future cannot
|
||||
/// swallow a wake: the message stays queued for the next call.
|
||||
pub async fn wait(&mut self, thread_id: &str) {
|
||||
loop {
|
||||
match self.receiver.recv().await {
|
||||
Ok(received) if received == thread_id => return,
|
||||
Ok(_) => continue,
|
||||
// A lagged reader already missed wakes, so let the caller re-read
|
||||
// the database rather than wait for a signal it cannot see.
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => return,
|
||||
// A closed channel can never signal again: parking here keeps the
|
||||
// caller on its fallback poll instead of spinning on a future
|
||||
// that resolves immediately.
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||
std::future::pending::<()>().await
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves on a wake for any thread, for a caller that only cares that
|
||||
/// something happened. Each subscriber owns its own receiver, so this never
|
||||
/// steals a wake from a thread-scoped one.
|
||||
pub async fn wait_any(&mut self) {
|
||||
loop {
|
||||
match self.receiver.recv().await {
|
||||
Ok(_) => return,
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => return,
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||
std::future::pending::<()>().await
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
pub db: Db,
|
||||
|
|
@ -62,6 +146,7 @@ pub struct AppState {
|
|||
pub memory: MemoryService,
|
||||
pub mcp: McpHub,
|
||||
pub calls: CallRegistry,
|
||||
pub wakes: WakeBus,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
|
|
@ -84,6 +169,7 @@ impl AppState {
|
|||
memory: MemoryService::from_env(),
|
||||
mcp: McpHub::new(),
|
||||
calls: CallRegistry::default(),
|
||||
wakes: WakeBus::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -115,3 +201,65 @@ fn sandbox_from_env() -> Arc<dyn SandboxProvider> {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod wake_bus_tests {
|
||||
use super::WakeBus;
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn every_subscription_for_a_thread_observes_the_wake() {
|
||||
let bus = WakeBus::with_capacity(4);
|
||||
let mut first = bus.subscribe();
|
||||
let mut second = bus.subscribe();
|
||||
bus.wake("thread-1");
|
||||
for subscription in [&mut first, &mut second] {
|
||||
tokio::time::timeout(Duration::from_secs(1), subscription.wait("thread-1"))
|
||||
.await
|
||||
.expect("every subscriber sees the wake");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn another_threads_wake_does_not_wake_me() {
|
||||
let bus = WakeBus::with_capacity(4);
|
||||
let mut mine = bus.subscribe();
|
||||
bus.wake("someone-else");
|
||||
tokio::time::timeout(Duration::from_millis(50), mine.wait("mine"))
|
||||
.await
|
||||
.expect_err("an unrelated thread stays silent");
|
||||
bus.wake("mine");
|
||||
tokio::time::timeout(Duration::from_secs(1), mine.wait("mine"))
|
||||
.await
|
||||
.expect("the matching thread resolves");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_lagged_reader_is_released_so_the_database_can_be_re_read() {
|
||||
let bus = WakeBus::with_capacity(4);
|
||||
let mut slow = bus.subscribe();
|
||||
for index in 0..32 {
|
||||
bus.wake(&format!("thread-{index}"));
|
||||
}
|
||||
// The backlog overflowed, so the wait resolves instead of hanging on a
|
||||
// signal this reader can no longer reach.
|
||||
tokio::time::timeout(Duration::from_secs(1), slow.wait("never-sent"))
|
||||
.await
|
||||
.expect("lag releases the reader");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wait_any_answers_for_a_thread_a_waiter_ignored() {
|
||||
let bus = WakeBus::with_capacity(4);
|
||||
let mut scoped = bus.subscribe();
|
||||
let mut any = bus.subscribe();
|
||||
bus.wake("somewhere-else");
|
||||
tokio::time::timeout(Duration::from_secs(1), any.wait_any())
|
||||
.await
|
||||
.expect("any-waker answers for the thread the scoped one skipped");
|
||||
// The scoped subscription has its own receiver and is still waiting.
|
||||
tokio::time::timeout(Duration::from_millis(50), scoped.wait("mine"))
|
||||
.await
|
||||
.expect_err("the scoped subscription is untouched");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -4,7 +4,7 @@ use aes_gcm::aead::{Aead, KeyInit};
|
|||
use aes_gcm::{Aes256Gcm, Nonce};
|
||||
use axum::extract::{Path, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::{delete, get, patch, post};
|
||||
use axum::routing::{get, patch};
|
||||
use axum::{Json, Router};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rand::RngCore;
|
||||
|
|
@ -161,14 +161,18 @@ pub async fn list_on(
|
|||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
pub async fn get_secret(
|
||||
state: &AppState,
|
||||
actor: &Actor,
|
||||
bot_id: &str,
|
||||
account_id: &str,
|
||||
) -> Result<Option<(VaultAccount, String, String)>, String> {
|
||||
get_secret_on(state.pool(), actor, bot_id, account_id).await
|
||||
}
|
||||
/// `get_secret_on` projection: vault columns plus the encrypted password.
|
||||
type SecretRow = (
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
DateTime<Utc>,
|
||||
DateTime<Utc>,
|
||||
String,
|
||||
);
|
||||
|
||||
pub async fn get_secret_on(
|
||||
pool: &sqlx::PgPool,
|
||||
|
|
@ -176,8 +180,7 @@ pub async fn get_secret_on(
|
|||
bot_id: &str,
|
||||
account_id: &str,
|
||||
) -> Result<Option<(VaultAccount, String, String)>, String> {
|
||||
let row: Option<(String, String, String, String, String, String, DateTime<Utc>, DateTime<Utc>, String)> =
|
||||
sqlx::query_as(
|
||||
let row: Option<SecretRow> = sqlx::query_as(
|
||||
"SELECT id, bot_id, site, host, username, notes, created_at, updated_at, password_ciphertext
|
||||
FROM vault_accounts
|
||||
WHERE id=$1 AND bot_id=$2 AND space_id=$3 AND user_id=$4",
|
||||
|
|
@ -328,8 +331,14 @@ fn vault_key() -> Result<[u8; 32], String> {
|
|||
let material = std::env::var("LAZYBOY_VAULT_KEY")
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty())
|
||||
.or_else(|| std::env::var("LAZYBOY_APP_TOKEN").ok().filter(|v| !v.is_empty()))
|
||||
.ok_or_else(|| "set LAZYBOY_VAULT_KEY or LAZYBOY_APP_TOKEN to encrypt saved passwords".to_string())?;
|
||||
.or_else(|| {
|
||||
std::env::var("LAZYBOY_APP_TOKEN")
|
||||
.ok()
|
||||
.filter(|v| !v.is_empty())
|
||||
})
|
||||
.ok_or_else(|| {
|
||||
"set LAZYBOY_VAULT_KEY or LAZYBOY_APP_TOKEN to encrypt saved passwords".to_string()
|
||||
})?;
|
||||
let digest = Sha256::digest(material.as_bytes());
|
||||
let mut key = [0u8; 32];
|
||||
key.copy_from_slice(&digest);
|
||||
|
|
@ -368,6 +377,9 @@ mod tests {
|
|||
use super::{decrypt, encrypt, normalize_host};
|
||||
|
||||
#[test]
|
||||
// Rust 2024 marks `set_var` unsafe; this only seeds a test-only key and no
|
||||
// other test in this binary reads it.
|
||||
#[allow(unsafe_code)]
|
||||
fn round_trips_a_password() {
|
||||
unsafe { std::env::set_var("LAZYBOY_VAULT_KEY", "test-vault-key-for-unit-tests") };
|
||||
let packed = encrypt("s3cret!").unwrap();
|
||||
|
|
@ -377,7 +389,10 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn host_strips_urls() {
|
||||
assert_eq!(normalize_host("https://mail.google.com/inbox", "Gmail"), "mail.google.com");
|
||||
assert_eq!(
|
||||
normalize_host("https://mail.google.com/inbox", "Gmail"),
|
||||
"mail.google.com"
|
||||
);
|
||||
assert_eq!(normalize_host("", "Gmail"), "gmail");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,10 @@ use crate::voice_call;
|
|||
|
||||
pub fn router() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/api/voice/settings", get(get_settings).patch(update_settings))
|
||||
.route(
|
||||
"/api/voice/settings",
|
||||
get(get_settings).patch(update_settings),
|
||||
)
|
||||
.route("/api/sessions/{id}/call", get(voice_call::call_ws))
|
||||
}
|
||||
|
||||
|
|
@ -193,7 +196,9 @@ async fn update_settings(
|
|||
if model_id.is_empty() || voice_id.is_empty() {
|
||||
return Err(StatusCode::BAD_REQUEST);
|
||||
}
|
||||
if catalog_voices(provider).iter().all(|(id, _)| *id != voice_id)
|
||||
if catalog_voices(provider)
|
||||
.iter()
|
||||
.all(|(id, _)| *id != voice_id)
|
||||
&& provider != VoiceProvider::Scripted
|
||||
{
|
||||
return Err(StatusCode::BAD_REQUEST);
|
||||
|
|
@ -210,7 +215,14 @@ async fn update_settings(
|
|||
};
|
||||
state
|
||||
.db
|
||||
.update_voice_settings(&actor, input.enabled, provider.as_str(), model_id, voice_id, api_key)
|
||||
.update_voice_settings(
|
||||
&actor,
|
||||
input.enabled,
|
||||
provider.as_str(),
|
||||
model_id,
|
||||
voice_id,
|
||||
api_key,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
get_settings(State(state)).await
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::ws::{Message as AxumMessage, WebSocket, WebSocketUpgrade};
|
||||
use axum::extract::{Path, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::IntoResponse;
|
||||
use axum::Json;
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use lazyboy_contracts::SessionAttachment;
|
||||
use lazyboy_harness::{VoiceConnectRequest, VoiceEvent, VoiceSocket, create_voice};
|
||||
|
|
@ -115,7 +115,9 @@ async fn prepare_call(
|
|||
.ok_or((StatusCode::NOT_FOUND, "workspace not found".into()))?;
|
||||
let (provider, resolved) =
|
||||
resolve_space_voice(&space).map_err(|message| (StatusCode::CONFLICT, message))?;
|
||||
let history = recent_text_history(state, session_id).await.unwrap_or_default();
|
||||
let history = recent_text_history(state, session_id)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let mut connect = VoiceConnectRequest::from_resolved(
|
||||
&resolved,
|
||||
voice_instructions(&bot_name, &bot_instructions),
|
||||
|
|
@ -291,7 +293,8 @@ async fn handle_provider_event(
|
|||
};
|
||||
user_partial.clear();
|
||||
if !body.is_empty() {
|
||||
let _ = persist_transcript(state, session_id, "user", &body, &prep.call_id).await;
|
||||
let _ =
|
||||
persist_transcript(state, session_id, "user", &body, &prep.call_id).await;
|
||||
send_json(
|
||||
client_write,
|
||||
json!({"type":"transcript","role":"user","text":body,"final":true}),
|
||||
|
|
@ -340,7 +343,15 @@ async fn handle_provider_event(
|
|||
name,
|
||||
arguments,
|
||||
} => {
|
||||
let output = dispatch_voice_tool(state, actor, &prep.bot_id, session_id, &prep.call_id, &name, &arguments)
|
||||
let output = dispatch_voice_tool(
|
||||
state,
|
||||
actor,
|
||||
&prep.bot_id,
|
||||
session_id,
|
||||
&prep.call_id,
|
||||
&name,
|
||||
&arguments,
|
||||
)
|
||||
.await;
|
||||
let _ = provider
|
||||
.send(VoiceEvent::FunctionCallOutput {
|
||||
|
|
@ -543,8 +554,7 @@ async fn watch_computer(
|
|||
)
|
||||
.await?;
|
||||
if let Some(line) = speakable_progress(&previous, &snapshot) {
|
||||
let urgent = snapshot.get("status").and_then(Value::as_str)
|
||||
== Some("takeover")
|
||||
let urgent = snapshot.get("status").and_then(Value::as_str) == Some("takeover")
|
||||
|| snapshot.get("status").and_then(Value::as_str) == Some("failed");
|
||||
if urgent || last_spoken_at.elapsed() > Duration::from_secs(6) {
|
||||
let _ = provider.send(VoiceEvent::SpeakNow { text: line }).await;
|
||||
|
|
@ -652,6 +662,10 @@ mod tests {
|
|||
);
|
||||
assert!(speakable_progress(&takeover.to_string(), &takeover).is_none());
|
||||
let idle = json!({"status":"idle"});
|
||||
assert!(speakable_progress("{\"status\":\"running\"}", &idle).unwrap().contains("做完"));
|
||||
assert!(
|
||||
speakable_progress("{\"status\":\"running\"}", &idle)
|
||||
.unwrap()
|
||||
.contains("做完")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,10 +62,8 @@ mod tests {
|
|||
use std::fs;
|
||||
|
||||
fn scratch(name: &str) -> PathBuf {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"lazyboy-web-static-{}-{name}",
|
||||
std::process::id()
|
||||
));
|
||||
let path =
|
||||
std::env::temp_dir().join(format!("lazyboy-web-static-{}-{name}", std::process::id()));
|
||||
let _ = fs::remove_dir_all(&path);
|
||||
fs::create_dir_all(&path).unwrap();
|
||||
path
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
name = "lazyboy-contracts"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
publish.workspace = true
|
||||
|
||||
|
|
@ -11,3 +12,6 @@ serde_json.workspace = true
|
|||
thiserror.workspace = true
|
||||
chrono.workspace = true
|
||||
base64.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ pub enum ComputerAction {
|
|||
#[serde(default)]
|
||||
button: Option<PointerButton>,
|
||||
},
|
||||
/// Semantic target (DOM selector or AT-SPI path). Execute via CDP/a11y, not xdotool.
|
||||
/// Snapshot-scoped semantic target resolved by Cua.
|
||||
Ref {
|
||||
verb: RefVerb,
|
||||
target: String,
|
||||
|
|
@ -53,6 +53,7 @@ pub enum ComputerAction {
|
|||
#[serde(default)]
|
||||
text: Option<String>,
|
||||
},
|
||||
CopySelection,
|
||||
Clipboard {
|
||||
text: String,
|
||||
},
|
||||
|
|
@ -103,7 +104,7 @@ pub struct UiElement {
|
|||
pub y: u32,
|
||||
pub w: u32,
|
||||
pub h: u32,
|
||||
/// CSS selector (DOM) or AT-SPI path (a11y). Native windows leave this empty.
|
||||
/// Opaque Cua browser or native element reference. Windows leave this empty.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selector: Option<String>,
|
||||
/// "dom" for page controls, "a11y" for AT-SPI widgets, "window" for native windows.
|
||||
|
|
@ -137,6 +138,9 @@ impl UiElement {
|
|||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ComputerObservation {
|
||||
/// The controller already populated native semantics; skip duplicate enrichment.
|
||||
#[serde(default)]
|
||||
pub native_observation_complete: bool,
|
||||
pub frame_id: String,
|
||||
pub captured_at: String,
|
||||
pub mime_type: String,
|
||||
|
|
|
|||
|
|
@ -113,8 +113,10 @@ pub fn computer_home_key(
|
|||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
#[derive(Default)]
|
||||
pub enum BrowserProfileMode {
|
||||
Shared,
|
||||
#[default]
|
||||
PerBot,
|
||||
PerTask,
|
||||
}
|
||||
|
|
@ -129,12 +131,6 @@ impl BrowserProfileMode {
|
|||
}
|
||||
}
|
||||
|
||||
impl Default for BrowserProfileMode {
|
||||
fn default() -> Self {
|
||||
Self::PerBot
|
||||
}
|
||||
}
|
||||
|
||||
impl std::str::FromStr for BrowserProfileMode {
|
||||
type Err = UnknownProfileMode;
|
||||
|
||||
|
|
@ -176,6 +172,9 @@ pub struct ComputerStatus {
|
|||
pub mode: ComputerMode,
|
||||
pub kind: SandboxKind,
|
||||
pub state: ComputerState,
|
||||
/// Human input is independent of the agent execution/pause lease.
|
||||
#[serde(default)]
|
||||
pub shared_input: bool,
|
||||
pub control_holder: ControlHolder,
|
||||
pub control_bot_id: Option<String>,
|
||||
pub takeover_requested: bool,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
name = "lazyboy-control"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
publish.workspace = true
|
||||
|
||||
|
|
@ -15,3 +16,9 @@ hex.workspace = true
|
|||
chrono.workspace = true
|
||||
async-trait = "0.1"
|
||||
image.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
base64.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
|
|
|||
|
|
@ -1,353 +0,0 @@
|
|||
import json, os, sys, time
|
||||
|
||||
def fail(msg):
|
||||
print(json.dumps({"ok": False, "error": msg, "elements": []}))
|
||||
sys.exit(0)
|
||||
|
||||
def load_session(display):
|
||||
display = display or os.environ.get("DISPLAY") or ":1"
|
||||
if not display.startswith(":"):
|
||||
display = ":" + display
|
||||
os.environ["DISPLAY"] = display
|
||||
number = display.lstrip(":")
|
||||
dbus_file = "/tmp/lazyboy/screen-%s.dbus" % number
|
||||
runtime_file = "/tmp/lazyboy/screen-%s.runtime" % number
|
||||
try:
|
||||
addr = open(dbus_file).read().strip()
|
||||
if addr:
|
||||
os.environ["DBUS_SESSION_BUS_ADDRESS"] = addr
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
runtime = open(runtime_file).read().strip()
|
||||
if runtime:
|
||||
os.environ["XDG_RUNTIME_DIR"] = runtime
|
||||
except Exception:
|
||||
if number == "1":
|
||||
candidate = "/tmp/xfce-home/runtime"
|
||||
else:
|
||||
candidate = "/tmp/xfce-home-%s/runtime" % number
|
||||
if os.path.isdir(candidate):
|
||||
os.environ["XDG_RUNTIME_DIR"] = candidate
|
||||
|
||||
def atspi():
|
||||
import gi
|
||||
gi.require_version("Atspi", "2.0")
|
||||
from gi.repository import Atspi
|
||||
try:
|
||||
Atspi.init()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
Atspi.set_timeout(200, 200)
|
||||
except Exception:
|
||||
pass
|
||||
return Atspi
|
||||
|
||||
INTERACTIVE = {
|
||||
"push button", "toggle button", "check box", "radio button",
|
||||
"combo box", "text", "password text", "menu item", "check menu item",
|
||||
"radio menu item", "tab", "page tab", "slider", "spin button",
|
||||
"link", "tree item", "entry", "password", "button", "menu",
|
||||
"list item", "column header", "toggle",
|
||||
}
|
||||
|
||||
BROWSER_APPS = ("chromium", "chrome", "google-chrome", "chromium-browser")
|
||||
|
||||
def is_browser_name(name):
|
||||
n = (name or "").lower()
|
||||
return any(token in n for token in BROWSER_APPS)
|
||||
|
||||
def child_count(acc):
|
||||
try:
|
||||
return int(acc.get_child_count())
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def child_at(acc, i):
|
||||
try:
|
||||
return acc.get_child_at_index(i)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def role_name(acc):
|
||||
try:
|
||||
return (acc.get_role_name() or "").lower()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
def acc_name(acc):
|
||||
try:
|
||||
text = (acc.get_name() or "").strip()
|
||||
if text:
|
||||
return text
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
return (acc.get_description() or "").strip()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
def state_names(Atspi, acc):
|
||||
out = []
|
||||
try:
|
||||
ss = acc.get_state_set()
|
||||
except Exception:
|
||||
return out
|
||||
for name in ("showing", "visible", "enabled", "sensitive", "checked",
|
||||
"selected", "focused", "editable", "defunct", "expandable",
|
||||
"expanded"):
|
||||
try:
|
||||
st = getattr(Atspi.StateType, name.upper())
|
||||
if ss.contains(st):
|
||||
out.append(name)
|
||||
except Exception:
|
||||
pass
|
||||
return out
|
||||
|
||||
def extents(Atspi, acc):
|
||||
try:
|
||||
ext = acc.get_extents(Atspi.CoordType.SCREEN)
|
||||
return int(ext.x), int(ext.y), int(ext.width), int(ext.height)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
comp = acc.get_component_iface()
|
||||
if comp is None:
|
||||
return None
|
||||
ext = comp.get_extents(Atspi.CoordType.SCREEN)
|
||||
return int(ext.x), int(ext.y), int(ext.width), int(ext.height)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def action_iface(acc):
|
||||
for getter in ("get_action_iface", "queryAction", "get_action"):
|
||||
fn = getattr(acc, getter, None)
|
||||
if not fn:
|
||||
continue
|
||||
try:
|
||||
iface = fn()
|
||||
if iface is not None:
|
||||
return iface
|
||||
except Exception:
|
||||
pass
|
||||
if hasattr(acc, "get_n_actions") and hasattr(acc, "do_action"):
|
||||
return acc
|
||||
return None
|
||||
|
||||
def text_ifaces(acc):
|
||||
edit = None
|
||||
text = None
|
||||
for getter in ("get_editable_text_iface", "queryEditableText"):
|
||||
fn = getattr(acc, getter, None)
|
||||
if not fn:
|
||||
continue
|
||||
try:
|
||||
edit = fn()
|
||||
if edit is not None:
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
for getter in ("get_text_iface", "queryText"):
|
||||
fn = getattr(acc, getter, None)
|
||||
if not fn:
|
||||
continue
|
||||
try:
|
||||
text = fn()
|
||||
if text is not None:
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
if edit is None and hasattr(acc, "insert_text"):
|
||||
edit = acc
|
||||
if text is None and hasattr(acc, "get_character_count"):
|
||||
text = acc
|
||||
return edit, text
|
||||
|
||||
def resolve(Atspi, path):
|
||||
desktop = Atspi.get_desktop(0)
|
||||
node = desktop
|
||||
for part in str(path).split("/"):
|
||||
if part == "":
|
||||
continue
|
||||
node = child_at(node, int(part))
|
||||
if node is None:
|
||||
return None
|
||||
return node
|
||||
|
||||
def grab_focus(acc):
|
||||
for getter in ("grab_focus",):
|
||||
fn = getattr(acc, getter, None)
|
||||
if fn:
|
||||
try:
|
||||
fn()
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
comp = acc.get_component_iface()
|
||||
if comp is not None:
|
||||
comp.grab_focus()
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
def do_click(acc):
|
||||
action = action_iface(acc)
|
||||
if action is None:
|
||||
return grab_focus(acc) and False
|
||||
try:
|
||||
n = int(action.get_n_actions())
|
||||
except Exception:
|
||||
n = 0
|
||||
idx = 0
|
||||
prefer = ("click", "press", "activate", "jump", "open", "toggle", "select")
|
||||
for i in range(n):
|
||||
try:
|
||||
name = (action.get_action_name(i) or "").lower()
|
||||
except Exception:
|
||||
name = ""
|
||||
if name in prefer:
|
||||
idx = i
|
||||
break
|
||||
if n <= 0:
|
||||
return False
|
||||
try:
|
||||
return bool(action.do_action(idx))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def set_text(acc, value):
|
||||
grab_focus(acc)
|
||||
edit, text = text_ifaces(acc)
|
||||
if edit is None:
|
||||
return False
|
||||
n = 0
|
||||
if text is not None:
|
||||
try:
|
||||
n = int(text.get_character_count())
|
||||
except Exception:
|
||||
n = 0
|
||||
try:
|
||||
edit.delete_text(0, n)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
edit.insert_text(0, value, len(value))
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def snapshot(Atspi, include_browser):
|
||||
deadline = time.time() + 2.8
|
||||
desktop = Atspi.get_desktop(0)
|
||||
found = []
|
||||
visited = 0
|
||||
apps = child_count(desktop)
|
||||
for app_i in range(apps):
|
||||
if time.time() > deadline or len(found) >= 50:
|
||||
break
|
||||
app = child_at(desktop, app_i)
|
||||
if app is None:
|
||||
continue
|
||||
app_label = acc_name(app) or role_name(app)
|
||||
if not include_browser and is_browser_name(app_label):
|
||||
continue
|
||||
stack = [(app, str(app_i), 0)]
|
||||
while stack:
|
||||
if time.time() > deadline or len(found) >= 50 or visited > 400:
|
||||
break
|
||||
acc, path, depth = stack.pop()
|
||||
visited += 1
|
||||
states = state_names(Atspi, acc)
|
||||
if "defunct" in states:
|
||||
continue
|
||||
role = role_name(acc)
|
||||
name = acc_name(acc)
|
||||
showing = ("showing" in states) or ("visible" in states) or not states
|
||||
if role in INTERACTIVE and showing and name:
|
||||
box = extents(Atspi, acc)
|
||||
if box and box[2] >= 2 and box[3] >= 2:
|
||||
x, y, w, h = box
|
||||
if x + w > 0 and y + h > 0:
|
||||
title = "%s %s" % (role, name.replace("\n", " ").strip())
|
||||
if "checked" in states:
|
||||
title += " (checked)"
|
||||
if "expanded" in states:
|
||||
title += " (expanded)"
|
||||
if "enabled" in states and "sensitive" in states:
|
||||
pass
|
||||
elif states and "enabled" not in states:
|
||||
title += " [disabled]"
|
||||
title = title[:80]
|
||||
found.append({
|
||||
"id": len(found) + 1,
|
||||
"title": title,
|
||||
"role": role,
|
||||
"kind": "a11y",
|
||||
"selector": path,
|
||||
"x": max(0, x),
|
||||
"y": max(0, y),
|
||||
"w": w,
|
||||
"h": h,
|
||||
})
|
||||
if depth >= 12:
|
||||
continue
|
||||
n = child_count(acc)
|
||||
# Walk children in reverse so index 0 is processed first with pop().
|
||||
for i in range(n - 1, -1, -1):
|
||||
child = child_at(acc, i)
|
||||
if child is None:
|
||||
continue
|
||||
stack.append((child, "%s/%d" % (path, i), depth + 1))
|
||||
return found
|
||||
|
||||
def main():
|
||||
req = json.loads(sys.argv[1]) if len(sys.argv) > 1 else {}
|
||||
action = req.get("action") or "snapshot"
|
||||
display = req.get("display") or ":1"
|
||||
load_session(display)
|
||||
try:
|
||||
Atspi = atspi()
|
||||
except Exception as error:
|
||||
fail("atspi unavailable: %s" % error)
|
||||
return
|
||||
if action == "snapshot":
|
||||
include_browser = bool(req.get("includeBrowser"))
|
||||
try:
|
||||
elements = snapshot(Atspi, include_browser)
|
||||
except Exception as error:
|
||||
fail("atspi snapshot failed: %s" % error)
|
||||
return
|
||||
print(json.dumps({"ok": True, "elements": elements}))
|
||||
return
|
||||
selector = req.get("selector") or ""
|
||||
if not selector:
|
||||
fail("a11y action needs selector")
|
||||
return
|
||||
try:
|
||||
acc = resolve(Atspi, selector)
|
||||
except Exception as error:
|
||||
fail("a11y resolve failed: %s" % error)
|
||||
return
|
||||
if acc is None:
|
||||
fail("a11y element gone")
|
||||
return
|
||||
ok = False
|
||||
if action == "click":
|
||||
ok = do_click(acc)
|
||||
elif action == "type":
|
||||
ok = set_text(acc, req.get("text") or "")
|
||||
elif action == "focus":
|
||||
ok = grab_focus(acc)
|
||||
else:
|
||||
fail("unsupported a11y action")
|
||||
return
|
||||
print(json.dumps({"ok": bool(ok), "error": None if ok else "a11y %s failed" % action}))
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception as error:
|
||||
fail(str(error))
|
||||
|
|
@ -1,59 +1,5 @@
|
|||
use crate::is_browser_title;
|
||||
use lazyboy_contracts::UiElement;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::x11::{is_browser_title, parse_ui_elements};
|
||||
use crate::{PRIMARY_DISPLAY, normalize_display};
|
||||
|
||||
const A11Y_PY: &str = include_str!("a11y.py");
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Default)]
|
||||
pub struct A11yPage {
|
||||
pub ok: bool,
|
||||
pub error: Option<String>,
|
||||
pub elements: Vec<UiElement>,
|
||||
}
|
||||
|
||||
pub fn a11y_command_on(display: &str, request: &Value) -> Vec<String> {
|
||||
let mut body = request.clone();
|
||||
if let Some(object) = body.as_object_mut() {
|
||||
object
|
||||
.entry("display")
|
||||
.or_insert_with(|| json!(normalize_display(display)));
|
||||
}
|
||||
vec![
|
||||
"env".into(),
|
||||
format!("DISPLAY={}", normalize_display(display)),
|
||||
"python3".into(),
|
||||
"-c".into(),
|
||||
A11Y_PY.into(),
|
||||
body.to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn a11y_command(request: &Value) -> Vec<String> {
|
||||
a11y_command_on(PRIMARY_DISPLAY, request)
|
||||
}
|
||||
|
||||
pub fn parse_a11y_page(raw: &str) -> A11yPage {
|
||||
let value: Value = serde_json::from_str(raw.trim()).unwrap_or(Value::Null);
|
||||
let ok = value.get("ok").and_then(Value::as_bool) == Some(true);
|
||||
A11yPage {
|
||||
ok,
|
||||
error: if ok {
|
||||
None
|
||||
} else {
|
||||
value
|
||||
.get("error")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string)
|
||||
.or_else(|| Some("atspi unavailable".into()))
|
||||
},
|
||||
elements: value
|
||||
.get("elements")
|
||||
.map(|items| parse_ui_elements(&items.to_string()))
|
||||
.unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Merge DOM (keep ids), then a11y, then native windows that are not already
|
||||
/// covered by a control tree. Chromium's whole window is dropped when the
|
||||
|
|
@ -130,31 +76,9 @@ mod tests {
|
|||
kind: Some("a11y".into()),
|
||||
selector: Some(format!("0/{id}")),
|
||||
role: Some("push button".into()),
|
||||
..UiElement::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn command_injects_display_and_script() {
|
||||
let argv = a11y_command_on(":2", &json!({"action": "snapshot"}));
|
||||
assert!(argv.contains(&"DISPLAY=:2".into()));
|
||||
assert!(argv.iter().any(|item| item.contains("python3")));
|
||||
assert!(argv.last().unwrap().contains("\"display\":\":2\""));
|
||||
assert!(argv.iter().any(|item| item.contains("Atspi") || item.contains("atspi")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_snapshot_elements() {
|
||||
let page = parse_a11y_page(
|
||||
r#"{"ok":true,"elements":[{"id":1,"title":"push button Open","role":"push button","kind":"a11y","selector":"0/2/1","x":10,"y":20,"w":80,"h":24}]}"#,
|
||||
);
|
||||
assert!(page.ok);
|
||||
assert_eq!(page.elements.len(), 1);
|
||||
assert_eq!(page.elements[0].kind.as_deref(), Some("a11y"));
|
||||
assert_eq!(page.elements[0].selector.as_deref(), Some("0/2/1"));
|
||||
assert_eq!(page.elements[0].role.as_deref(), Some("push button"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_keeps_dom_ids_and_replaces_covered_windows() {
|
||||
let windows = vec![
|
||||
|
|
@ -191,11 +115,4 @@ mod tests {
|
|||
assert_eq!(merged.len(), 1);
|
||||
assert_eq!(merged[0].title, "Thunar");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_tree_is_not_ok() {
|
||||
let page = parse_a11y_page(r#"{"ok":false,"error":"atspi unavailable"}"#);
|
||||
assert!(!page.ok);
|
||||
assert_eq!(page.error.as_deref(), Some("atspi unavailable"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
use crate::is_browser_title;
|
||||
use lazyboy_contracts::{
|
||||
ComputerAction, PointerButton, PointerType, RefVerb, ScrollDirection, UiElement,
|
||||
};
|
||||
|
|
@ -33,9 +32,12 @@ pub enum ActionError {
|
|||
/// instead of failing the click.
|
||||
pub fn element_id(value: Option<&Value>) -> Option<u64> {
|
||||
match value? {
|
||||
Value::Number(number) => number
|
||||
.as_u64()
|
||||
.or_else(|| number.as_f64().filter(|f| *f >= 0.0).map(|f| f.round() as u64)),
|
||||
Value::Number(number) => number.as_u64().or_else(|| {
|
||||
number
|
||||
.as_f64()
|
||||
.filter(|f| *f >= 0.0)
|
||||
.map(|f| f.round() as u64)
|
||||
}),
|
||||
Value::String(text) => text
|
||||
.trim()
|
||||
.trim_start_matches(['#', '['])
|
||||
|
|
@ -122,62 +124,6 @@ pub fn should_block_stale_click(miss_streak: u32, last: Option<&str>, next: Opti
|
|||
miss_streak >= 2 && next.is_some() && next == last
|
||||
}
|
||||
|
||||
/// When a CDP page snapshot is live, refuse pixel-clicking the Chromium window.
|
||||
pub fn browser_gui_block(actions: &Value, elements: &[UiElement]) -> Option<String> {
|
||||
if !elements
|
||||
.iter()
|
||||
.any(|element| element.kind.as_deref() == Some("dom"))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let items = actions.as_array()?;
|
||||
for raw in items {
|
||||
let Some(action) = raw.as_object() else {
|
||||
continue;
|
||||
};
|
||||
let kind = action
|
||||
.get("kind")
|
||||
.and_then(Value::as_str)
|
||||
.or_else(|| action.get("type").and_then(Value::as_str))
|
||||
.unwrap_or("");
|
||||
if !matches!(kind, "click" | "move" | "down" | "up" | "hover" | "drag") {
|
||||
continue;
|
||||
}
|
||||
if let Some(id) = element_id(action.get("element")) {
|
||||
match elements.iter().find(|element| u64::from(element.id) == id) {
|
||||
Some(element) if element.has_ref() => continue,
|
||||
Some(element)
|
||||
if element.kind.as_deref() == Some("window")
|
||||
&& is_browser_title(&element.title) =>
|
||||
{
|
||||
return Some(browser_block_message(elements));
|
||||
}
|
||||
_ => continue,
|
||||
}
|
||||
} else {
|
||||
return Some(browser_block_message(elements));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn browser_block_message(elements: &[UiElement]) -> String {
|
||||
let known: Vec<String> = elements
|
||||
.iter()
|
||||
.filter(|element| element.kind.as_deref() == Some("dom"))
|
||||
.take(12)
|
||||
.map(|element| format!("[{}] {}", element.id, element.title))
|
||||
.collect();
|
||||
format!(
|
||||
"Chromium is in front: use the browser tool (snapshot / click element N) instead of computer_act pixel clicks. Known page elements: {}",
|
||||
if known.is_empty() {
|
||||
"call browser snapshot first".to_string()
|
||||
} else {
|
||||
known.join(", ")
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
pub fn parse_computer_actions(value: &Value) -> Result<Vec<ComputerAction>, ActionError> {
|
||||
let Value::Array(items) = value else {
|
||||
return Err(ActionError::Empty);
|
||||
|
|
@ -200,8 +146,9 @@ pub fn parse_computer_actions(value: &Value) -> Result<Vec<ComputerAction>, Acti
|
|||
.unwrap_or_default();
|
||||
match kind {
|
||||
"click" | "move" | "down" | "up" => {
|
||||
if kind == "click" {
|
||||
if let Some(target) = ref_target(action) {
|
||||
if kind == "click"
|
||||
&& let Some(target) = ref_target(action)
|
||||
{
|
||||
let pointer = ComputerAction::Ref {
|
||||
verb: RefVerb::Click,
|
||||
target,
|
||||
|
|
@ -216,7 +163,6 @@ pub fn parse_computer_actions(value: &Value) -> Result<Vec<ComputerAction>, Acti
|
|||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let x = coordinate(action.get("x"), "x")?;
|
||||
let y = coordinate(action.get("y"), "y")?;
|
||||
let pointer_type = match kind {
|
||||
|
|
@ -403,7 +349,7 @@ fn ref_kind(action: &serde_json::Map<String, Value>) -> String {
|
|||
|
||||
fn coordinate(value: Option<&Value>, name: &'static str) -> Result<u32, ActionError> {
|
||||
let number = value.and_then(Value::as_f64).unwrap_or(f64::NAN).round();
|
||||
if !number.is_finite() || number < 0.0 || number > 100_000.0 {
|
||||
if !number.is_finite() || !(0.0..=100_000.0).contains(&number) {
|
||||
return Err(ActionError::BadCoordinate(name));
|
||||
}
|
||||
Ok(number as u32)
|
||||
|
|
@ -558,7 +504,6 @@ mod tests {
|
|||
y: 20,
|
||||
w: 80,
|
||||
h: 24,
|
||||
..UiElement::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -612,9 +557,7 @@ mod tests {
|
|||
assert!(actions[0].get("x").is_none());
|
||||
let parsed = parse_computer_actions(&actions).unwrap();
|
||||
match &parsed[0] {
|
||||
ComputerAction::Ref {
|
||||
ref_kind, verb, ..
|
||||
} => {
|
||||
ComputerAction::Ref { ref_kind, verb, .. } => {
|
||||
assert_eq!(ref_kind, "dom");
|
||||
assert_eq!(*verb, RefVerb::Click);
|
||||
}
|
||||
|
|
@ -622,51 +565,6 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pixel_clicks_are_blocked_when_dom_is_live() {
|
||||
let elements = vec![UiElement {
|
||||
id: 1,
|
||||
title: "Submit".into(),
|
||||
selector: Some("[data-lazyboy=\"1\"]".into()),
|
||||
kind: Some("dom".into()),
|
||||
x: 10,
|
||||
y: 20,
|
||||
w: 80,
|
||||
h: 24,
|
||||
..UiElement::default()
|
||||
}];
|
||||
let blocked = browser_gui_block(&json!([{"kind":"click","x":40,"y":80}]), &elements);
|
||||
assert!(blocked.unwrap().contains("browser"));
|
||||
assert!(
|
||||
browser_gui_block(&json!([{"kind":"click","element":1}]), &elements).is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chromium_window_clicks_are_blocked_when_dom_is_live() {
|
||||
let elements = vec![
|
||||
UiElement {
|
||||
id: 1,
|
||||
title: "Submit".into(),
|
||||
selector: Some("[data-lazyboy=\"1\"]".into()),
|
||||
kind: Some("dom".into()),
|
||||
..UiElement::default()
|
||||
},
|
||||
UiElement {
|
||||
id: 2,
|
||||
title: "Chromium".into(),
|
||||
kind: Some("window".into()),
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: 1280,
|
||||
h: 800,
|
||||
..UiElement::default()
|
||||
},
|
||||
];
|
||||
let blocked = browser_gui_block(&json!([{"kind":"click","element":2}]), &elements);
|
||||
assert!(blocked.unwrap().contains("browser"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_repeat_blocks_the_third_same_click() {
|
||||
assert!(!should_block_stale_click(1, Some("e3"), Some("e3")));
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
use lazyboy_contracts::UiElement;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub fn sanitize_skill_id(skill_id: &str) -> String {
|
||||
let cleaned: String = skill_id
|
||||
.chars()
|
||||
.filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_')
|
||||
.take(80)
|
||||
.collect();
|
||||
if cleaned.is_empty() {
|
||||
"unknown".into()
|
||||
} else {
|
||||
cleaned
|
||||
}
|
||||
}
|
||||
|
||||
pub fn teach_trajectory_dir(skill_id: &str) -> String {
|
||||
format!("/tmp/lazyboy/teach-{}", sanitize_skill_id(skill_id))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BrowserPage {
|
||||
pub ok: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
#[serde(default)]
|
||||
pub url: String,
|
||||
#[serde(default)]
|
||||
pub title: String,
|
||||
#[serde(default)]
|
||||
pub text: String,
|
||||
#[serde(default)]
|
||||
pub restarted: bool,
|
||||
/// Seconds the click waited for a disabled control to become enabled.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub waited_seconds: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub elements: Vec<UiElement>,
|
||||
}
|
||||
|
|
@ -1,665 +0,0 @@
|
|||
import json, os, sys, time, socket, base64, struct, subprocess, urllib.request
|
||||
|
||||
def fail(msg):
|
||||
print(json.dumps({"ok": False, "error": msg}))
|
||||
sys.exit(0)
|
||||
|
||||
def http_json(url, timeout=2):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=timeout) as r:
|
||||
return json.loads(r.read().decode())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
class Ws:
|
||||
"""Use a maintained RFC6455 transport (fragmentation, ping/pong, handshake)."""
|
||||
def __init__(self, url):
|
||||
import websocket
|
||||
self.sock = websocket.create_connection(url, timeout=5, suppress_origin=True,
|
||||
http_no_proxy=["127.0.0.1", "localhost"])
|
||||
self.n = 0
|
||||
|
||||
def recv_json(self):
|
||||
return json.loads(self.sock.recv())
|
||||
|
||||
def call(self, method, params=None):
|
||||
self.n += 1
|
||||
self.sock.send(json.dumps({"id": self.n, "method": method, "params": params or {}}))
|
||||
deadline = time.monotonic() + 10
|
||||
while time.monotonic() < deadline:
|
||||
self.sock.settimeout(max(.1, deadline-time.monotonic()))
|
||||
obj = self.recv_json()
|
||||
if obj.get("id") == self.n:
|
||||
if "error" in obj:
|
||||
raise RuntimeError(str(obj["error"]))
|
||||
return obj.get("result") or {}
|
||||
raise TimeoutError("CDP response deadline exceeded")
|
||||
|
||||
def close(self):
|
||||
self.sock.close()
|
||||
|
||||
def probe(port):
|
||||
return http_json("http://127.0.0.1:%s/json/version" % port) is not None
|
||||
|
||||
def profile_alive(profile):
|
||||
if not profile:
|
||||
return False
|
||||
try:
|
||||
out = subprocess.check_output(["pgrep", "-af", "chromium"], text=True, stderr=subprocess.DEVNULL)
|
||||
except Exception:
|
||||
return False
|
||||
for line in out.splitlines():
|
||||
if ("--user-data-dir=%s" % profile) in line and "--type=" not in line:
|
||||
return True
|
||||
return False
|
||||
|
||||
def active_port(profile):
|
||||
# Chromium writes the DevTools port it actually bound here. Trust it over
|
||||
# our expected port so we attach to the window the human already sees.
|
||||
if not profile:
|
||||
return None
|
||||
try:
|
||||
with open(os.path.join(profile, "DevToolsActivePort")) as f:
|
||||
return int(f.readline().strip())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def spawn_browser(display, profile, port):
|
||||
env = os.environ.copy()
|
||||
env["DISPLAY"] = display
|
||||
if profile:
|
||||
env["LAZYBOY_BROWSER_PROFILE"] = profile
|
||||
subprocess.Popen(
|
||||
["lazyboy-browser", "--remote-debugging-port=%s" % port],
|
||||
env=env,
|
||||
start_new_session=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
def connect(port):
|
||||
tabs = http_json("http://127.0.0.1:%s/json/list" % port) or []
|
||||
pages = [t for t in tabs if t.get("type") == "page" and t.get("webSocketDebuggerUrl")]
|
||||
if not pages:
|
||||
fail("no browser tab")
|
||||
# /json/list is ordered by last activity, so pages[0] is the tab the human
|
||||
# is looking at. Bring it to the front anyway so what the model reads and
|
||||
# clicks is always the tab shown on the live screen.
|
||||
pages.sort(key=lambda t: (t.get("url") or "").startswith("chrome://"), reverse=False)
|
||||
page = pages[0]
|
||||
if page.get("id"):
|
||||
http_json("http://127.0.0.1:%s/json/activate/%s" % (port, page["id"]))
|
||||
ws = Ws(page["webSocketDebuggerUrl"])
|
||||
ws.call("Runtime.enable")
|
||||
ws.call("Page.enable")
|
||||
return ws
|
||||
|
||||
SNAP_JS = r"""
|
||||
(() => {
|
||||
const sels = 'a, button, input, textarea, select, summary, label, [role="button"], [role="link"], [role="textbox"], [role="checkbox"], [role="menuitem"], [contenteditable="true"]';
|
||||
const chromeH = Math.max(0, (window.outerHeight || 0) - (window.innerHeight || 0));
|
||||
const chromeW = Math.max(0, (window.outerWidth || 0) - (window.innerWidth || 0));
|
||||
const sx0 = (window.screenX || 0) + Math.floor(chromeW / 2);
|
||||
const sy0 = (window.screenY || 0) + chromeH;
|
||||
document.querySelectorAll('[data-lazyboy]').forEach(el => el.removeAttribute('data-lazyboy'));
|
||||
const generation = crypto.randomUUID();
|
||||
const seen = new Set();
|
||||
const inView = [];
|
||||
const offView = [];
|
||||
for (const el of document.querySelectorAll(sels)) {
|
||||
const r = el.getBoundingClientRect();
|
||||
if (r.width < 4 || r.height < 4) continue;
|
||||
const st = getComputedStyle(el);
|
||||
if (st.visibility === "hidden" || st.display === "none" || Number(st.opacity) === 0) continue;
|
||||
const type = (el.getAttribute("type") || "").toLowerCase();
|
||||
let text;
|
||||
if (type === "password" || /password|secret|token|one-time-code/i.test([el.name, el.id, el.autocomplete].join(" "))) {
|
||||
text = "[protected input]";
|
||||
} else if (el.tagName === "INPUT" && (type === "radio" || type === "checkbox")) {
|
||||
// Quiz answers: the value is usually "on"; the label next to it is what
|
||||
// the model must read to pick the right option.
|
||||
const owner = (el.labels && el.labels[0]) || el.closest("label") || el.parentElement;
|
||||
const label = (owner && owner.innerText || el.getAttribute("aria-label") || el.value || "").replace(/\s+/g, " ").trim().slice(0, 70);
|
||||
text = type + " " + label + (el.checked ? " (checked)" : "");
|
||||
} else {
|
||||
text = (el.innerText || el.value || el.getAttribute("aria-label") || el.getAttribute("placeholder") || el.getAttribute("name") || el.tagName)
|
||||
.replace(/\s+/g, " ").trim().slice(0, 80);
|
||||
}
|
||||
if (!text.trim()) continue;
|
||||
const key = [el.tagName, text, Math.round(r.x), Math.round(r.y)].join("|");
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
if (el.disabled || el.getAttribute("aria-disabled") === "true") text += " [disabled]";
|
||||
const visible = !(r.bottom < 0 || r.right < 0 || r.top > innerHeight || r.left > innerWidth);
|
||||
if (visible) {
|
||||
inView.push({el, text, x: Math.max(0, Math.round(sx0 + r.x)), y: Math.max(0, Math.round(sy0 + r.y)), w: Math.round(r.width), h: Math.round(r.height)});
|
||||
} else {
|
||||
// Controls outside the viewport are still clickable by id: the click
|
||||
// handler scrolls them into view. Zero size tells the desktop side not
|
||||
// to paint or pixel-click them.
|
||||
const where = r.top > innerHeight ? "below" : r.bottom < 0 ? "above" : "beside";
|
||||
// Buttons and form controls (Next, Submit, radios) matter more than the
|
||||
// hundredth body link, so they win the limited off-screen slots.
|
||||
const link = el.tagName === "A" || el.getAttribute("role") === "link";
|
||||
offView.push({el, text: text + " [" + where + " viewport]", x: 0, y: 0, w: 0, h: 0, rank: (link ? 1 : 0), dist: Math.abs(r.top > innerHeight ? r.top - innerHeight : r.bottom)});
|
||||
}
|
||||
}
|
||||
offView.sort((a, b) => a.rank - b.rank || a.dist - b.dist);
|
||||
const out = [];
|
||||
let n = 1;
|
||||
for (const item of inView.slice(0, 50).concat(offView.slice(0, 20))) {
|
||||
item.el.setAttribute("data-lazyboy", generation + "-" + n);
|
||||
out.push({
|
||||
id: n,
|
||||
title: item.text,
|
||||
tag: item.el.tagName.toLowerCase(),
|
||||
selector: '[data-lazyboy="' + generation + "-" + n + '"]',
|
||||
kind: "dom",
|
||||
x: item.x,
|
||||
y: item.y,
|
||||
w: item.w,
|
||||
h: item.h
|
||||
});
|
||||
n += 1;
|
||||
}
|
||||
const body = (document.body && document.body.innerText || "").replace(/\s+/g, " ").trim().slice(0, 3000);
|
||||
return {url: location.href, title: document.title || "", text: body, elements: out};
|
||||
})()
|
||||
"""
|
||||
|
||||
CLICK_JS = r"""
|
||||
(sel) => {
|
||||
const el = document.querySelector(sel);
|
||||
if (!el) return {ok: false, error: "element gone"};
|
||||
el.scrollIntoView({block: "center", inline: "nearest"});
|
||||
const r = el.getBoundingClientRect();
|
||||
const style = getComputedStyle(el);
|
||||
const hit = document.elementFromPoint(r.x+r.width/2, r.y+r.height/2);
|
||||
if (el.disabled || el.getAttribute("aria-disabled") === "true" || r.width <= 0 || r.height <= 0 || style.visibility === "hidden" || style.display === "none" || !hit || !(hit === el || el.contains(hit))) {
|
||||
return {ok:false,error:"element is disabled, hidden, or covered; observe again"};
|
||||
}
|
||||
el.focus();
|
||||
el.click();
|
||||
const chromeH = Math.max(0, (window.outerHeight || 0) - (window.innerHeight || 0));
|
||||
const chromeW = Math.max(0, (window.outerWidth || 0) - (window.innerWidth || 0));
|
||||
const sx = (window.screenX || 0) + Math.floor(chromeW / 2) + r.x + r.width / 2;
|
||||
const sy = (window.screenY || 0) + chromeH + r.y + r.height / 2;
|
||||
return {ok: true, x: Math.round(sx), y: Math.round(sy)};
|
||||
}
|
||||
"""
|
||||
|
||||
STATE_JS = r"""
|
||||
(sel) => {
|
||||
const el = document.querySelector(sel);
|
||||
if (!el) return {found: false};
|
||||
const disabled = !!el.disabled || el.getAttribute("aria-disabled") === "true";
|
||||
// Training sites explain the lock next to the button ("Please watch the
|
||||
// video", a countdown); surface that text so the model can decide how
|
||||
// long to wait.
|
||||
let hint = "";
|
||||
const near = el.parentElement && el.parentElement.parentElement;
|
||||
if (disabled && near) hint = (near.innerText || "").replace(/\s+/g, " ").trim().slice(0, 120);
|
||||
return {found: true, disabled, hint};
|
||||
}
|
||||
"""
|
||||
|
||||
# Pages often lock Next for a few seconds (stay timers) or until a video ends.
|
||||
# A human just waits and clicks; do the same instead of making the model plan
|
||||
# a wait/observe/click loop it tends to abandon.
|
||||
CLICK_WAIT_MS = 45000
|
||||
|
||||
def wait_until_enabled(ws, sel, wait_ms=None):
|
||||
budget = CLICK_WAIT_MS if wait_ms is None else max(0, min(int(wait_ms), 120000))
|
||||
started = time.time()
|
||||
while True:
|
||||
state = evaluate(ws, STATE_JS, sel) or {}
|
||||
if not state.get("found"):
|
||||
return None
|
||||
if not state.get("disabled"):
|
||||
return time.time() - started
|
||||
if (time.time() - started) * 1000 >= budget:
|
||||
return None
|
||||
time.sleep(0.5)
|
||||
|
||||
def evaluate(ws, expression, args=None):
|
||||
params = {"expression": expression, "returnByValue": True, "awaitPromise": True}
|
||||
if args is not None:
|
||||
params = {
|
||||
"expression": "(%s)(%s)" % (expression, json.dumps(args)),
|
||||
"returnByValue": True,
|
||||
"awaitPromise": True,
|
||||
}
|
||||
result = ws.call("Runtime.evaluate", params)
|
||||
val = (result.get("result") or {}).get("value")
|
||||
if result.get("exceptionDetails"):
|
||||
raise RuntimeError(str(result["exceptionDetails"]))
|
||||
return val
|
||||
|
||||
def wait_for_visual_update(ws):
|
||||
# CDP input and DOM clicks can complete before Chromium commits the next
|
||||
# painted frame. The caller captures X11 immediately after this process
|
||||
# exits, so wait for two animation frames to keep that screenshot aligned
|
||||
# with the framebuffer streamed by VNC.
|
||||
try:
|
||||
evaluate(ws, "new Promise(resolve => {setTimeout(resolve, 250); requestAnimationFrame(() => requestAnimationFrame(resolve));})")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def snapshot(ws):
|
||||
val = evaluate(ws, SNAP_JS) or {}
|
||||
return {
|
||||
"ok": True,
|
||||
"action": "snapshot",
|
||||
"url": val.get("url") or "",
|
||||
"title": val.get("title") or "",
|
||||
"text": val.get("text") or "",
|
||||
"elements": val.get("elements") or [],
|
||||
}
|
||||
|
||||
def pointer(display, x, y):
|
||||
try:
|
||||
subprocess.check_call(
|
||||
["env", "DISPLAY=%s" % display, "xdotool", "mousemove", "--sync", "--", str(int(x)), str(int(y))],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
KEYS = {
|
||||
"Return": (13, "Enter", "Enter"),
|
||||
"Enter": (13, "Enter", "Enter"),
|
||||
"Tab": (9, "Tab", "Tab"),
|
||||
"BackSpace": (8, "Backspace", "Backspace"),
|
||||
"Backspace": (8, "Backspace", "Backspace"),
|
||||
"Escape": (27, "Escape", "Escape"),
|
||||
"Esc": (27, "Escape", "Escape"),
|
||||
"Space": (32, " ", "Space"),
|
||||
}
|
||||
|
||||
def press(ws, key):
|
||||
spec = KEYS.get(key) or KEYS.get(key.title())
|
||||
if spec:
|
||||
code, name, key_id = spec
|
||||
for typ in ("keyDown", "keyUp"):
|
||||
ws.call("Input.dispatchKeyEvent", {
|
||||
"type": typ,
|
||||
"windowsVirtualKeyCode": code,
|
||||
"key": name,
|
||||
"code": key_id,
|
||||
})
|
||||
return
|
||||
ws.call("Input.dispatchKeyEvent", {"type": "keyDown", "text": key[:1]})
|
||||
ws.call("Input.dispatchKeyEvent", {"type": "keyUp", "text": key[:1]})
|
||||
|
||||
# Injected into every page while a human demonstrates a task. It reports what
|
||||
# the person did in terms of page semantics (which control, what text, which
|
||||
# URL) rather than pixels, so the distilled skill can generalise. Secrets are
|
||||
# masked before they leave the page.
|
||||
RECORD_JS = r"""
|
||||
(() => {
|
||||
if (window.__lbTeachInstalled) return;
|
||||
window.__lbTeachInstalled = true;
|
||||
const send = (ev) => { try { ev.at = Date.now(); ev.url = location.href; window.__lbTeach(JSON.stringify(ev)); } catch (e) {} };
|
||||
const clean = (s) => (s || "").replace(/\s+/g, " ").trim().slice(0, 120);
|
||||
const secretRe = /pass|pwd|secret|token|otp|cvv|card|pin\b/i;
|
||||
const isSecret = (el) => !el ? false : (el.type === "password" || secretRe.test(el.name || "") || secretRe.test(el.id || "") || secretRe.test(el.autocomplete || "") || secretRe.test(el.getAttribute && el.getAttribute("aria-label") || ""));
|
||||
const labelFor = (el) => {
|
||||
if (!el) return "";
|
||||
if (el.labels && el.labels.length) return clean(el.labels[0].innerText);
|
||||
const id = el.id && document.querySelector('label[for="' + el.id + '"]');
|
||||
if (id) return clean(id.innerText);
|
||||
return clean(el.getAttribute("aria-label") || el.placeholder || el.title || el.name || "");
|
||||
};
|
||||
const describe = (el) => {
|
||||
if (!el || el.nodeType !== 1) return null;
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const d = { tag, role: el.getAttribute("role") || "", text: clean(el.innerText || el.value || el.alt || el.getAttribute("aria-label") || el.title || el.placeholder || ""), label: labelFor(el) };
|
||||
if (el.id) d.id = el.id;
|
||||
if (el.name) d.name = el.name;
|
||||
if (tag === "a" && el.href) d.href = el.href.slice(0, 200);
|
||||
if (tag === "input") d.type = el.type || "text";
|
||||
return d;
|
||||
};
|
||||
const actionable = (node) => {
|
||||
let el = node;
|
||||
for (let i = 0; el && i < 6; i++) {
|
||||
if (el.nodeType === 1) {
|
||||
const t = el.tagName.toLowerCase();
|
||||
if (["a","button","input","select","textarea","summary","label","option"].includes(t) || el.getAttribute("role") || el.onclick || el.getAttribute("tabindex") !== null || el.isContentEditable) return el;
|
||||
}
|
||||
el = el.parentNode;
|
||||
}
|
||||
return node && node.nodeType === 1 ? node : null;
|
||||
};
|
||||
send({ t: "page", title: document.title });
|
||||
document.addEventListener("click", (e) => {
|
||||
const el = actionable(e.target);
|
||||
const d = describe(el);
|
||||
if (d) send({ t: "click", el: d, x: Math.round(e.clientX), y: Math.round(e.clientY) });
|
||||
}, true);
|
||||
const pending = new Map();
|
||||
const flush = (el) => {
|
||||
pending.delete(el);
|
||||
const d = describe(el);
|
||||
if (!d) return;
|
||||
let value = el.isContentEditable ? el.innerText : (el.value || "");
|
||||
if (el.tagName === "SELECT" && el.selectedOptions && el.selectedOptions[0]) value = el.selectedOptions[0].text;
|
||||
if (el.type === "checkbox" || el.type === "radio") value = el.checked ? "checked" : "unchecked";
|
||||
send({ t: "input", el: d, value: isSecret(el) ? "[redacted]" : clean(value) });
|
||||
};
|
||||
document.addEventListener("input", (e) => {
|
||||
const el = e.target;
|
||||
if (!el || !(el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable)) return;
|
||||
clearTimeout(pending.get(el));
|
||||
pending.set(el, setTimeout(() => flush(el), 900));
|
||||
}, true);
|
||||
document.addEventListener("change", (e) => { const el = e.target; if (el && el.nodeType === 1) { clearTimeout(pending.get(el)); flush(el); } }, true);
|
||||
document.addEventListener("keydown", (e) => {
|
||||
const special = ["Enter","Escape","Tab"].includes(e.key) || e.ctrlKey || e.metaKey || e.altKey;
|
||||
if (!special || e.key === "Control" || e.key === "Meta" || e.key === "Alt" || e.key === "Shift") return;
|
||||
const el = document.activeElement;
|
||||
if (el && pending.has(el)) { clearTimeout(pending.get(el)); flush(el); }
|
||||
const combo = [e.ctrlKey ? "Ctrl" : "", e.metaKey ? "Meta" : "", e.altKey ? "Alt" : "", e.shiftKey ? "Shift" : "", e.key].filter(Boolean).join("+");
|
||||
send({ t: "key", key: combo, el: describe(el) });
|
||||
}, true);
|
||||
document.addEventListener("submit", (e) => { const f = e.target; send({ t: "submit", form: { action: (f && f.action || "").slice(0, 200), name: f && (f.name || f.id) || "" } }); }, true);
|
||||
let lastScroll = 0;
|
||||
window.addEventListener("scroll", () => { const now = Date.now(); if (now - lastScroll > 2000) { lastScroll = now; send({ t: "scroll", y: Math.round(window.scrollY) }); } }, true);
|
||||
})()
|
||||
"""
|
||||
|
||||
class Recorder:
|
||||
"""Browser-level CDP session with flattened page sessions. Events from
|
||||
every tab are appended to a JSONL file until the process is killed."""
|
||||
|
||||
def __init__(self, port, out):
|
||||
info = http_json("http://127.0.0.1:%s/json/version" % port) or {}
|
||||
url = info.get("webSocketDebuggerUrl")
|
||||
if not url:
|
||||
raise RuntimeError("browser has no DevTools endpoint")
|
||||
self.ws = Ws(url)
|
||||
self.out = open(out, "a", buffering=1)
|
||||
self.sessions = {}
|
||||
self.pending = []
|
||||
self.last = (None, 0)
|
||||
|
||||
def emit(self, ev):
|
||||
ev.setdefault("at", int(time.time() * 1000))
|
||||
# Two sessions on one page (auto-attach + explicit) deliver the same
|
||||
# binding call twice; a key repeat is never that fast either.
|
||||
key = json.dumps({k: v for k, v in ev.items() if k != "at"}, sort_keys=True)
|
||||
if key == self.last[0] and ev["at"] - self.last[1] < 800:
|
||||
return
|
||||
self.last = (key, ev["at"])
|
||||
self.out.write(json.dumps(ev, ensure_ascii=False) + "\n")
|
||||
|
||||
def call(self, method, params=None, session=None):
|
||||
self.ws.n += 1
|
||||
msg = {"id": self.ws.n, "method": method}
|
||||
if params:
|
||||
msg["params"] = params
|
||||
if session:
|
||||
msg["sessionId"] = session
|
||||
self.ws.sock.send(json.dumps(msg))
|
||||
while True:
|
||||
obj = self.ws.recv_json()
|
||||
if obj.get("id") == self.ws.n:
|
||||
if "error" in obj:
|
||||
raise RuntimeError(str(obj["error"]))
|
||||
return obj.get("result") or {}
|
||||
self.pending.append(obj)
|
||||
|
||||
def attach(self, session, target):
|
||||
if target.get("type") != "page" or not session:
|
||||
return
|
||||
target_id = target.get("targetId")
|
||||
if session in self.sessions:
|
||||
return
|
||||
if target_id in self.sessions.values():
|
||||
try:
|
||||
self.call("Target.detachFromTarget", {"sessionId": session})
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
self.sessions[session] = target_id
|
||||
for method, params in (
|
||||
("Runtime.enable", None),
|
||||
("Page.enable", None),
|
||||
("Runtime.addBinding", {"name": "__lbTeach"}),
|
||||
("Page.addScriptToEvaluateOnNewDocument", {"source": RECORD_JS}),
|
||||
("Runtime.evaluate", {"expression": RECORD_JS}),
|
||||
):
|
||||
try:
|
||||
self.call(method, params, session)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def handle(self, obj):
|
||||
method = obj.get("method")
|
||||
params = obj.get("params") or {}
|
||||
if method == "Target.attachedToTarget":
|
||||
self.attach(params.get("sessionId"), params.get("targetInfo") or {})
|
||||
elif method == "Target.detachedFromTarget":
|
||||
self.sessions.pop(params.get("sessionId"), None)
|
||||
elif method == "Runtime.bindingCalled" and params.get("name") == "__lbTeach":
|
||||
try:
|
||||
self.emit(json.loads(params.get("payload") or "{}"))
|
||||
except Exception:
|
||||
pass
|
||||
elif method == "Page.frameNavigated":
|
||||
frame = params.get("frame") or {}
|
||||
if not frame.get("parentId"):
|
||||
self.emit({"t": "navigate", "url": frame.get("url") or ""})
|
||||
elif method == "Target.targetInfoChanged":
|
||||
info = params.get("targetInfo") or {}
|
||||
if info.get("type") == "page" and info.get("title"):
|
||||
self.emit({"t": "title", "url": info.get("url") or "", "title": info.get("title")})
|
||||
|
||||
def run(self):
|
||||
self.ws.sock.settimeout(None)
|
||||
self.call("Target.setDiscoverTargets", {"discover": True})
|
||||
self.call("Target.setAutoAttach", {"autoAttach": True, "waitForDebuggerOnStart": False, "flatten": True})
|
||||
for target in (self.call("Target.getTargets") or {}).get("targetInfos", []):
|
||||
if target.get("type") == "page":
|
||||
try:
|
||||
result = self.call("Target.attachToTarget", {"targetId": target["targetId"], "flatten": True})
|
||||
self.attach(result.get("sessionId"), target)
|
||||
except Exception:
|
||||
pass
|
||||
self.emit({"t": "recorder", "state": "started"})
|
||||
while True:
|
||||
while self.pending:
|
||||
self.handle(self.pending.pop(0))
|
||||
self.handle(self.ws.recv_json())
|
||||
|
||||
FILL_LOGIN_JS = r"""
|
||||
(creds) => {
|
||||
if (!creds.expectedHost || location.protocol !== "https:" || location.hostname.toLowerCase() !== creds.expectedHost.toLowerCase()) {
|
||||
return {ok: false, error: "saved login requires the exact configured HTTPS host"};
|
||||
}
|
||||
const user = creds.username || "";
|
||||
const pass = creds.password || "";
|
||||
const inputs = Array.from(document.querySelectorAll("input"));
|
||||
const visible = (el) => {
|
||||
const s = getComputedStyle(el);
|
||||
const r = el.getBoundingClientRect();
|
||||
return s.display !== "none" && s.visibility !== "hidden" && el.type !== "hidden" && r.width > 0 && r.height > 0;
|
||||
};
|
||||
const password = inputs.find((el) => el.type === "password" && visible(el) && !el.disabled);
|
||||
if (!password) return {ok: false, error: "no password field on this page"};
|
||||
const userish = /user|email|login|account|phone|id/i;
|
||||
const username = inputs.find((el) => {
|
||||
if (el === password || !visible(el) || el.disabled) return false;
|
||||
const type = (el.type || "text").toLowerCase();
|
||||
if (["email", "tel", "url"].includes(type)) return true;
|
||||
if (type !== "text" && type !== "search") return false;
|
||||
const blob = [el.name, el.id, el.placeholder, el.autocomplete, el.getAttribute("aria-label")].join(" ");
|
||||
return userish.test(blob) || el === inputs[0];
|
||||
});
|
||||
function setValue(el, value) {
|
||||
const proto = HTMLInputElement.prototype;
|
||||
const desc = Object.getOwnPropertyDescriptor(proto, "value");
|
||||
if (desc && desc.set) desc.set.call(el, value);
|
||||
else el.value = value;
|
||||
el.dispatchEvent(new Event("input", {bubbles: true}));
|
||||
el.dispatchEvent(new Event("change", {bubbles: true}));
|
||||
}
|
||||
if (username) setValue(username, user);
|
||||
setValue(password, pass);
|
||||
return {ok: true, filledUsername: Boolean(username), submitted: false};
|
||||
}
|
||||
"""
|
||||
|
||||
def main():
|
||||
raw = sys.argv[1] if len(sys.argv) > 1 and sys.argv[1].strip() else sys.stdin.read()
|
||||
req = json.loads(raw)
|
||||
action = req.get("action") or "snapshot"
|
||||
display = req.get("display") or ":1"
|
||||
profile = req.get("profile") or ""
|
||||
port = int(req.get("port") or 9222)
|
||||
ensure = bool(req.get("ensure"))
|
||||
|
||||
if action == "probe":
|
||||
print(json.dumps({"ok": probe(port)}))
|
||||
return
|
||||
|
||||
def bound_port():
|
||||
if probe(port):
|
||||
return port
|
||||
bound = active_port(profile)
|
||||
if bound and bound != port and probe(bound):
|
||||
return bound
|
||||
return None
|
||||
|
||||
restarted = False
|
||||
ready_port = bound_port()
|
||||
if ready_port is None and profile_alive(profile):
|
||||
# The window may still be booting (lazyboy-screen just spawned it).
|
||||
for _ in range(12):
|
||||
time.sleep(0.25)
|
||||
ready_port = bound_port()
|
||||
if ready_port is not None:
|
||||
break
|
||||
if ready_port is not None:
|
||||
port = ready_port
|
||||
else:
|
||||
if profile_alive(profile):
|
||||
# Never kill the window the human is watching. Fall back to the
|
||||
# screenshot tools, which see exactly what the live screen shows.
|
||||
fail("browser is open but has no DevTools; use computer_observe/computer_act on it instead. Do not restart the browser.")
|
||||
if not ensure:
|
||||
fail("cdp unavailable")
|
||||
spawn_browser(display, profile, port)
|
||||
ready = False
|
||||
for _ in range(24):
|
||||
time.sleep(0.25)
|
||||
if probe(port):
|
||||
ready = True
|
||||
break
|
||||
if not ready:
|
||||
fail("cdp unavailable")
|
||||
restarted = True
|
||||
|
||||
if action == "ensure":
|
||||
print(json.dumps({"ok": True, "restarted": restarted}))
|
||||
return
|
||||
|
||||
if action == "record":
|
||||
# Long-running: the API starts this detached and kills it on stop.
|
||||
out = req.get("out") or "/tmp/lazyboy-teach.jsonl"
|
||||
Recorder(port, out).run()
|
||||
return
|
||||
|
||||
ws = connect(port)
|
||||
try:
|
||||
if action == "snapshot":
|
||||
body = snapshot(ws)
|
||||
body["restarted"] = restarted
|
||||
print(json.dumps(body))
|
||||
return
|
||||
if action == "navigate":
|
||||
url = req.get("url") or ""
|
||||
if not url:
|
||||
fail("url required")
|
||||
ws.call("Page.navigate", {"url": url})
|
||||
time.sleep(1.2)
|
||||
body = snapshot(ws)
|
||||
body["restarted"] = restarted
|
||||
print(json.dumps(body))
|
||||
return
|
||||
if action == "click":
|
||||
sel = req.get("selector") or ""
|
||||
if not sel:
|
||||
fail("selector required")
|
||||
waited = wait_until_enabled(ws, sel, req.get("waitMs"))
|
||||
if waited is None:
|
||||
state = evaluate(ws, STATE_JS, sel) or {}
|
||||
if not state.get("found"):
|
||||
# Ids are renumbered whenever the page changes; hand back
|
||||
# the fresh numbering so the model does not have to ask.
|
||||
body = snapshot(ws)
|
||||
body.update({"ok": False, "error": "element gone: the page changed and ids were renumbered. Use the fresh element list in this result."})
|
||||
print(json.dumps(body))
|
||||
return
|
||||
fail("control %s is still disabled after waiting %ss (page says: %s). Use wait for longer if a video or timer must finish, then click again."
|
||||
% (sel, int(req.get("waitMs") or CLICK_WAIT_MS) // 1000, state.get("hint") or "nothing"))
|
||||
val = evaluate(ws, CLICK_JS, sel) or {}
|
||||
if not val.get("ok"):
|
||||
fail(val.get("error") or "click failed")
|
||||
pointer(display, val.get("x") or 0, val.get("y") or 0)
|
||||
wait_for_visual_update(ws)
|
||||
out = {"ok": True, "action": "click", "selector": sel, "restarted": restarted}
|
||||
if waited >= 1.0:
|
||||
out["waitedSeconds"] = round(waited, 1)
|
||||
print(json.dumps(out))
|
||||
return
|
||||
if action == "fill_login":
|
||||
val = evaluate(ws, FILL_LOGIN_JS, {
|
||||
"expectedHost": req.get("expectedHost") or "",
|
||||
"username": req.get("username") or "",
|
||||
"password": req.get("password") or "",
|
||||
}) or {}
|
||||
if not val.get("ok"):
|
||||
fail(val.get("error") or "could not fill the login form")
|
||||
wait_for_visual_update(ws)
|
||||
print(json.dumps({
|
||||
"ok": True,
|
||||
"action": "fill_login",
|
||||
"filledUsername": bool(val.get("filledUsername")),
|
||||
"submitted": bool(val.get("submitted")),
|
||||
"restarted": restarted,
|
||||
}))
|
||||
return
|
||||
if action == "type":
|
||||
sel = req.get("selector") or ""
|
||||
text = req.get("text") or ""
|
||||
if sel:
|
||||
val = evaluate(ws, CLICK_JS, sel) or {}
|
||||
if val.get("ok"):
|
||||
pointer(display, val.get("x") or 0, val.get("y") or 0)
|
||||
if text:
|
||||
ws.call("Input.insertText", {"text": text})
|
||||
wait_for_visual_update(ws)
|
||||
print(json.dumps({"ok": True, "action": "type", "restarted": restarted}))
|
||||
return
|
||||
if action == "press":
|
||||
key = req.get("key") or "Return"
|
||||
press(ws, key)
|
||||
wait_for_visual_update(ws)
|
||||
print(json.dumps({"ok": True, "action": "press", "key": key, "restarted": restarted}))
|
||||
return
|
||||
if action == "wait":
|
||||
ms = min(max(int(req.get("ms") or 400), 0), 5000)
|
||||
time.sleep(ms / 1000.0)
|
||||
print(json.dumps({"ok": True, "action": "wait", "ms": ms}))
|
||||
return
|
||||
fail("unsupported action")
|
||||
finally:
|
||||
ws.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception as e:
|
||||
fail(str(e))
|
||||
|
|
@ -1,246 +0,0 @@
|
|||
use lazyboy_contracts::UiElement;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::x11::parse_ui_elements;
|
||||
use crate::{PRIMARY_DISPLAY, normalize_display};
|
||||
|
||||
const CDP_PY: &str = include_str!("cdp.py");
|
||||
|
||||
pub fn devtools_port(display: &str) -> u16 {
|
||||
let number = normalize_display(display)
|
||||
.trim_start_matches(':')
|
||||
.parse::<u16>()
|
||||
.unwrap_or(1)
|
||||
.max(1);
|
||||
9221 + number
|
||||
}
|
||||
|
||||
pub fn cdp_command_on(display: &str, profile: Option<&str>, request: &Value) -> Vec<String> {
|
||||
let mut body = request.clone();
|
||||
if let Some(object) = body.as_object_mut() {
|
||||
object
|
||||
.entry("display")
|
||||
.or_insert_with(|| json!(normalize_display(display)));
|
||||
object
|
||||
.entry("port")
|
||||
.or_insert_with(|| json!(devtools_port(display)));
|
||||
if let Some(profile) = profile.filter(|value| !value.is_empty()) {
|
||||
object.entry("profile").or_insert_with(|| json!(profile));
|
||||
}
|
||||
}
|
||||
vec![
|
||||
"env".into(),
|
||||
format!("DISPLAY={}", normalize_display(display)),
|
||||
"python3".into(),
|
||||
"-c".into(),
|
||||
CDP_PY.into(),
|
||||
body.to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn cdp_command(request: &Value) -> Vec<String> {
|
||||
cdp_command_on(PRIMARY_DISPLAY, None, request)
|
||||
}
|
||||
|
||||
/// Same as `cdp_command_on` but the JSON body is meant to arrive on stdin
|
||||
/// so secrets never appear on the process argv.
|
||||
pub fn cdp_stdin_command_on(display: &str, profile: Option<&str>) -> Vec<String> {
|
||||
let mut env = vec![
|
||||
"env".into(),
|
||||
format!("DISPLAY={}", normalize_display(display)),
|
||||
];
|
||||
if let Some(profile) = profile.filter(|value| !value.is_empty()) {
|
||||
env.push(format!("LAZYBOY_BROWSER_PROFILE={profile}"));
|
||||
}
|
||||
env.extend([
|
||||
"python3".into(),
|
||||
"-c".into(),
|
||||
CDP_PY.into(),
|
||||
]);
|
||||
env
|
||||
}
|
||||
|
||||
/// Marker embedded in the recorder's argv so `pkill -f` can find exactly one
|
||||
/// teaching session without touching other python processes.
|
||||
pub fn teach_recorder_tag(skill_id: &str) -> String {
|
||||
format!("lazyboy-teach-{skill_id}")
|
||||
}
|
||||
|
||||
pub fn teach_recorder_output(skill_id: &str) -> String {
|
||||
format!("/tmp/{}.jsonl", teach_recorder_tag(skill_id))
|
||||
}
|
||||
|
||||
/// Detached, long-running CDP recorder for a human demonstration. The script
|
||||
/// is handed to `sh` as a positional argument so no shell quoting touches it.
|
||||
pub fn cdp_record_command_on(display: &str, profile: Option<&str>, skill_id: &str) -> Vec<String> {
|
||||
let mut request = json!({
|
||||
"action": "record",
|
||||
"ensure": true,
|
||||
"out": teach_recorder_output(skill_id),
|
||||
"tag": teach_recorder_tag(skill_id),
|
||||
"display": normalize_display(display),
|
||||
"port": devtools_port(display),
|
||||
});
|
||||
if let Some(profile) = profile.filter(|value| !value.is_empty()) {
|
||||
request["profile"] = json!(profile);
|
||||
}
|
||||
vec![
|
||||
"sh".into(),
|
||||
"-c".into(),
|
||||
"setsid nohup env DISPLAY=\"$0\" python3 -c \"$1\" \"$2\" >/dev/null 2>&1 </dev/null &".into(),
|
||||
normalize_display(display).to_string(),
|
||||
CDP_PY.into(),
|
||||
request.to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn cdp_record_stop_command(skill_id: &str) -> Vec<String> {
|
||||
vec!["pkill".into(), "-f".into(), teach_recorder_tag(skill_id)]
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Default)]
|
||||
pub struct CdpPage {
|
||||
pub ok: bool,
|
||||
pub error: Option<String>,
|
||||
pub url: String,
|
||||
pub title: String,
|
||||
pub text: String,
|
||||
pub restarted: bool,
|
||||
/// Seconds the click waited for a disabled control to become enabled.
|
||||
pub waited_seconds: Option<f64>,
|
||||
pub elements: Vec<UiElement>,
|
||||
}
|
||||
|
||||
pub fn parse_cdp_page(raw: &str) -> CdpPage {
|
||||
let value: Value = serde_json::from_str(raw.trim()).unwrap_or(Value::Null);
|
||||
let ok = value.get("ok").and_then(Value::as_bool) == Some(true);
|
||||
CdpPage {
|
||||
ok,
|
||||
error: if ok {
|
||||
None
|
||||
} else {
|
||||
value
|
||||
.get("error")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string)
|
||||
.or_else(|| Some("cdp unavailable".into()))
|
||||
},
|
||||
waited_seconds: value.get("waitedSeconds").and_then(Value::as_f64),
|
||||
url: value
|
||||
.get("url")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string(),
|
||||
title: value
|
||||
.get("title")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string(),
|
||||
text: value
|
||||
.get("text")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string(),
|
||||
restarted: value.get("restarted").and_then(Value::as_bool) == Some(true),
|
||||
elements: value
|
||||
.get("elements")
|
||||
.map(|items| parse_ui_elements(&items.to_string()))
|
||||
.unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn merge_page_elements(windows: Vec<UiElement>, page: &[UiElement]) -> Vec<UiElement> {
|
||||
crate::merge_ui_elements(windows, page, &[])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn debug_port_follows_the_display() {
|
||||
assert_eq!(devtools_port(":1"), 9222);
|
||||
assert_eq!(devtools_port(":2"), 9223);
|
||||
assert_eq!(devtools_port("3"), 9224);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn command_passes_port_profile_and_script() {
|
||||
let argv = cdp_command_on(
|
||||
":2",
|
||||
Some("/home/lazyboy/.browser-profiles/bots/a"),
|
||||
&json!({"action": "snapshot", "ensure": true}),
|
||||
);
|
||||
assert!(argv.contains(&"DISPLAY=:2".into()));
|
||||
assert!(argv.iter().any(|item| item.contains("python3")));
|
||||
let payload = argv.last().unwrap();
|
||||
assert!(payload.contains("\"port\":9223"));
|
||||
assert!(payload.contains("bots/a"));
|
||||
assert!(payload.contains("snapshot"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_snapshot_elements() {
|
||||
let page = parse_cdp_page(
|
||||
r#"{"ok":true,"url":"https://example.com","title":"Example","text":"Hello","elements":[{"id":1,"title":"Submit","selector":"[data-lazyboy=\"1\"]","kind":"dom","x":10,"y":20,"w":80,"h":24}]}"#,
|
||||
);
|
||||
assert!(page.ok);
|
||||
assert_eq!(page.url, "https://example.com");
|
||||
assert_eq!(page.elements.len(), 1);
|
||||
assert_eq!(page.elements[0].title, "Submit");
|
||||
assert_eq!(
|
||||
page.elements[0].selector.as_deref(),
|
||||
Some("[data-lazyboy=\"1\"]")
|
||||
);
|
||||
assert_eq!(page.elements[0].center(), (50, 32));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_keeps_page_controls_and_native_dialogs() {
|
||||
let windows = vec![
|
||||
UiElement {
|
||||
id: 1,
|
||||
title: "Chromium".into(),
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: 1280,
|
||||
h: 800,
|
||||
kind: Some("window".into()),
|
||||
..UiElement::default()
|
||||
},
|
||||
UiElement {
|
||||
id: 2,
|
||||
title: "Open File".into(),
|
||||
x: 100,
|
||||
y: 100,
|
||||
w: 400,
|
||||
h: 300,
|
||||
kind: Some("window".into()),
|
||||
..UiElement::default()
|
||||
},
|
||||
];
|
||||
let page = vec![UiElement {
|
||||
id: 1,
|
||||
title: "Login".into(),
|
||||
selector: Some("[data-lazyboy=\"1\"]".into()),
|
||||
kind: Some("dom".into()),
|
||||
x: 40,
|
||||
y: 80,
|
||||
w: 60,
|
||||
h: 20,
|
||||
..UiElement::default()
|
||||
}];
|
||||
let merged = merge_page_elements(windows, &page);
|
||||
assert_eq!(merged.len(), 2);
|
||||
assert_eq!(merged[0].title, "Login");
|
||||
assert_eq!(merged[1].id, 2);
|
||||
assert_eq!(merged[1].title, "Open File");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_page_is_not_ok() {
|
||||
let page = parse_cdp_page(r#"{"ok":false,"error":"cdp unavailable"}"#);
|
||||
assert!(!page.ok);
|
||||
assert_eq!(page.error.as_deref(), Some("cdp unavailable"));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
"""Set X11 clipboard, confirm ownership/content, then paste into the active app."""
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def run(argv, **kwargs):
|
||||
return subprocess.run(argv, check=True, timeout=2, **kwargs)
|
||||
|
||||
|
||||
def paste(text):
|
||||
if not text:
|
||||
return
|
||||
raw = text.encode('utf-8')
|
||||
# xclip forks after reading stdin; detached descriptors avoid pipe hangs.
|
||||
run(['xclip', '-selection', 'clipboard', '-in'], input=raw,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
deadline = time.monotonic() + 2
|
||||
while True:
|
||||
actual = run(['xclip', '-selection', 'clipboard', '-out'], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL).stdout
|
||||
if actual == raw:
|
||||
break
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError('clipboard synchronization timed out; nothing pasted')
|
||||
time.sleep(.02)
|
||||
key_for_active_app('v')
|
||||
|
||||
|
||||
def key_for_active_app(key):
|
||||
window = run(['xdotool', 'getactivewindow'], stdout=subprocess.PIPE).stdout.decode().strip()
|
||||
wmclass = run(['xprop', '-id', window, 'WM_CLASS'], stdout=subprocess.PIPE).stdout.decode().lower()
|
||||
terminal = any(name in wmclass for name in ('terminal', 'xterm', 'kitty', 'alacritty', 'konsole'))
|
||||
run(['xdotool', 'key', '--clearmodifiers', ('ctrl+shift+' if terminal else 'ctrl+') + key],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def copy_selection():
|
||||
key_for_active_app('c')
|
||||
# Wait for the application to process the shortcut before reading selection.
|
||||
time.sleep(.1)
|
||||
return run(['xclip', '-selection', 'clipboard', '-out'], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL).stdout.decode('utf-8')
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv)>1 and sys.argv[1]=='copy':
|
||||
sys.stdout.write(copy_selection())
|
||||
else:
|
||||
paste(sys.stdin.read())
|
||||
|
|
@ -0,0 +1,214 @@
|
|||
use std::str::FromStr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::cua::CuaController;
|
||||
use crate::{
|
||||
ActionRequest, ActionResult, BrowserPage, BrowserRequest, RecordingRequest, RecordingResult,
|
||||
RecordingSession,
|
||||
};
|
||||
use lazyboy_contracts::ComputerObservation;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ControlContext {
|
||||
pub display: String,
|
||||
pub profile_path: Option<String>,
|
||||
}
|
||||
|
||||
impl ControlContext {
|
||||
pub fn new(display: impl Into<String>, profile_path: Option<String>) -> Self {
|
||||
Self {
|
||||
display: display.into(),
|
||||
profile_path,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ControllerHealth {
|
||||
pub backend: String,
|
||||
pub version: Option<String>,
|
||||
pub healthy: bool,
|
||||
pub degraded: bool,
|
||||
pub details: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Error, Clone, PartialEq, Eq)]
|
||||
pub enum ControlError {
|
||||
#[error("computer driver is not installed")]
|
||||
DriverUnavailable,
|
||||
#[error("computer driver is unhealthy")]
|
||||
DriverUnhealthy,
|
||||
#[error("display is unavailable")]
|
||||
DisplayUnavailable,
|
||||
#[error("accessibility is unavailable")]
|
||||
AccessibilityUnavailable,
|
||||
#[error("browser is unavailable")]
|
||||
BrowserUnavailable,
|
||||
#[error("target not found")]
|
||||
TargetNotFound,
|
||||
#[error("stale UI reference; take a fresh observation")]
|
||||
StaleReference,
|
||||
#[error("permission denied")]
|
||||
PermissionDenied,
|
||||
#[error("computer action timed out")]
|
||||
Timeout,
|
||||
#[error("unsupported computer action")]
|
||||
Unsupported,
|
||||
#[error("computer is busy")]
|
||||
Busy,
|
||||
#[error("{0}")]
|
||||
InvalidAction(String),
|
||||
#[error("{0}")]
|
||||
Internal(String),
|
||||
}
|
||||
|
||||
impl ControlError {
|
||||
pub fn internal(text: impl Into<String>) -> Self {
|
||||
let text = text.into();
|
||||
Self::Internal(truncate_error(&text))
|
||||
}
|
||||
|
||||
pub fn is_client_error(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::TargetNotFound
|
||||
| Self::StaleReference
|
||||
| Self::Unsupported
|
||||
| Self::InvalidAction(_)
|
||||
| Self::PermissionDenied
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate_error(text: &str) -> String {
|
||||
const LIMIT: usize = 800;
|
||||
let trimmed = text.trim();
|
||||
if trimmed.len() <= LIMIT {
|
||||
trimmed.to_string()
|
||||
} else {
|
||||
format!("{}…", &trimmed[..trimmed.floor_char_boundary(LIMIT)])
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ComputerDriver {
|
||||
Cua,
|
||||
}
|
||||
|
||||
impl ComputerDriver {
|
||||
pub const ENV: &'static str = "LAZYBOY_COMPUTER_DRIVER";
|
||||
|
||||
pub fn from_env() -> Self {
|
||||
if let Ok(value) = std::env::var(Self::ENV)
|
||||
&& !value.trim().is_empty()
|
||||
&& value.parse::<Self>().is_err()
|
||||
{
|
||||
tracing::warn!(
|
||||
"Only the Cua computer driver is supported; ignoring obsolete driver setting"
|
||||
);
|
||||
}
|
||||
Self::Cua
|
||||
}
|
||||
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Cua => "cua",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn controller(self) -> Arc<dyn ComputerController> {
|
||||
match self {
|
||||
Self::Cua => Arc::new(CuaController::default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for ComputerDriver {
|
||||
type Err = UnknownComputerDriver;
|
||||
|
||||
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"cua" => Ok(Self::Cua),
|
||||
other => Err(UnknownComputerDriver(other.to_string())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error, Clone, PartialEq, Eq)]
|
||||
#[error("unknown computer driver {0:?}; expected cua")]
|
||||
pub struct UnknownComputerDriver(pub String);
|
||||
|
||||
#[async_trait]
|
||||
pub trait ComputerController: Send + Sync {
|
||||
fn backend(&self) -> ComputerDriver;
|
||||
|
||||
async fn health(&self, ctx: &ControlContext) -> Result<ControllerHealth, ControlError>;
|
||||
|
||||
async fn observe(&self, ctx: &ControlContext) -> Result<ComputerObservation, ControlError>;
|
||||
|
||||
async fn act(
|
||||
&self,
|
||||
request: &ActionRequest,
|
||||
ctx: &ControlContext,
|
||||
) -> Result<ActionResult, ControlError>;
|
||||
|
||||
async fn browser(
|
||||
&self,
|
||||
request: &BrowserRequest,
|
||||
ctx: &ControlContext,
|
||||
) -> Result<BrowserPage, ControlError>;
|
||||
|
||||
async fn start_recording(
|
||||
&self,
|
||||
request: &RecordingRequest,
|
||||
ctx: &ControlContext,
|
||||
) -> Result<RecordingSession, ControlError>;
|
||||
|
||||
async fn stop_recording(
|
||||
&self,
|
||||
request: &RecordingRequest,
|
||||
ctx: &ControlContext,
|
||||
) -> Result<(), ControlError>;
|
||||
|
||||
async fn collect_recording(
|
||||
&self,
|
||||
request: &RecordingRequest,
|
||||
ctx: &ControlContext,
|
||||
) -> Result<RecordingResult, ControlError>;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_driver_names() {
|
||||
assert!("legacy".parse::<ComputerDriver>().is_err());
|
||||
assert_eq!(
|
||||
"CUA".parse::<ComputerDriver>().unwrap(),
|
||||
ComputerDriver::Cua
|
||||
);
|
||||
assert!("xdotool".parse::<ComputerDriver>().is_err());
|
||||
assert_eq!(ComputerDriver::Cua.as_str(), "cua");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_unicode_errors_do_not_panic() {
|
||||
let text = "錯".repeat(300);
|
||||
let error = ControlError::internal(&text).to_string();
|
||||
assert!(error.ends_with('…'));
|
||||
assert!(error.len() <= 803);
|
||||
assert!(text.starts_with(error.trim_end_matches('…')));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_browser_action_is_a_client_error() {
|
||||
let error = ControlError::InvalidAction(
|
||||
"browser navigate only accepts http, https, or about URLs".into(),
|
||||
);
|
||||
assert!(error.is_client_error());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,707 @@
|
|||
use lazyboy_contracts::UiElement;
|
||||
use serde_json::{Value, json};
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
use super::ListedWindow;
|
||||
use super::client::CuaClient;
|
||||
use crate::controller::ControlError;
|
||||
use crate::{BrowserPage, BrowserRequest, launch_argv_on};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BrowserBind {
|
||||
pub pid: u64,
|
||||
pub window_id: u64,
|
||||
pub target_id: String,
|
||||
pub tab_id: String,
|
||||
pub page: Option<BrowserPage>,
|
||||
}
|
||||
|
||||
pub fn is_cua_ref(selector: &str) -> bool {
|
||||
let mut parts = selector.split(':');
|
||||
matches!(
|
||||
(parts.next(), parts.next(), parts.next()),
|
||||
(Some(prefix), Some(index), None)
|
||||
if prefix.starts_with('p')
|
||||
&& prefix[1..].chars().all(|ch| ch.is_ascii_digit())
|
||||
&& !prefix[1..].is_empty()
|
||||
&& index.chars().all(|ch| ch.is_ascii_digit())
|
||||
&& !index.is_empty()
|
||||
)
|
||||
}
|
||||
|
||||
pub fn allowed_navigate_url(url: &str) -> bool {
|
||||
url.starts_with("http://") || url.starts_with("https://") || url.starts_with("about:")
|
||||
}
|
||||
|
||||
pub fn page_from_semantic(value: &Value) -> BrowserPage {
|
||||
let page = value.get("page").unwrap_or(value);
|
||||
let url = page
|
||||
.get("url")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let title = page
|
||||
.get("title")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let outline = value.get("outline").and_then(Value::as_str).unwrap_or("");
|
||||
let mut elements = Vec::new();
|
||||
for (index, item) in value
|
||||
.get("refs")
|
||||
.and_then(Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.enumerate()
|
||||
{
|
||||
let Some(element) = element_from_ref(index as u32 + 1, item) else {
|
||||
continue;
|
||||
};
|
||||
elements.push(element);
|
||||
}
|
||||
let ok = value.get("status").and_then(Value::as_str) != Some("refused")
|
||||
&& value.get("ok").and_then(Value::as_bool) != Some(false);
|
||||
BrowserPage {
|
||||
ok,
|
||||
error: if ok {
|
||||
None
|
||||
} else {
|
||||
value
|
||||
.get("message")
|
||||
.or_else(|| value.get("error"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string)
|
||||
},
|
||||
url,
|
||||
title,
|
||||
text: outline.to_string(),
|
||||
restarted: false,
|
||||
waited_seconds: None,
|
||||
elements,
|
||||
}
|
||||
}
|
||||
|
||||
fn element_from_ref(id: u32, item: &Value) -> Option<UiElement> {
|
||||
let selector = item.get("ref").and_then(Value::as_str)?.to_string();
|
||||
if selector.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let name = item
|
||||
.get("name")
|
||||
.or_else(|| item.get("label"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let role = item
|
||||
.get("role")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|role| !role.is_empty())
|
||||
.map(str::to_string);
|
||||
let visibility = item
|
||||
.get("visibility")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("in_viewport");
|
||||
let (x, y, w, h) = match item.get("frame") {
|
||||
Some(frame) if frame.is_object() => (
|
||||
number(frame, "x").unwrap_or(0),
|
||||
number(frame, "y").unwrap_or(0),
|
||||
number(frame, "w")
|
||||
.or_else(|| number(frame, "width"))
|
||||
.unwrap_or(0),
|
||||
number(frame, "h")
|
||||
.or_else(|| number(frame, "height"))
|
||||
.unwrap_or(0),
|
||||
),
|
||||
_ if visibility == "in_viewport" => (0, 0, 1, 1),
|
||||
_ => (0, 0, 0, 0),
|
||||
};
|
||||
Some(UiElement {
|
||||
id,
|
||||
title: if name.is_empty() {
|
||||
selector.clone()
|
||||
} else {
|
||||
name
|
||||
},
|
||||
x,
|
||||
y,
|
||||
w,
|
||||
h,
|
||||
selector: Some(selector),
|
||||
kind: Some("dom".into()),
|
||||
role,
|
||||
})
|
||||
}
|
||||
|
||||
fn number(value: &Value, key: &str) -> Option<u32> {
|
||||
value
|
||||
.get(key)
|
||||
.and_then(Value::as_u64)
|
||||
.or_else(|| {
|
||||
value
|
||||
.get(key)
|
||||
.and_then(Value::as_f64)
|
||||
.filter(|n| *n >= 0.0)
|
||||
.map(|n| n as u64)
|
||||
})
|
||||
.map(|n| n as u32)
|
||||
}
|
||||
|
||||
pub fn find_ref<'a>(page: &'a BrowserPage, selector: &'a str) -> Option<&'a str> {
|
||||
if is_cua_ref(selector) {
|
||||
return page
|
||||
.elements
|
||||
.iter()
|
||||
.find(|element| element.selector.as_deref() == Some(selector))
|
||||
.and_then(|element| element.selector.as_deref());
|
||||
}
|
||||
if let Ok(id) = selector.parse::<u32>() {
|
||||
return page
|
||||
.elements
|
||||
.iter()
|
||||
.find(|element| element.id == id)
|
||||
.and_then(|element| element.selector.as_deref());
|
||||
}
|
||||
// Labels are accepted only when exact and unique; do not reinterpret CSS
|
||||
// fragments as substring matches that can click a different control.
|
||||
if selector.trim().is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut matches = page
|
||||
.elements
|
||||
.iter()
|
||||
.filter(|element| element.title == selector);
|
||||
let first = matches.next()?;
|
||||
if matches.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
first.selector.as_deref()
|
||||
}
|
||||
|
||||
pub fn chromium_window(windows: &[ListedWindow]) -> Option<&ListedWindow> {
|
||||
windows.iter().find(|window| {
|
||||
let blob = format!("{} {}", window.title, window.app_name).to_ascii_lowercase();
|
||||
blob.contains("chrom")
|
||||
})
|
||||
}
|
||||
|
||||
fn ids_from(value: &Value) -> Option<(String, String)> {
|
||||
let mut target_id = None;
|
||||
let mut tab_id = None;
|
||||
let mut nodes = Vec::new();
|
||||
super::client::walk(value, &mut nodes);
|
||||
for node in nodes {
|
||||
if target_id.is_none() {
|
||||
target_id = node
|
||||
.get("target_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string);
|
||||
}
|
||||
if tab_id.is_none() {
|
||||
tab_id = node
|
||||
.get("tab_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string);
|
||||
}
|
||||
if let Some(tabs) = node.get("tabs").and_then(Value::as_array)
|
||||
&& let Some(first) = tabs.first()
|
||||
&& tab_id.is_none()
|
||||
{
|
||||
tab_id = first
|
||||
.get("tab_id")
|
||||
.or_else(|| first.get("id"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string);
|
||||
}
|
||||
}
|
||||
Some((target_id?, tab_id?))
|
||||
}
|
||||
|
||||
pub async fn ensure_bind(
|
||||
client: &CuaClient,
|
||||
display: &str,
|
||||
profile: Option<&str>,
|
||||
ensure: bool,
|
||||
windows: &[ListedWindow],
|
||||
) -> Result<BrowserBind, ControlError> {
|
||||
let mut listed = windows.to_vec();
|
||||
if chromium_window(&listed).is_none() && ensure {
|
||||
if let Some(argv) = launch_argv_on(display, profile, "browser", None) {
|
||||
super::launch::run(client, display, &argv).await?;
|
||||
}
|
||||
for _ in 0..24 {
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
listed = super::CuaController::list_windows_now(client, display).await?;
|
||||
if chromium_window(&listed).is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
let window = chromium_window(&listed)
|
||||
.cloned()
|
||||
.ok_or(ControlError::BrowserUnavailable)?;
|
||||
attach(client, display, &window).await
|
||||
}
|
||||
|
||||
async fn attach(
|
||||
client: &CuaClient,
|
||||
display: &str,
|
||||
window: &ListedWindow,
|
||||
) -> Result<BrowserBind, ControlError> {
|
||||
client
|
||||
.call(display, "start_session", &json!({}), &[])
|
||||
.await?;
|
||||
let pid = window.pid;
|
||||
let window_id = window.id;
|
||||
let prepare = client
|
||||
.call(
|
||||
display,
|
||||
"browser_prepare",
|
||||
&json!({
|
||||
"pid": pid,
|
||||
"window_id": window_id,
|
||||
"strategy": { "kind": "existing_profile" },
|
||||
"allow_launch": false,
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
if let Err(error) = &prepare {
|
||||
let text = error.to_string();
|
||||
if !text.contains("consent") && !text.contains("prepare") && !text.contains("grant") {
|
||||
tracing::warn!(error = %error, "browser_prepare failed");
|
||||
}
|
||||
}
|
||||
let state = client
|
||||
.call(
|
||||
display,
|
||||
"get_browser_state",
|
||||
&json!({
|
||||
"pid": pid,
|
||||
"window_id": window_id,
|
||||
"include_screenshot": false,
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
let (target_id, tab_id) = ids_from(&state).ok_or(ControlError::BrowserUnavailable)?;
|
||||
Ok(BrowserBind {
|
||||
pid,
|
||||
window_id,
|
||||
target_id,
|
||||
tab_id,
|
||||
page: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn snapshot(
|
||||
client: &CuaClient,
|
||||
display: &str,
|
||||
bind: &BrowserBind,
|
||||
) -> Result<BrowserPage, ControlError> {
|
||||
let value = client
|
||||
.call(
|
||||
display,
|
||||
"get_browser_state",
|
||||
&json!({
|
||||
"target_id": bind.target_id,
|
||||
"tab_id": bind.tab_id,
|
||||
"snapshot_format": "semantic_v2",
|
||||
"include_screenshot": false,
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
Ok(page_from_semantic(&value))
|
||||
}
|
||||
|
||||
pub async fn run(
|
||||
client: &CuaClient,
|
||||
display: &str,
|
||||
profile: Option<&str>,
|
||||
request: &BrowserRequest,
|
||||
windows: &[ListedWindow],
|
||||
bind: &mut Option<BrowserBind>,
|
||||
) -> Result<BrowserPage, ControlError> {
|
||||
if request.action == "probe" {
|
||||
return Ok(BrowserPage {
|
||||
ok: chromium_window(windows).is_some(),
|
||||
..BrowserPage::default()
|
||||
});
|
||||
}
|
||||
let attached = match bind.as_ref() {
|
||||
Some(current) => current.clone(),
|
||||
None => {
|
||||
let attached = ensure_bind(client, display, profile, request.ensure, windows).await?;
|
||||
*bind = Some(attached.clone());
|
||||
attached
|
||||
}
|
||||
};
|
||||
if request.action == "ensure" {
|
||||
return Ok(BrowserPage {
|
||||
ok: true,
|
||||
..BrowserPage::default()
|
||||
});
|
||||
}
|
||||
if matches!(
|
||||
request.action.as_str(),
|
||||
"navigate" | "click" | "type" | "press"
|
||||
) {
|
||||
client
|
||||
.call(
|
||||
display,
|
||||
"bring_to_front",
|
||||
&json!({
|
||||
"pid": attached.pid, "window_id": attached.window_id,
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
match request.action.as_str() {
|
||||
"snapshot" => snapshot(client, display, &attached).await,
|
||||
"wait" => {
|
||||
let ms = request.ms.unwrap_or(400).min(5000);
|
||||
sleep(Duration::from_millis(ms)).await;
|
||||
snapshot(client, display, &attached).await
|
||||
}
|
||||
"navigate" => {
|
||||
let url = request.url.as_deref().unwrap_or("");
|
||||
if !allowed_navigate_url(url) {
|
||||
return Err(ControlError::InvalidAction(
|
||||
"browser navigate only accepts http, https, or about URLs".into(),
|
||||
));
|
||||
}
|
||||
client
|
||||
.call(
|
||||
display,
|
||||
"browser_navigate",
|
||||
&json!({
|
||||
"target_id": attached.target_id,
|
||||
"tab_id": attached.tab_id,
|
||||
"url": url,
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
sleep(Duration::from_millis(800)).await;
|
||||
snapshot(client, display, &attached).await
|
||||
}
|
||||
"click" => click(client, display, &attached, request).await,
|
||||
"type" => type_into(client, display, &attached, request).await,
|
||||
"press" => {
|
||||
let key = map_press_key(request.key.as_deref().unwrap_or("return"));
|
||||
client
|
||||
.call(
|
||||
display,
|
||||
"press_key",
|
||||
&json!({
|
||||
"key": key,
|
||||
"pid": attached.pid,
|
||||
"window_id": attached.window_id,
|
||||
"delivery_mode": "foreground",
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
snapshot(client, display, &attached).await
|
||||
}
|
||||
other => Err(ControlError::InvalidAction(format!(
|
||||
"unsupported browser action {other}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn click(
|
||||
client: &CuaClient,
|
||||
display: &str,
|
||||
bind: &BrowserBind,
|
||||
request: &BrowserRequest,
|
||||
) -> Result<BrowserPage, ControlError> {
|
||||
let selector = request
|
||||
.selector
|
||||
.as_deref()
|
||||
.ok_or_else(|| ControlError::InvalidAction("browser click needs a selector".into()))?;
|
||||
let page = bind.page.as_ref().ok_or(ControlError::StaleReference)?;
|
||||
let Some(r#ref) = find_ref(page, selector) else {
|
||||
return Ok(BrowserPage {
|
||||
ok: false,
|
||||
error: Some(
|
||||
"element gone: the page changed and ids were renumbered. Use the fresh element list in this result."
|
||||
.into(),
|
||||
),
|
||||
url: page.url.clone(),
|
||||
title: page.title.clone(),
|
||||
text: page.text.clone(),
|
||||
elements: page.elements.clone(),
|
||||
..BrowserPage::default()
|
||||
});
|
||||
};
|
||||
let r#ref = r#ref.to_string();
|
||||
client
|
||||
.call(
|
||||
display,
|
||||
"browser_click",
|
||||
&json!({
|
||||
"target_id": bind.target_id,
|
||||
"tab_id": bind.tab_id,
|
||||
"ref": r#ref,
|
||||
"input_route": "dom_event",
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
snapshot(client, display, bind).await
|
||||
}
|
||||
|
||||
fn unique_native_web_entry<'a>(state: &'a Value, label: &str) -> Result<&'a Value, ControlError> {
|
||||
if label.is_empty() {
|
||||
return Err(ControlError::TargetNotFound);
|
||||
}
|
||||
let mut matches = state
|
||||
.get("elements")
|
||||
.and_then(Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter(|element| {
|
||||
element.get("label").and_then(Value::as_str) == Some(label)
|
||||
&& element.get("in_web_content").and_then(Value::as_bool) == Some(true)
|
||||
&& matches!(
|
||||
element.get("role").and_then(Value::as_str),
|
||||
Some("entry" | "password text" | "text")
|
||||
)
|
||||
&& element.get("frame").is_some_and(|frame| {
|
||||
frame["w"].as_f64().unwrap_or(0.0) > 0.0
|
||||
&& frame["h"].as_f64().unwrap_or(0.0) > 0.0
|
||||
})
|
||||
});
|
||||
let first = matches.next().ok_or(ControlError::TargetNotFound)?;
|
||||
if matches.next().is_some() {
|
||||
return Err(ControlError::TargetNotFound);
|
||||
}
|
||||
Ok(first)
|
||||
}
|
||||
|
||||
async fn type_into(
|
||||
client: &CuaClient,
|
||||
display: &str,
|
||||
bind: &BrowserBind,
|
||||
request: &BrowserRequest,
|
||||
) -> Result<BrowserPage, ControlError> {
|
||||
let text = request.text.clone().unwrap_or_default();
|
||||
tracing::info!(backend = "cua", tool = "browser_type", length = text.len());
|
||||
if let Some(selector) = request.selector.as_deref() {
|
||||
let page = bind.page.as_ref().ok_or(ControlError::StaleReference)?;
|
||||
let Some(r#ref) = find_ref(page, selector) else {
|
||||
return Ok(BrowserPage {
|
||||
ok: false,
|
||||
error: Some("target field is unavailable; no text inserted".into()),
|
||||
url: page.url.clone(),
|
||||
title: page.title.clone(),
|
||||
text: page.text.clone(),
|
||||
elements: page.elements.clone(),
|
||||
..BrowserPage::default()
|
||||
});
|
||||
};
|
||||
let r#ref = r#ref.to_string();
|
||||
let typed = client
|
||||
.call(
|
||||
display,
|
||||
"browser_type",
|
||||
&json!({
|
||||
"target_id": bind.target_id,
|
||||
"tab_id": bind.tab_id,
|
||||
"ref": r#ref,
|
||||
"text": text,
|
||||
"replace": true,
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
match typed {
|
||||
Ok(_) => {}
|
||||
Err(ControlError::Unsupported) => {
|
||||
// The pinned driver can refuse Input.insertText for email
|
||||
// fields. Resolve a unique visible native web entry from Cua;
|
||||
// never guess a pixel or a similarly named browser-chrome field.
|
||||
let label = page
|
||||
.elements
|
||||
.iter()
|
||||
.find(|element| element.selector.as_deref() == Some(r#ref.as_str()))
|
||||
.map(|element| element.title.as_str())
|
||||
.ok_or(ControlError::StaleReference)?;
|
||||
let native = client.call(display, "get_window_state", &json!({
|
||||
"pid": bind.pid, "window_id": bind.window_id, "include_screenshot": false,
|
||||
}), &[]).await?;
|
||||
let entry = unique_native_web_entry(&native, label)?;
|
||||
let frame = &entry["frame"];
|
||||
let x = frame["x"].as_f64().ok_or(ControlError::TargetNotFound)?
|
||||
+ frame["w"].as_f64().unwrap_or(0.0) / 2.0;
|
||||
let y = frame["y"].as_f64().ok_or(ControlError::TargetNotFound)?
|
||||
+ frame["h"].as_f64().unwrap_or(0.0) / 2.0;
|
||||
client
|
||||
.call(
|
||||
display,
|
||||
"click",
|
||||
&json!({
|
||||
"x": x, "y": y, "scope": "desktop",
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
client
|
||||
.call(
|
||||
display,
|
||||
"hotkey",
|
||||
&json!({
|
||||
"pid": bind.pid, "window_id": bind.window_id,
|
||||
"keys": ["ctrl", "a"], "delivery_mode": "foreground",
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
if text.is_empty() {
|
||||
client
|
||||
.call(
|
||||
display,
|
||||
"press_key",
|
||||
&json!({
|
||||
"pid": bind.pid, "window_id": bind.window_id,
|
||||
"key": "backspace", "delivery_mode": "foreground",
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
super::clipboard::paste(client, display, &text).await?;
|
||||
}
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
} else if !text.is_ascii() || text.contains('\n') {
|
||||
super::clipboard::paste(client, display, &text).await?;
|
||||
} else if !text.is_empty() {
|
||||
client
|
||||
.call(
|
||||
display,
|
||||
"type_text",
|
||||
&json!({
|
||||
"text": text,
|
||||
"pid": bind.pid,
|
||||
"window_id": bind.window_id,
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
snapshot(client, display, bind).await
|
||||
}
|
||||
|
||||
fn map_press_key(key: &str) -> String {
|
||||
match key.to_ascii_lowercase().as_str() {
|
||||
"enter" | "return" => "return".into(),
|
||||
"esc" | "escape" => "escape".into(),
|
||||
other => other.to_ascii_lowercase(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn native_typing_requires_a_unique_visible_web_field() {
|
||||
let entry = json!({"label":"Email","role":"entry","in_web_content":true,"frame":{"w":100,"h":20},"element_token":"s1:1"});
|
||||
assert_eq!(
|
||||
unique_native_web_entry(&json!({"elements":[entry.clone()]}), "Email").unwrap()["element_token"],
|
||||
"s1:1"
|
||||
);
|
||||
assert!(
|
||||
unique_native_web_entry(&json!({"elements":[entry.clone(),entry.clone()]}), "Email")
|
||||
.is_err()
|
||||
);
|
||||
let mut chrome = entry.clone();
|
||||
chrome["in_web_content"] = json!(false);
|
||||
assert!(unique_native_web_entry(&json!({"elements":[chrome]}), "Email").is_err());
|
||||
let mut hidden = entry;
|
||||
hidden["frame"]["w"] = json!(0);
|
||||
assert!(unique_native_web_entry(&json!({"elements":[hidden]}), "Email").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_snapshot_scoped_refs() {
|
||||
assert!(is_cua_ref("p1:1"));
|
||||
assert!(is_cua_ref("p28:12"));
|
||||
assert!(!is_cua_ref("#submit"));
|
||||
assert!(!is_cua_ref("button.primary"));
|
||||
assert!(!is_cua_ref("p:1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn semantic_snapshot_becomes_browser_page() {
|
||||
let raw = json!({
|
||||
"status": "ok",
|
||||
"outline": "- button \"Smoke Click\"\n- textbox \"Smoke Entry\"",
|
||||
"page": { "title": "LazyBoy Cua Smoke", "url": "http://127.0.0.1:8765/cua-smoke.html" },
|
||||
"content_refs": [{ "ref": "p1:0", "name": "Page heading", "role": "heading" }],
|
||||
"refs": [
|
||||
{ "name": "Smoke Click", "ref": "p1:1", "role": "button", "frame": "main", "visibility": "in_viewport" },
|
||||
{ "name": "Smoke Entry", "ref": "p1:2", "role": "textbox", "visibility": "in_viewport" }
|
||||
]
|
||||
});
|
||||
let page = page_from_semantic(&raw);
|
||||
assert!(page.ok);
|
||||
assert_eq!(page.title, "LazyBoy Cua Smoke");
|
||||
assert_eq!(page.elements.len(), 2);
|
||||
assert_eq!(page.elements[0].id, 1);
|
||||
assert_eq!(page.elements[0].selector.as_deref(), Some("p1:1"));
|
||||
assert_eq!(page.elements[0].kind.as_deref(), Some("dom"));
|
||||
assert!(!page.elements[0].is_offscreen());
|
||||
assert_eq!(find_ref(&page, "1"), Some("p1:1"));
|
||||
assert_eq!(find_ref(&page, "p1:1"), Some("p1:1"));
|
||||
assert_eq!(find_ref(&page, "p9:1"), None);
|
||||
assert_eq!(find_ref(&page, ""), None);
|
||||
assert_eq!(find_ref(&page, "#"), None);
|
||||
assert_eq!(find_ref(&page, "Smoke Entry"), Some("p1:2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn navigate_accepts_http_https_about_only() {
|
||||
assert!(allowed_navigate_url("https://example.com"));
|
||||
assert!(allowed_navigate_url("http://127.0.0.1:8765/cua-smoke.html"));
|
||||
assert!(allowed_navigate_url("about:blank"));
|
||||
assert!(!allowed_navigate_url("file:///tmp/x.html"));
|
||||
assert!(!allowed_navigate_url("javascript:alert(1)"));
|
||||
assert!(!allowed_navigate_url(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chromium_window_matches_title_or_app() {
|
||||
let chrome = ListedWindow {
|
||||
id: 9,
|
||||
pid: 334,
|
||||
title: "LazyBoy Cua Smoke - Chromium".into(),
|
||||
app_name: "Chromium".into(),
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: 1280,
|
||||
h: 800,
|
||||
z: 2,
|
||||
};
|
||||
let terminal = ListedWindow {
|
||||
id: 3,
|
||||
pid: 20,
|
||||
title: "終端機".into(),
|
||||
app_name: "xfce4-terminal".into(),
|
||||
x: 10,
|
||||
y: 10,
|
||||
w: 400,
|
||||
h: 300,
|
||||
z: 1,
|
||||
};
|
||||
assert!(chromium_window(&[terminal.clone(), chrome.clone()]).is_some());
|
||||
assert!(chromium_window(&[terminal]).is_none());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,589 @@
|
|||
use std::path::PathBuf;
|
||||
use std::process::Stdio;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde_json::{Value, json};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::process::Command;
|
||||
|
||||
use crate::controller::ControlError;
|
||||
use crate::screen::normalize_display;
|
||||
|
||||
pub const PRIMARY_SOCKET: &str = "/tmp/lazyboy/cua.sock";
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CuaClient {
|
||||
bin: PathBuf,
|
||||
}
|
||||
|
||||
impl Default for CuaClient {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
bin: PathBuf::from("cua-driver"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CuaClient {
|
||||
pub fn socket_for_display(display: &str) -> PathBuf {
|
||||
let number = normalize_display(display)
|
||||
.trim_start_matches(':')
|
||||
.to_string();
|
||||
if number == "1" {
|
||||
PathBuf::from(PRIMARY_SOCKET)
|
||||
} else {
|
||||
PathBuf::from(format!("/tmp/lazyboy/cua-{number}.sock"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Every call runs in its own CLI process, so the driver would otherwise
|
||||
/// give each one an implicit session that dies with the process. Trajectory
|
||||
/// recording, snapshots, and browser binds only line up under one label.
|
||||
pub fn session_for_display(display: &str) -> String {
|
||||
let number = normalize_display(display)
|
||||
.trim_start_matches(':')
|
||||
.to_string();
|
||||
if let Ok(name) =
|
||||
std::fs::read_to_string(format!("/tmp/lazyboy/screen-{number}.agent-name"))
|
||||
{
|
||||
let name = public_agent_name(&name);
|
||||
if !name.is_empty() {
|
||||
return name;
|
||||
}
|
||||
}
|
||||
format!(
|
||||
"lazyboy-{}",
|
||||
normalize_display(display).trim_start_matches(':')
|
||||
)
|
||||
}
|
||||
|
||||
pub fn dbus_file(display: &str) -> PathBuf {
|
||||
let number = normalize_display(display)
|
||||
.trim_start_matches(':')
|
||||
.to_string();
|
||||
PathBuf::from(format!("/tmp/lazyboy/screen-{number}.dbus"))
|
||||
}
|
||||
|
||||
pub async fn version(&self) -> Result<String, ControlError> {
|
||||
let mut command = Command::new(&self.bin);
|
||||
command.arg("--version");
|
||||
let output = bounded_output(&mut command, Duration::from_secs(10)).await?;
|
||||
if !output.status.success() {
|
||||
return Err(ControlError::DriverUnavailable);
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
|
||||
}
|
||||
|
||||
pub async fn status(&self, display: &str) -> Result<String, ControlError> {
|
||||
let socket = Self::socket_for_display(display);
|
||||
if !socket.exists() {
|
||||
return Err(ControlError::DriverUnavailable);
|
||||
}
|
||||
let mut command = Command::new(&self.bin);
|
||||
command.args(["status", "--socket", &socket.to_string_lossy()]);
|
||||
let output = bounded_output(&mut command, Duration::from_secs(10)).await?;
|
||||
let text = format!(
|
||||
"{}{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
if !output.status.success() || text.to_ascii_lowercase().contains("not running") {
|
||||
return Err(ControlError::DriverUnhealthy);
|
||||
}
|
||||
Ok(text)
|
||||
}
|
||||
|
||||
pub async fn call(
|
||||
&self,
|
||||
screen: &str,
|
||||
tool: &str,
|
||||
payload: &Value,
|
||||
extra: &[&str],
|
||||
) -> Result<Value, ControlError> {
|
||||
let mut body = with_session_label(screen, payload);
|
||||
let mut escalated = false;
|
||||
let mut revived = false;
|
||||
loop {
|
||||
let outcome = self.attempt(screen, tool, &body, extra).await?;
|
||||
if outcome.session_ended && !revived && tool != "start_session" {
|
||||
let session = with_session_label(screen, &json!({}));
|
||||
let started = self.attempt(screen, "start_session", &session, &[]).await?;
|
||||
if let Some(error) = started.error {
|
||||
return Err(error);
|
||||
}
|
||||
revived = true;
|
||||
if !read_after_session_restart(tool) {
|
||||
return Err(ControlError::StaleReference);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if !escalated
|
||||
&& outcome.error.is_some()
|
||||
&& let Some(mode) = recommended_delivery(&outcome.value)
|
||||
&& let Some(map) = body.as_object_mut()
|
||||
{
|
||||
map.insert("delivery_mode".to_string(), json!(mode));
|
||||
escalated = true;
|
||||
continue;
|
||||
}
|
||||
return match outcome.error {
|
||||
Some(error) => Err(error),
|
||||
None => Ok(outcome.value),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async fn attempt(
|
||||
&self,
|
||||
screen: &str,
|
||||
tool: &str,
|
||||
payload: &Value,
|
||||
extra: &[&str],
|
||||
) -> Result<Outcome, ControlError> {
|
||||
let socket = Self::socket_for_display(screen);
|
||||
if !socket.exists() {
|
||||
return Err(ControlError::DriverUnavailable);
|
||||
}
|
||||
let mut command = Command::new(&self.bin);
|
||||
command
|
||||
.env("DISPLAY", normalize_display(screen))
|
||||
.env(
|
||||
"CUA_DRIVER_RS_HOME",
|
||||
format!(
|
||||
"/tmp/lazyboy/cua-home-{}",
|
||||
normalize_display(screen).trim_start_matches(':')
|
||||
),
|
||||
)
|
||||
.args(["call", "--socket", &socket.to_string_lossy()]);
|
||||
command.args(extra);
|
||||
command.arg(tool);
|
||||
|
||||
apply_desktop_bus(&mut command, screen);
|
||||
let started = Instant::now();
|
||||
let output = bounded_input_output(&mut command, payload).await?;
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let (value, error) = decode_stdout(
|
||||
&stdout,
|
||||
&stderr,
|
||||
output.status.success(),
|
||||
classify_cua_failure,
|
||||
);
|
||||
tracing::info!(
|
||||
backend = "cua",
|
||||
tool,
|
||||
screen,
|
||||
duration_ms = started.elapsed().as_millis() as u64,
|
||||
success = error.is_none()
|
||||
);
|
||||
let session_ended = !output.status.success()
|
||||
&& [&*stdout, &*stderr].iter().any(|text| {
|
||||
text.trim_start().starts_with("session '") && text.contains("has ended; tool call")
|
||||
});
|
||||
Ok(Outcome {
|
||||
value,
|
||||
error,
|
||||
session_ended,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn public_agent_name(name: &str) -> String {
|
||||
name.split_whitespace()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
.chars()
|
||||
.filter(|ch| !ch.is_control())
|
||||
.take(80)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn read_after_session_restart(tool: &str) -> bool {
|
||||
matches!(
|
||||
tool,
|
||||
"get_desktop_state"
|
||||
| "list_windows"
|
||||
| "get_window_state"
|
||||
| "get_accessibility_tree"
|
||||
| "get_browser_state"
|
||||
| "health_report"
|
||||
| "get_cursor_position"
|
||||
)
|
||||
}
|
||||
|
||||
struct Outcome {
|
||||
session_ended: bool,
|
||||
value: Value,
|
||||
error: Option<ControlError>,
|
||||
}
|
||||
|
||||
/// Refusals and prose diagnostics can arrive with exit code 0, so only a JSON
|
||||
/// object counts as proof that the driver ran the tool.
|
||||
fn decode_stdout(
|
||||
stdout: &str,
|
||||
stderr: &str,
|
||||
success: bool,
|
||||
classify: impl Fn(&str) -> ControlError,
|
||||
) -> (Value, Option<ControlError>) {
|
||||
let combined = format!("{stdout}\n{stderr}");
|
||||
let trimmed = stdout.trim();
|
||||
let empty = Value::Null;
|
||||
let decoded = (!trimmed.is_empty())
|
||||
.then(|| parse_jsonish(trimmed))
|
||||
.flatten()
|
||||
.filter(Value::is_object);
|
||||
let error = if !success || trimmed.starts_with('\u{274c}') || decoded.is_none() {
|
||||
Some(classify(&combined))
|
||||
} else {
|
||||
response_error(decoded.as_ref().unwrap_or(&empty))
|
||||
};
|
||||
(decoded.unwrap_or(empty), error)
|
||||
}
|
||||
|
||||
/// The documented contract for `background_unavailable` is one retry with the
|
||||
/// delivery mode named in the structured escalation.
|
||||
fn recommended_delivery(value: &Value) -> Option<String> {
|
||||
["/escalation/recommended", "/error/escalation/recommended"]
|
||||
.into_iter()
|
||||
.filter_map(|pointer| value.pointer(pointer).and_then(Value::as_str))
|
||||
.find(|mode| matches!(*mode, "foreground" | "background"))
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
fn with_session_label(display: &str, payload: &Value) -> Value {
|
||||
let mut body = payload.clone();
|
||||
if let Some(map) = body.as_object_mut() {
|
||||
map.entry("session")
|
||||
.or_insert_with(|| json!(CuaClient::session_for_display(display)));
|
||||
}
|
||||
body
|
||||
}
|
||||
|
||||
fn response_error(value: &Value) -> Option<ControlError> {
|
||||
if value
|
||||
.get("code")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|code| code != "ok")
|
||||
|| value.get("ok").and_then(Value::as_bool) == Some(false)
|
||||
|| value.get("isError").and_then(Value::as_bool) == Some(true)
|
||||
|| ["status", "effect"].iter().any(|key| {
|
||||
matches!(
|
||||
value.get(*key).and_then(Value::as_str),
|
||||
Some("refused" | "error")
|
||||
)
|
||||
})
|
||||
{
|
||||
Some(classify_cua_failure(&value.to_string()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
async fn bounded_input_output(
|
||||
command: &mut Command,
|
||||
payload: &Value,
|
||||
) -> Result<std::process::Output, ControlError> {
|
||||
command
|
||||
.kill_on_drop(true)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
tokio::time::timeout(Duration::from_secs(120), async {
|
||||
let mut child = command
|
||||
.spawn()
|
||||
.map_err(|_| ControlError::DriverUnavailable)?;
|
||||
let mut input = child.stdin.take().ok_or(ControlError::DriverUnhealthy)?;
|
||||
let bytes = payload.to_string();
|
||||
// Drain stdout/stderr while writing so large inputs cannot deadlock.
|
||||
let write = async {
|
||||
input.write_all(bytes.as_bytes()).await?;
|
||||
input.shutdown().await?;
|
||||
drop(input);
|
||||
Ok::<(), std::io::Error>(())
|
||||
};
|
||||
let (written, output) = tokio::join!(write, child.wait_with_output());
|
||||
// A driver that exits before reading stdin (unknown tool, rejected
|
||||
// arguments) closes the pipe, so a broken pipe is expected and the exit
|
||||
// status plus stderr hold the real reason. Losing them here would
|
||||
// downgrade every fast refusal to a generic driver failure.
|
||||
if let Err(error) = written
|
||||
&& error.kind() != std::io::ErrorKind::BrokenPipe
|
||||
{
|
||||
return Err(ControlError::DriverUnhealthy);
|
||||
}
|
||||
output.map_err(|_| ControlError::DriverUnhealthy)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ControlError::Timeout)?
|
||||
}
|
||||
|
||||
async fn bounded_output(
|
||||
command: &mut Command,
|
||||
timeout: Duration,
|
||||
) -> Result<std::process::Output, ControlError> {
|
||||
command.kill_on_drop(true);
|
||||
tokio::time::timeout(timeout, command.output())
|
||||
.await
|
||||
.map_err(|_| ControlError::Timeout)?
|
||||
.map_err(|error| match error.kind() {
|
||||
std::io::ErrorKind::NotFound => ControlError::DriverUnavailable,
|
||||
std::io::ErrorKind::PermissionDenied => ControlError::PermissionDenied,
|
||||
_ => ControlError::DriverUnhealthy,
|
||||
})
|
||||
}
|
||||
|
||||
fn apply_desktop_bus(command: &mut Command, display: &str) {
|
||||
let dbus = CuaClient::dbus_file(display);
|
||||
if let Ok(address) = std::fs::read_to_string(&dbus) {
|
||||
let address = address.trim();
|
||||
if !address.is_empty() {
|
||||
command.env("DBUS_SESSION_BUS_ADDRESS", address);
|
||||
}
|
||||
}
|
||||
let number = normalize_display(display)
|
||||
.trim_start_matches(':')
|
||||
.to_string();
|
||||
let runtime = PathBuf::from(format!("/tmp/lazyboy/screen-{number}.runtime"));
|
||||
if let Ok(value) = std::fs::read_to_string(runtime) {
|
||||
let value = value.trim();
|
||||
if !value.is_empty() {
|
||||
command.env("XDG_RUNTIME_DIR", value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_jsonish(text: &str) -> Option<Value> {
|
||||
let text = text.trim();
|
||||
if text.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if let Ok(value) = serde_json::from_str::<Value>(text) {
|
||||
return Some(value);
|
||||
}
|
||||
let start = text.find('{')?;
|
||||
let end = text.rfind('}')?;
|
||||
if end > start {
|
||||
serde_json::from_str(&text[start..=end]).ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn walk<'a>(value: &'a Value, out: &mut Vec<&'a Value>) {
|
||||
out.push(value);
|
||||
match value {
|
||||
Value::Object(map) => {
|
||||
for item in map.values() {
|
||||
walk(item, out);
|
||||
}
|
||||
}
|
||||
Value::Array(items) => {
|
||||
for item in items {
|
||||
walk(item, out);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn first_array_of_objects<'a>(value: &'a Value, required: &str) -> Vec<&'a Value> {
|
||||
let mut nodes = Vec::new();
|
||||
walk(value, &mut nodes);
|
||||
for node in nodes {
|
||||
if let Some(items) = node.as_array()
|
||||
&& items.iter().any(|item| item.get(required).is_some())
|
||||
{
|
||||
return items.iter().collect();
|
||||
}
|
||||
if let Some(items) = node.get(required).and_then(Value::as_array)
|
||||
&& items.iter().any(Value::is_object)
|
||||
{
|
||||
return items.iter().collect();
|
||||
}
|
||||
}
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
fn classify_cua_failure(text: &str) -> ControlError {
|
||||
let lower = text.to_ascii_lowercase();
|
||||
if lower.contains("stale")
|
||||
|| lower.contains("not a live binding in this session")
|
||||
|| (lower.contains("session") && lower.contains("ended"))
|
||||
{
|
||||
ControlError::StaleReference
|
||||
} else if lower.contains("not_found") || lower.contains("not found") {
|
||||
ControlError::TargetNotFound
|
||||
} else if lower.contains("timeout") {
|
||||
ControlError::Timeout
|
||||
} else if lower.contains("permission") || lower.contains("consent") {
|
||||
ControlError::PermissionDenied
|
||||
} else if lower.contains("invalid_action_target") {
|
||||
ControlError::InvalidAction("Cua rejected the action target".into())
|
||||
} else if lower.contains("unsupported") || lower.contains("route_unavailable") {
|
||||
ControlError::Unsupported
|
||||
} else {
|
||||
// Driver diagnostics can echo typed text or credentials. Keep raw
|
||||
// output out of Agent-visible errors and downstream logs.
|
||||
ControlError::DriverUnhealthy
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn agent_name_preserves_unicode_without_control_characters() {
|
||||
assert_eq!(
|
||||
super::public_agent_name(" 小幫手\n Alice\u{0007} "),
|
||||
"小幫手 Alice"
|
||||
);
|
||||
assert_eq!(
|
||||
super::public_agent_name(&"小".repeat(100)).chars().count(),
|
||||
80
|
||||
);
|
||||
}
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn expired_sessions_only_retry_observations() {
|
||||
assert_eq!(
|
||||
classify_cua_failure(
|
||||
"confirmation provider failed: target bt-old is not a live binding in this session — re-run get_browser_state with pid + window_id"
|
||||
),
|
||||
ControlError::StaleReference
|
||||
);
|
||||
assert!(read_after_session_restart("get_desktop_state"));
|
||||
assert!(read_after_session_restart("get_browser_state"));
|
||||
for tool in ["click", "type_text", "browser_type", "hotkey", "launch_app"] {
|
||||
assert!(!read_after_session_restart(tool));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn structured_refusals_are_errors_but_page_text_is_not() {
|
||||
assert!(response_error(&serde_json::json!({"code": "invalid_action_target"})).is_some());
|
||||
assert!(matches!(
|
||||
response_error(
|
||||
&json!({"effect":"refused","escalation":{"reason":"route_unavailable"}})
|
||||
),
|
||||
Some(ControlError::Unsupported)
|
||||
));
|
||||
assert!(response_error(&serde_json::json!({"outline": "❌ payment declined"})).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_json_object_proves_the_tool_ran() {
|
||||
let classify = |text: &str| classify_cua_failure(text);
|
||||
let (value, error) = decode_stdout(r#"{"width":1280}"#, "", true, classify);
|
||||
assert!(error.is_none());
|
||||
assert_eq!(value["width"], 1280);
|
||||
|
||||
for malformed in ["null", "true", "42", "[]", r#""ok""#] {
|
||||
let (_, error) = decode_stdout(malformed, "", true, classify);
|
||||
assert!(error.is_some(), "non-object response accepted: {malformed}");
|
||||
}
|
||||
|
||||
// Prose with a zero exit code used to be reported as success.
|
||||
let (value, error) = decode_stdout("no window matched", "", true, classify);
|
||||
assert!(matches!(error, Some(ControlError::DriverUnhealthy)));
|
||||
assert!(value.is_null());
|
||||
|
||||
let (_, error) = decode_stdout("\u{274c} unsupported tool", "", true, classify);
|
||||
assert!(matches!(error, Some(ControlError::Unsupported)));
|
||||
|
||||
let (_, error) = decode_stdout(r#"{"code":"background_unavailable"}"#, "", true, classify);
|
||||
assert!(error.is_some());
|
||||
|
||||
// A crash with empty stdout must never look like an empty success.
|
||||
let (_, error) = decode_stdout("", "signal: 11", false, classify);
|
||||
assert!(error.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn escalation_is_read_from_the_documented_pointers() {
|
||||
assert_eq!(
|
||||
recommended_delivery(&json!({"escalation": {"recommended": "foreground"}})).as_deref(),
|
||||
Some("foreground")
|
||||
);
|
||||
assert_eq!(
|
||||
recommended_delivery(&json!({"error": {"escalation": {"recommended": "background"}}}))
|
||||
.as_deref(),
|
||||
Some("background")
|
||||
);
|
||||
assert!(recommended_delivery(&json!({"escalation": {"recommended": "reboot"}})).is_none());
|
||||
assert!(recommended_delivery(&json!({"ok": true})).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn piped_json_reaches_eof_without_argv_exposure() {
|
||||
let mut command = Command::new("sh");
|
||||
command.args(["-c", "cat"]);
|
||||
let payload = serde_json::json!({"text": "秘密🙂"});
|
||||
let output = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
bounded_input_output(&mut command, &payload),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<Value>(&output.stdout).unwrap(),
|
||||
payload
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hung_driver_is_bounded() {
|
||||
let mut command = Command::new("sh");
|
||||
command.args(["-c", "exec sleep 30"]);
|
||||
assert!(matches!(
|
||||
bounded_output(&mut command, Duration::from_millis(20)).await,
|
||||
Err(ControlError::Timeout)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn driver_that_never_reads_stdin_still_reports_its_exit() {
|
||||
// Bigger than the pipe buffer: the child never reads, so the write can
|
||||
// only fail with EPIPE and must not swallow the driver's own error.
|
||||
let payload = json!({ "blob": "x".repeat(1 << 20) });
|
||||
let mut command = Command::new("sh");
|
||||
command.args(["-c", "echo invalid_action_target >&2; exit 1"]);
|
||||
let output = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
bounded_input_output(&mut command, &payload),
|
||||
)
|
||||
.await
|
||||
.expect("bounded")
|
||||
.expect("exit status survives a broken stdin pipe");
|
||||
assert!(!output.status.success());
|
||||
assert!(
|
||||
String::from_utf8_lossy(&output.stderr).contains("invalid_action_target"),
|
||||
"driver stderr must reach the classifier"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_driver_failure_does_not_echo_secret() {
|
||||
let error = classify_cua_failure("failed typing secret-password");
|
||||
assert_eq!(error, ControlError::DriverUnhealthy);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn primary_display_uses_well_known_socket() {
|
||||
assert_eq!(
|
||||
CuaClient::socket_for_display(":1"),
|
||||
PathBuf::from(PRIMARY_SOCKET)
|
||||
);
|
||||
assert_eq!(
|
||||
CuaClient::socket_for_display(":2"),
|
||||
PathBuf::from("/tmp/lazyboy/cua-2.sock")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_json_object_from_noisy_stdout() {
|
||||
let parsed = parse_jsonish("✅ ok\n{\"status\":\"ok\",\"x\":1}\n").unwrap();
|
||||
assert_eq!(parsed["status"], "ok");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,144 @@
|
|||
//! GTK clipboard editor operated via Cua, for Linux driver builds without
|
||||
//! clipboard_read/write support. No direct X11/AT-SPI/clipboard subprocesses.
|
||||
use super::{CuaClient, CuaController, ListedWindow};
|
||||
use crate::ControlError;
|
||||
use serde_json::{Value, json};
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
async fn front(
|
||||
client: &CuaClient,
|
||||
display: &str,
|
||||
window: &ListedWindow,
|
||||
) -> Result<(), ControlError> {
|
||||
client
|
||||
.call(
|
||||
display,
|
||||
"bring_to_front",
|
||||
&json!({"pid":window.pid,"window_id":window.id}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
async fn key(
|
||||
client: &CuaClient,
|
||||
display: &str,
|
||||
window: &ListedWindow,
|
||||
keys: &[&str],
|
||||
) -> Result<(), ControlError> {
|
||||
client.call(display, "hotkey", &json!({"pid":window.pid,"window_id":window.id,"keys":keys,"delivery_mode":"foreground"}), &[]).await?;
|
||||
Ok(())
|
||||
}
|
||||
fn shortcut<'a>(window: &ListedWindow, letter: &'a str) -> Vec<&'a str> {
|
||||
if window.app_name.to_lowercase().contains("terminal") {
|
||||
vec!["ctrl", "shift", letter]
|
||||
} else {
|
||||
vec!["ctrl", letter]
|
||||
}
|
||||
}
|
||||
async fn active(client: &CuaClient, display: &str) -> Result<ListedWindow, ControlError> {
|
||||
CuaController::list_windows_now(client, display)
|
||||
.await?
|
||||
.into_iter()
|
||||
.max_by_key(|w| w.z)
|
||||
.ok_or(ControlError::TargetNotFound)
|
||||
}
|
||||
async fn editor(client: &CuaClient, display: &str) -> Result<(ListedWindow, Value), ControlError> {
|
||||
super::launch::run(
|
||||
client,
|
||||
display,
|
||||
&[
|
||||
"env".into(),
|
||||
format!("DISPLAY={display}"),
|
||||
"lazyboy-clipboard".into(),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
let window = CuaController::list_windows_now(client, display)
|
||||
.await?
|
||||
.into_iter()
|
||||
.find(|w| w.title == "Clipboard · LazyBoy")
|
||||
.ok_or(ControlError::TargetNotFound)?;
|
||||
let state = client
|
||||
.call(
|
||||
display,
|
||||
"get_window_state",
|
||||
&json!({"pid":window.pid,"window_id":window.id,"include_screenshot":false}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
Ok((window, state))
|
||||
}
|
||||
fn element<'a>(state: &'a Value, label: &str) -> Result<&'a Value, ControlError> {
|
||||
state
|
||||
.get("elements")
|
||||
.and_then(Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.find(|item| {
|
||||
item.get("label")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|text| {
|
||||
if label == "Clipboard text" {
|
||||
text.starts_with("Clipboard text: ")
|
||||
} else {
|
||||
text == label
|
||||
}
|
||||
})
|
||||
})
|
||||
.ok_or(ControlError::TargetNotFound)
|
||||
}
|
||||
async fn restore(
|
||||
client: &CuaClient,
|
||||
display: &str,
|
||||
editor: &ListedWindow,
|
||||
original: &ListedWindow,
|
||||
) -> Result<(), ControlError> {
|
||||
key(client, display, editor, &["alt", "f4"]).await?;
|
||||
front(client, display, original).await
|
||||
}
|
||||
|
||||
pub(super) async fn paste(
|
||||
client: &CuaClient,
|
||||
display: &str,
|
||||
text: &str,
|
||||
) -> Result<(), ControlError> {
|
||||
let original = active(client, display).await?;
|
||||
let (window, state) = editor(client, display).await?;
|
||||
let result = async {
|
||||
let entry = element(&state, "Clipboard text")?;
|
||||
client.call(display, "set_value", &json!({"pid":window.pid,"window_id":window.id,"element_token":entry["element_token"],"value":text}), &[]).await?;
|
||||
// Fresh snapshot verifies exact content and supplies fresh action refs.
|
||||
let state = client.call(display, "get_window_state", &json!({"pid":window.pid,"window_id":window.id,"include_screenshot":false}), &[]).await?;
|
||||
if element(&state, "Clipboard text")?.get("label").and_then(Value::as_str) != Some(format!("Clipboard text: {text}").as_str()) {
|
||||
return Err(ControlError::internal("clipboard text did not roundtrip; nothing pasted"));
|
||||
}
|
||||
let copy = element(&state, "Copy")?;
|
||||
client.call(display, "click", &json!({"pid":window.pid,"window_id":window.id,"element_token":copy["element_token"],"delivery_mode":"foreground"}), &[]).await?;
|
||||
front(client, display, &original).await?;
|
||||
key(client, display, &original, &shortcut(&original,"v")).await?;
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
Ok(())
|
||||
}.await;
|
||||
// Keep the editor alive until the destination consumes its clipboard.
|
||||
let restored = restore(client, display, &window, &original).await;
|
||||
result.and(restored)
|
||||
}
|
||||
|
||||
pub(super) async fn copy(client: &CuaClient, display: &str) -> Result<String, ControlError> {
|
||||
let original = active(client, display).await?;
|
||||
front(client, display, &original).await?;
|
||||
key(client, display, &original, &shortcut(&original, "c")).await?;
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
let (window, state) = editor(client, display).await?;
|
||||
let result = element(&state, "Clipboard text").map(|item| {
|
||||
item.get("label")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("")
|
||||
.strip_prefix("Clipboard text: ")
|
||||
.unwrap_or("")
|
||||
.to_string()
|
||||
});
|
||||
restore(client, display, &window, &original).await?;
|
||||
result
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
//! Launch and foreground applications using Cua on the shared desktop.
|
||||
use serde_json::json;
|
||||
use tokio::time::{Duration, Instant, sleep};
|
||||
|
||||
use super::{CuaClient, CuaController};
|
||||
use crate::ControlError;
|
||||
|
||||
pub(super) async fn run(
|
||||
client: &CuaClient,
|
||||
display: &str,
|
||||
argv: &[String],
|
||||
) -> Result<(), ControlError> {
|
||||
let (program, arguments) = argv
|
||||
.split_first()
|
||||
.ok_or_else(|| ControlError::InvalidAction("empty application command".into()))?;
|
||||
let before = CuaController::list_windows_now(client, display).await?;
|
||||
let result = client
|
||||
.call(
|
||||
display,
|
||||
"launch_app",
|
||||
&json!({
|
||||
"name": program, "additional_arguments": arguments,
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
let pid = result.get("pid").and_then(serde_json::Value::as_u64);
|
||||
let browser = arguments.iter().any(|arg| arg == "lazyboy-browser");
|
||||
let deadline = Instant::now() + Duration::from_secs(8);
|
||||
loop {
|
||||
let windows = CuaController::list_windows_now(client, display).await?;
|
||||
let window = windows
|
||||
.iter()
|
||||
.find(|window| Some(window.pid) == pid)
|
||||
.or_else(|| {
|
||||
windows.iter().find(|window| {
|
||||
!before
|
||||
.iter()
|
||||
.any(|old| old.id == window.id && old.title == window.title)
|
||||
})
|
||||
})
|
||||
.or_else(|| {
|
||||
if browser {
|
||||
super::browser::chromium_window(&windows)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
if let Some(window) = window {
|
||||
client
|
||||
.call(
|
||||
display,
|
||||
"bring_to_front",
|
||||
&json!({"pid":window.pid,"window_id":window.id}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
return Ok(());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(ControlError::TargetNotFound);
|
||||
}
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,922 @@
|
|||
mod browser;
|
||||
mod client;
|
||||
mod clipboard;
|
||||
mod launch;
|
||||
mod native;
|
||||
mod record;
|
||||
mod translate;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use lazyboy_contracts::{
|
||||
ActiveWindow, ComputerAction, ComputerObservation, CursorPosition, PointerType, UiElement,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
use crate::controller::{
|
||||
ComputerController, ComputerDriver, ControlContext, ControlError, ControllerHealth,
|
||||
};
|
||||
use crate::{
|
||||
ActionRequest, ActionResult, BrowserPage, BrowserRequest, RecordingRequest, RecordingResult,
|
||||
RecordingSession, action_pause_ms, image_dimensions, normalize_display, observation_from_png,
|
||||
observation_with_elements, teach_trajectory_dir,
|
||||
};
|
||||
use client::first_array_of_objects;
|
||||
|
||||
pub use client::CuaClient;
|
||||
|
||||
/// Last resort when neither the screenshot nor the driver reports a mode.
|
||||
const FALLBACK_SCREEN: (u32, u32) = (1280, 800);
|
||||
|
||||
/// `image/computer/Dockerfile` pins `CUA_DRIVER_RS_VERSION`; only that tool
|
||||
/// surface is guaranteed. Patch releases stay compatible, a new minor does not.
|
||||
const PINNED_DRIVER: (u32, u32) = (0, 23);
|
||||
|
||||
/// `cua-driver --version` prints `cua-driver 0.23.2`, and some builds append a
|
||||
/// target suffix (`cua-driver 0.23.2 (x86_64-linux)`), so only the leading
|
||||
/// numeric version is trusted.
|
||||
fn driver_release(version: &str) -> Option<(u32, u32)> {
|
||||
let digits = version.find(|character: char| character.is_ascii_digit())?;
|
||||
let mut parts = version[digits..].split('.');
|
||||
Some((parts.next()?.parse().ok()?, parts.next()?.parse().ok()?))
|
||||
}
|
||||
|
||||
pub use translate::{TranslatedAction, translate_action};
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct CuaController {
|
||||
client: CuaClient,
|
||||
screens: tokio::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
|
||||
native: tokio::sync::Mutex<HashMap<String, HashMap<String, native::NativeTarget>>>,
|
||||
browser: tokio::sync::Mutex<HashMap<String, browser::BrowserBind>>,
|
||||
snapshots: AtomicU64,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ComputerController for CuaController {
|
||||
fn backend(&self) -> ComputerDriver {
|
||||
ComputerDriver::Cua
|
||||
}
|
||||
|
||||
async fn health(&self, ctx: &ControlContext) -> Result<ControllerHealth, ControlError> {
|
||||
let version = self.client.version().await.ok();
|
||||
let report = self
|
||||
.client
|
||||
.call(&ctx.display, "health_report", &json!({}), &[])
|
||||
.await;
|
||||
match report {
|
||||
Ok(report) => {
|
||||
let compatible = version
|
||||
.as_deref()
|
||||
.is_some_and(|text| driver_release(text) == Some(PINNED_DRIVER));
|
||||
let healthy =
|
||||
compatible && report.get("overall").and_then(Value::as_str) == Some("ok");
|
||||
let mut details: Vec<String> = report
|
||||
.get("checks")
|
||||
.and_then(Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter(|check| check["status"] == "fail")
|
||||
.filter_map(|check| check["message"].as_str().map(str::to_string))
|
||||
.collect();
|
||||
if !compatible {
|
||||
details.push(format!(
|
||||
"Cua Driver {} is not the pinned {}.{} series",
|
||||
version.as_deref().unwrap_or("unknown"),
|
||||
PINNED_DRIVER.0,
|
||||
PINNED_DRIVER.1
|
||||
));
|
||||
}
|
||||
Ok(ControllerHealth {
|
||||
backend: "cua".into(),
|
||||
version,
|
||||
healthy,
|
||||
degraded: !healthy,
|
||||
details,
|
||||
})
|
||||
}
|
||||
Err(error) => Ok(ControllerHealth {
|
||||
backend: "cua".into(),
|
||||
version,
|
||||
healthy: false,
|
||||
degraded: true,
|
||||
details: vec![error.to_string()],
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn observe(&self, ctx: &ControlContext) -> Result<ComputerObservation, ControlError> {
|
||||
let _screen = self.lock_screen(&ctx.display).await;
|
||||
// A desktop screenshot does not change the browser binding. The next
|
||||
// browser snapshot refreshes page refs; keep the session attachment warm.
|
||||
self.observe_display(&ctx.display).await
|
||||
}
|
||||
|
||||
async fn act(
|
||||
&self,
|
||||
request: &ActionRequest,
|
||||
ctx: &ControlContext,
|
||||
) -> Result<ActionResult, ControlError> {
|
||||
let _screen = self.lock_screen(&ctx.display).await;
|
||||
if matches!(request.actions.as_slice(), [ComputerAction::CopySelection]) {
|
||||
let text = clipboard::copy(&self.client, &ctx.display).await?;
|
||||
return Ok(ActionResult {
|
||||
completed: 1,
|
||||
clipboard_text: Some(text),
|
||||
observation: None,
|
||||
});
|
||||
}
|
||||
let display = ctx.display.as_str();
|
||||
let profile = ctx.profile_path.as_deref();
|
||||
let key = normalize_display(display).to_string();
|
||||
self.browser.lock().await.remove(&key);
|
||||
// One snapshot for the whole batch. Taking the map per action made a
|
||||
// leading wait (or a second ref) look stale.
|
||||
let mut targets = self.native.lock().await.remove(&key).unwrap_or_default();
|
||||
match self
|
||||
.run_actions(request, display, profile, &mut targets)
|
||||
.await
|
||||
{
|
||||
Ok(completed) => {
|
||||
let observation = if request.observe {
|
||||
Some(self.observe_display(display).await?)
|
||||
} else {
|
||||
self.native.lock().await.insert(key, targets);
|
||||
None
|
||||
};
|
||||
Ok(ActionResult {
|
||||
clipboard_text: None,
|
||||
completed,
|
||||
observation,
|
||||
})
|
||||
}
|
||||
Err(error) => {
|
||||
self.native.lock().await.insert(key, targets);
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn browser(
|
||||
&self,
|
||||
request: &BrowserRequest,
|
||||
ctx: &ControlContext,
|
||||
) -> Result<BrowserPage, ControlError> {
|
||||
let _screen = self.lock_screen(&ctx.display).await;
|
||||
if !matches!(
|
||||
request.action.as_str(),
|
||||
"snapshot" | "wait" | "probe" | "ensure"
|
||||
) {
|
||||
self.native
|
||||
.lock()
|
||||
.await
|
||||
.remove(normalize_display(&ctx.display));
|
||||
}
|
||||
let key = normalize_display(&ctx.display).to_string();
|
||||
let mut windows = self.windows(&ctx.display).await.unwrap_or_default();
|
||||
// The screen lock serializes this display. Never hold the shared cache
|
||||
// lock across driver calls or waits on behalf of other displays.
|
||||
let mut bind = self.browser.lock().await.remove(&key);
|
||||
for attempt in 0..2 {
|
||||
let had_bind = bind.is_some();
|
||||
match browser::run(
|
||||
&self.client,
|
||||
&ctx.display,
|
||||
ctx.profile_path.as_deref(),
|
||||
request,
|
||||
&windows,
|
||||
&mut bind,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(ControlError::BrowserUnavailable) if !had_bind => {
|
||||
return Ok(BrowserPage {
|
||||
ok: false,
|
||||
error: Some("Cua browser unavailable".into()),
|
||||
..BrowserPage::default()
|
||||
});
|
||||
}
|
||||
Err(error)
|
||||
if attempt == 0
|
||||
&& had_bind
|
||||
&& matches!(request.action.as_str(), "snapshot" | "wait" | "ensure")
|
||||
&& matches!(
|
||||
error,
|
||||
ControlError::BrowserUnavailable
|
||||
| ControlError::StaleReference
|
||||
| ControlError::TargetNotFound
|
||||
) =>
|
||||
{
|
||||
bind = None;
|
||||
windows = self.windows(&ctx.display).await.unwrap_or_default();
|
||||
}
|
||||
other => {
|
||||
if let Some(mut current) = bind {
|
||||
update_browser_page(&mut current, &request.action, &other);
|
||||
self.browser.lock().await.insert(key, current);
|
||||
}
|
||||
return map_browser_unavailable(other);
|
||||
}
|
||||
}
|
||||
}
|
||||
map_browser_unavailable(Err(ControlError::BrowserUnavailable))
|
||||
}
|
||||
|
||||
async fn start_recording(
|
||||
&self,
|
||||
request: &RecordingRequest,
|
||||
ctx: &ControlContext,
|
||||
) -> Result<RecordingSession, ControlError> {
|
||||
if request.skill_id.trim().is_empty() {
|
||||
return Err(ControlError::InvalidAction(
|
||||
"recording needs a skill id".into(),
|
||||
));
|
||||
}
|
||||
let output_dir = teach_trajectory_dir(&request.skill_id);
|
||||
let _ = tokio::fs::create_dir_all(&output_dir).await;
|
||||
let _ = self
|
||||
.client
|
||||
.call(&ctx.display, "stop_recording", &json!({}), &[])
|
||||
.await;
|
||||
self.client
|
||||
.call(
|
||||
&ctx.display,
|
||||
"start_recording",
|
||||
&json!({ "output_dir": output_dir, "record_video": false }),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
Ok(RecordingSession {
|
||||
skill_id: request.skill_id.clone(),
|
||||
output_dir,
|
||||
})
|
||||
}
|
||||
|
||||
async fn stop_recording(
|
||||
&self,
|
||||
_request: &RecordingRequest,
|
||||
ctx: &ControlContext,
|
||||
) -> Result<(), ControlError> {
|
||||
self.client
|
||||
.call(&ctx.display, "stop_recording", &json!({}), &[])
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn collect_recording(
|
||||
&self,
|
||||
request: &RecordingRequest,
|
||||
_ctx: &ControlContext,
|
||||
) -> Result<RecordingResult, ControlError> {
|
||||
let dir = teach_trajectory_dir(&request.skill_id);
|
||||
let mut events = record::events_from_dir(std::path::Path::new(&dir));
|
||||
events.sort_by_key(|event| event.get("at").and_then(Value::as_i64).unwrap_or(0));
|
||||
let _ = tokio::fs::remove_dir_all(&dir).await;
|
||||
Ok(RecordingResult { events })
|
||||
}
|
||||
}
|
||||
|
||||
fn update_browser_page(
|
||||
bind: &mut browser::BrowserBind,
|
||||
action: &str,
|
||||
result: &Result<BrowserPage, ControlError>,
|
||||
) {
|
||||
// These checks neither observe nor mutate the page. Keep the refs returned
|
||||
// by the previous snapshot usable for the next click/type.
|
||||
if matches!(action, "probe" | "ensure") && result.is_ok() {
|
||||
return;
|
||||
}
|
||||
bind.page = match result {
|
||||
Ok(page) if page.ok => Some(page.clone()),
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
|
||||
impl CuaController {
|
||||
async fn run_actions(
|
||||
&self,
|
||||
request: &ActionRequest,
|
||||
display: &str,
|
||||
profile: Option<&str>,
|
||||
targets: &mut HashMap<String, native::NativeTarget>,
|
||||
) -> Result<usize, ControlError> {
|
||||
let mut completed = 0usize;
|
||||
while completed < request.actions.len() {
|
||||
let action = &request.actions[completed];
|
||||
if let Some(payload) = drag_payload(&request.actions[completed..]) {
|
||||
self.dispatch(
|
||||
display,
|
||||
TranslatedAction::Cua {
|
||||
tool: "drag",
|
||||
payload,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
completed += 5;
|
||||
continue;
|
||||
}
|
||||
if let ComputerAction::Ref {
|
||||
verb,
|
||||
target,
|
||||
ref_kind,
|
||||
text,
|
||||
} = action
|
||||
{
|
||||
if ref_kind != "a11y" {
|
||||
return Err(ControlError::Unsupported);
|
||||
}
|
||||
let target = targets
|
||||
.get(target)
|
||||
.cloned()
|
||||
.ok_or(ControlError::StaleReference)?;
|
||||
native::act(&self.client, display, target, *verb, text.as_deref()).await?;
|
||||
} else {
|
||||
let translated = translate_action(action, display, profile)?;
|
||||
self.dispatch(display, translated).await?;
|
||||
}
|
||||
let pause = action_pause_ms(action);
|
||||
if pause > 0 {
|
||||
sleep(Duration::from_millis(pause)).await;
|
||||
}
|
||||
completed += 1;
|
||||
}
|
||||
if request.settle_ms > 0 {
|
||||
sleep(Duration::from_millis(u64::from(request.settle_ms))).await;
|
||||
}
|
||||
Ok(completed)
|
||||
}
|
||||
|
||||
async fn lock_screen(&self, display: &str) -> tokio::sync::OwnedMutexGuard<()> {
|
||||
let lock = self
|
||||
.screens
|
||||
.lock()
|
||||
.await
|
||||
.entry(normalize_display(display).into())
|
||||
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
|
||||
.clone();
|
||||
lock.lock_owned().await
|
||||
}
|
||||
|
||||
async fn observe_display(&self, display: &str) -> Result<ComputerObservation, ControlError> {
|
||||
self.native.lock().await.remove(normalize_display(display));
|
||||
let png_path = observe_png_path(display);
|
||||
let _ = tokio::fs::remove_file(&png_path).await;
|
||||
self.client
|
||||
.call(
|
||||
display,
|
||||
"get_desktop_state",
|
||||
&json!({ "screenshot_out_file": png_path.to_string_lossy() }),
|
||||
&["--screenshot-out-file", &png_path.to_string_lossy()],
|
||||
)
|
||||
.await?;
|
||||
let png = tokio::fs::read(&png_path)
|
||||
.await
|
||||
.map_err(|_| ControlError::Internal("screenshot failed".into()))?;
|
||||
let _ = tokio::fs::remove_file(&png_path).await;
|
||||
if png.is_empty() {
|
||||
return Err(ControlError::Internal("screenshot failed".into()));
|
||||
}
|
||||
let (width, height) = match image_dimensions(&png) {
|
||||
Some(dimensions) => dimensions,
|
||||
None => self.screen_size(display).await,
|
||||
};
|
||||
let cursor = self.cursor(display).await;
|
||||
let windows = self.windows(display).await.unwrap_or_default();
|
||||
let active = windows
|
||||
.iter()
|
||||
.max_by_key(|window| window.z)
|
||||
.map(|window| ActiveWindow {
|
||||
id: window.id.to_string(),
|
||||
title: Some(window.title.clone()).filter(|title| !title.is_empty()),
|
||||
});
|
||||
// Selectors only need to be short and unique: a process-local counter
|
||||
// keeps stale handles from resolving without pasting a temp path into
|
||||
// every identifier the Agent echoes back.
|
||||
let snapshot = self.snapshots.fetch_add(1, Ordering::Relaxed);
|
||||
let native::NativeObservation {
|
||||
mut elements,
|
||||
targets,
|
||||
complete,
|
||||
} = native::observe(&self.client, display, &windows, &format!("s{snapshot}")).await;
|
||||
elements.extend(
|
||||
windows
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, window)| UiElement {
|
||||
id: (index + 1) as u32,
|
||||
title: window.title.chars().take(256).collect(),
|
||||
x: window.x.max(0) as u32,
|
||||
y: window.y.max(0) as u32,
|
||||
w: window.w,
|
||||
h: window.h,
|
||||
selector: None,
|
||||
kind: Some("window".into()),
|
||||
role: None,
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
for (index, element) in elements.iter_mut().enumerate() {
|
||||
element.id = (index + 1) as u32;
|
||||
}
|
||||
self.native
|
||||
.lock()
|
||||
.await
|
||||
.insert(normalize_display(display).into(), targets);
|
||||
let mut observation = observation_with_elements(
|
||||
observation_from_png(png, width, height, cursor, active),
|
||||
elements,
|
||||
);
|
||||
observation.native_observation_complete = complete;
|
||||
Ok(observation)
|
||||
}
|
||||
|
||||
/// `scroll` on the desktop plane is aimed at a point and the action DSL
|
||||
/// does not carry one, so aim at the pointer; the screen centre is the
|
||||
/// next best guess when the pointer cannot be read.
|
||||
async fn scroll_point(&self, display: &str) -> (u32, u32) {
|
||||
if let Some(cursor) = self.cursor(display).await
|
||||
&& cursor.x >= 0
|
||||
&& cursor.y >= 0
|
||||
{
|
||||
return (cursor.x as u32, cursor.y as u32);
|
||||
}
|
||||
let (width, height) = self.screen_size(display).await;
|
||||
(width / 2, height / 2)
|
||||
}
|
||||
|
||||
async fn screen_size(&self, display: &str) -> (u32, u32) {
|
||||
self.client
|
||||
.call(display, "get_screen_size", &json!({}), &[])
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|value| {
|
||||
Some((
|
||||
value.get("width").and_then(Value::as_u64)? as u32,
|
||||
value.get("height").and_then(Value::as_u64)? as u32,
|
||||
))
|
||||
})
|
||||
.unwrap_or(FALLBACK_SCREEN)
|
||||
}
|
||||
|
||||
async fn cursor(&self, display: &str) -> Option<CursorPosition> {
|
||||
let value = self
|
||||
.client
|
||||
.call(display, "get_cursor_position", &json!({}), &[])
|
||||
.await
|
||||
.ok()?;
|
||||
cursor_from_value(&value)
|
||||
}
|
||||
|
||||
async fn windows(&self, display: &str) -> Result<Vec<ListedWindow>, ControlError> {
|
||||
Self::list_windows_now(&self.client, display).await
|
||||
}
|
||||
|
||||
pub(crate) async fn list_windows_now(
|
||||
client: &CuaClient,
|
||||
display: &str,
|
||||
) -> Result<Vec<ListedWindow>, ControlError> {
|
||||
let value = client
|
||||
.call(
|
||||
display,
|
||||
"list_windows",
|
||||
&json!({ "on_screen_only": true }),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
Ok(parse_listed_windows(&value))
|
||||
}
|
||||
|
||||
async fn dispatch(
|
||||
&self,
|
||||
display: &str,
|
||||
translated: TranslatedAction,
|
||||
) -> Result<(), ControlError> {
|
||||
match translated {
|
||||
TranslatedAction::Sleep { ms } => {
|
||||
sleep(Duration::from_millis(ms)).await;
|
||||
Ok(())
|
||||
}
|
||||
TranslatedAction::Launch { argv } => launch::run(&self.client, display, &argv).await,
|
||||
TranslatedAction::FocusTitle { title } => self.focus_title(display, &title).await,
|
||||
TranslatedAction::Cua { tool, mut payload } => {
|
||||
if tool == "type_text"
|
||||
&& let Some(text) = payload.get("text").and_then(Value::as_str)
|
||||
&& (!text.is_ascii() || text.contains(['\n', '\r']))
|
||||
{
|
||||
return clipboard::paste(&self.client, display, text).await;
|
||||
}
|
||||
|
||||
if tool == "scroll" && payload.get("x").is_none() {
|
||||
let (x, y) = self.scroll_point(display).await;
|
||||
payload["x"] = json!(x);
|
||||
payload["y"] = json!(y);
|
||||
}
|
||||
if tool == "drag" {
|
||||
let x = payload["from_x"].as_f64().unwrap_or(0.0);
|
||||
let y = payload["from_y"].as_f64().unwrap_or(0.0);
|
||||
let windows = self.windows(display).await?;
|
||||
let window = window_containing(&windows, x, y)
|
||||
.cloned()
|
||||
.ok_or(ControlError::TargetNotFound)?;
|
||||
for (key, offset) in [
|
||||
("from_x", window.x),
|
||||
("to_x", window.x),
|
||||
("from_y", window.y),
|
||||
("to_y", window.y),
|
||||
] {
|
||||
payload[key] = json!(payload[key].as_f64().unwrap_or(0.0) - offset as f64);
|
||||
}
|
||||
payload["pid"] = json!(window.pid);
|
||||
payload["window_id"] = json!(window.id);
|
||||
payload["delivery_mode"] = json!("foreground");
|
||||
}
|
||||
self.client.call(display, tool, &payload, &[]).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn focus_title(&self, display: &str, title: &str) -> Result<(), ControlError> {
|
||||
let windows = self.windows(display).await?;
|
||||
let window = window_matching_title(&windows, title).ok_or(ControlError::TargetNotFound)?;
|
||||
self.client
|
||||
.call(
|
||||
display,
|
||||
"bring_to_front",
|
||||
&json!({ "pid": window.pid, "window_id": window.id }),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// The public drag DSL expands into these five actions. Keep the gesture in
|
||||
// one Cua call; separate CLI leases do not preserve held-button state.
|
||||
fn drag_payload(actions: &[ComputerAction]) -> Option<Value> {
|
||||
let [
|
||||
ComputerAction::Pointer {
|
||||
x,
|
||||
y,
|
||||
pointer_type: PointerType::Down,
|
||||
button,
|
||||
},
|
||||
ComputerAction::Wait { ms: 40 },
|
||||
ComputerAction::Pointer {
|
||||
x: to_x,
|
||||
y: to_y,
|
||||
pointer_type: PointerType::Move,
|
||||
button: move_button,
|
||||
},
|
||||
ComputerAction::Wait { ms: 40 },
|
||||
ComputerAction::Pointer {
|
||||
x: up_x,
|
||||
y: up_y,
|
||||
pointer_type: PointerType::Up,
|
||||
button: up_button,
|
||||
},
|
||||
..,
|
||||
] = actions
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
if (to_x, to_y, button) != (up_x, up_y, up_button) || button != move_button {
|
||||
return None;
|
||||
}
|
||||
Some(json!({"from_x": x, "from_y": y, "to_x": to_x, "to_y": to_y,
|
||||
"button": button.unwrap_or(lazyboy_contracts::PointerButton::Left), "duration_ms": 500}))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ListedWindow {
|
||||
pub(crate) id: u64,
|
||||
pub(crate) pid: u64,
|
||||
pub(crate) title: String,
|
||||
pub(crate) app_name: String,
|
||||
pub(crate) x: i64,
|
||||
pub(crate) y: i64,
|
||||
pub(crate) w: u32,
|
||||
pub(crate) h: u32,
|
||||
pub(crate) z: i64,
|
||||
}
|
||||
|
||||
fn map_browser_unavailable(
|
||||
result: Result<BrowserPage, ControlError>,
|
||||
) -> Result<BrowserPage, ControlError> {
|
||||
match result {
|
||||
Err(ControlError::BrowserUnavailable) => Ok(BrowserPage {
|
||||
ok: false,
|
||||
error: Some("Cua browser unavailable".into()),
|
||||
..BrowserPage::default()
|
||||
}),
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn cursor_from_value(value: &Value) -> Option<CursorPosition> {
|
||||
let mut nodes = Vec::new();
|
||||
client::walk(value, &mut nodes);
|
||||
for node in nodes {
|
||||
if let (Some(x), Some(y)) = (
|
||||
node.get("x").and_then(Value::as_i64),
|
||||
node.get("y").and_then(Value::as_i64),
|
||||
) {
|
||||
return Some(CursorPosition {
|
||||
x: x as i32,
|
||||
y: y as i32,
|
||||
});
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn parse_listed_windows(value: &Value) -> Vec<ListedWindow> {
|
||||
first_array_of_objects(value, "window_id")
|
||||
.into_iter()
|
||||
.filter_map(listed_window)
|
||||
.filter(|window| !ignored_window(window))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn listed_window(value: &Value) -> Option<ListedWindow> {
|
||||
let bounds = value.get("bounds");
|
||||
let x = value
|
||||
.get("x")
|
||||
.or_else(|| bounds.and_then(|b| b.get("x")))?
|
||||
.as_f64()? as i64;
|
||||
let y = value
|
||||
.get("y")
|
||||
.or_else(|| bounds.and_then(|b| b.get("y")))?
|
||||
.as_f64()? as i64;
|
||||
let w = number(value, "width")
|
||||
.or_else(|| number(value, "w"))
|
||||
.or_else(|| bounds.and_then(|bounds| number(bounds, "width")))?;
|
||||
let h = number(value, "height")
|
||||
.or_else(|| number(value, "h"))
|
||||
.or_else(|| bounds.and_then(|bounds| number(bounds, "height")))?;
|
||||
if w < 32 || h < 16 {
|
||||
return None;
|
||||
}
|
||||
Some(ListedWindow {
|
||||
id: number(value, "window_id")?,
|
||||
pid: number(value, "pid").unwrap_or(0),
|
||||
title: value
|
||||
.get("title")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string(),
|
||||
app_name: value
|
||||
.get("app_name")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string(),
|
||||
x,
|
||||
y,
|
||||
w: w as u32,
|
||||
h: h as u32,
|
||||
z: number(value, "z_index").unwrap_or(0) as i64,
|
||||
})
|
||||
}
|
||||
|
||||
fn window_containing(windows: &[ListedWindow], x: f64, y: f64) -> Option<&ListedWindow> {
|
||||
windows
|
||||
.iter()
|
||||
.filter(|window| {
|
||||
x >= window.x as f64
|
||||
&& y >= window.y as f64
|
||||
&& x < window.x as f64 + f64::from(window.w)
|
||||
&& y < window.y as f64 + f64::from(window.h)
|
||||
})
|
||||
.min_by_key(|window| u64::from(window.w.saturating_mul(window.h.max(1))))
|
||||
}
|
||||
|
||||
fn window_matching_title<'a>(windows: &'a [ListedWindow], title: &str) -> Option<&'a ListedWindow> {
|
||||
let needle = title.to_ascii_lowercase();
|
||||
windows
|
||||
.iter()
|
||||
.find(|window| window.title.eq_ignore_ascii_case(title))
|
||||
.or_else(|| {
|
||||
windows.iter().find(|window| {
|
||||
window.title.to_ascii_lowercase().contains(&needle)
|
||||
&& !window.app_name.to_ascii_lowercase().contains("chrom")
|
||||
})
|
||||
})
|
||||
.or_else(|| {
|
||||
windows
|
||||
.iter()
|
||||
.find(|window| window.title.to_ascii_lowercase().contains(&needle))
|
||||
})
|
||||
}
|
||||
|
||||
fn ignored_window(window: &ListedWindow) -> bool {
|
||||
let title = window.title.to_ascii_lowercase();
|
||||
title.is_empty()
|
||||
|| title == "desktop"
|
||||
|| title == "xfce4-panel"
|
||||
|| window.app_name.to_ascii_lowercase().contains("xfdesktop")
|
||||
}
|
||||
|
||||
fn number(value: &Value, key: &str) -> Option<u64> {
|
||||
value.get(key).and_then(Value::as_u64).or_else(|| {
|
||||
value
|
||||
.get(key)
|
||||
.and_then(Value::as_f64)
|
||||
.filter(|n| *n >= 0.0)
|
||||
.map(|n| n as u64)
|
||||
})
|
||||
}
|
||||
|
||||
fn observe_png_path(display: &str) -> PathBuf {
|
||||
let nanos = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|duration| duration.as_nanos())
|
||||
.unwrap_or(0);
|
||||
PathBuf::from(format!(
|
||||
"/tmp/lazyboy/cua-obs-{}-{nanos}.png",
|
||||
display.trim_start_matches(':')
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn browser_checks_preserve_refs_but_failed_actions_invalidate_them() {
|
||||
let mut bind = browser::BrowserBind {
|
||||
pid: 1,
|
||||
window_id: 2,
|
||||
target_id: "target".into(),
|
||||
tab_id: "tab".into(),
|
||||
page: Some(BrowserPage {
|
||||
ok: true,
|
||||
title: "original snapshot".into(),
|
||||
..BrowserPage::default()
|
||||
}),
|
||||
};
|
||||
for action in ["probe", "ensure"] {
|
||||
update_browser_page(
|
||||
&mut bind,
|
||||
action,
|
||||
&Ok(BrowserPage {
|
||||
ok: true,
|
||||
..BrowserPage::default()
|
||||
}),
|
||||
);
|
||||
assert_eq!(bind.page.as_ref().unwrap().title, "original snapshot");
|
||||
}
|
||||
update_browser_page(&mut bind, "click", &Err(ControlError::StaleReference));
|
||||
assert!(bind.page.is_none());
|
||||
update_browser_page(
|
||||
&mut bind,
|
||||
"snapshot",
|
||||
&Ok(BrowserPage {
|
||||
ok: true,
|
||||
title: "fresh".into(),
|
||||
..BrowserPage::default()
|
||||
}),
|
||||
);
|
||||
assert_eq!(bind.page.as_ref().unwrap().title, "fresh");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn driver_release_reads_the_pinned_minor_series() {
|
||||
assert_eq!(driver_release("cua-driver 0.23.2"), Some((0, 23)));
|
||||
assert_eq!(
|
||||
driver_release("cua-driver 0.23.2 (x86_64-linux)"),
|
||||
Some((0, 23))
|
||||
);
|
||||
assert_eq!(driver_release("cua-driver"), None);
|
||||
assert_eq!(driver_release(""), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_new_minor_series_is_not_compatible() {
|
||||
assert_ne!(driver_release("cua-driver 0.24.0"), Some(PINNED_DRIVER));
|
||||
assert_eq!(driver_release("cua-driver 0.23.9"), Some(PINNED_DRIVER));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalized_drag_uses_one_driver_gesture() {
|
||||
let actions = crate::parse_computer_actions(
|
||||
&json!([{"kind": "drag", "x": 10, "y": 20, "x2": 30, "y2": 40}]),
|
||||
)
|
||||
.unwrap();
|
||||
let payload = drag_payload(&actions).unwrap();
|
||||
assert_eq!(payload["from_x"], 10);
|
||||
assert_eq!(payload["to_y"], 40);
|
||||
assert!(drag_payload(&actions[..4]).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn window_list_skips_panel_and_numbers_from_one() {
|
||||
let raw = json!([
|
||||
{
|
||||
"window_id": 1,
|
||||
"pid": 8,
|
||||
"title": "xfce4-panel",
|
||||
"x": 0,
|
||||
"y": 759,
|
||||
"width": 1280,
|
||||
"height": 41,
|
||||
"z_index": 3
|
||||
},
|
||||
{
|
||||
"window_id": 9,
|
||||
"pid": 20,
|
||||
"title": "終端機",
|
||||
"app_name": "xfce4-terminal",
|
||||
"bounds": { "x": 53, "y": 55, "width": 753, "height": 699 },
|
||||
"z_index": 1
|
||||
}
|
||||
]);
|
||||
let windows = parse_listed_windows(&raw);
|
||||
assert_eq!(windows.len(), 1);
|
||||
assert_eq!(windows[0].title, "終端機");
|
||||
assert_eq!(windows[0].app_name, "xfce4-terminal");
|
||||
assert_eq!(windows[0].w, 753);
|
||||
}
|
||||
|
||||
fn listed(
|
||||
title: &str,
|
||||
app: &str,
|
||||
x: i64,
|
||||
y: i64,
|
||||
w: u32,
|
||||
h: u32,
|
||||
z: i64,
|
||||
id: u64,
|
||||
) -> ListedWindow {
|
||||
ListedWindow {
|
||||
id,
|
||||
pid: id,
|
||||
title: title.into(),
|
||||
app_name: app.into(),
|
||||
x,
|
||||
y,
|
||||
w,
|
||||
h,
|
||||
z,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drag_targets_the_smallest_containing_window() {
|
||||
let gtk = listed(
|
||||
"LazyBoy Cua Smoke",
|
||||
"lazyboy-cua-smoke-gtk",
|
||||
40,
|
||||
40,
|
||||
480,
|
||||
240,
|
||||
1,
|
||||
2,
|
||||
);
|
||||
let chrome = listed(
|
||||
"LazyBoy Cua Smoke - Chromium",
|
||||
"Chromium",
|
||||
0,
|
||||
0,
|
||||
1280,
|
||||
759,
|
||||
4,
|
||||
3,
|
||||
);
|
||||
let windows = [chrome.clone(), gtk.clone()];
|
||||
let hit = window_containing(&windows, 60.0, 70.0).unwrap();
|
||||
assert_eq!(hit.id, gtk.id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn focus_prefers_the_exact_native_title_over_chromium() {
|
||||
let gtk = listed(
|
||||
"LazyBoy Cua Smoke",
|
||||
"lazyboy-cua-smoke-gtk",
|
||||
40,
|
||||
40,
|
||||
480,
|
||||
240,
|
||||
1,
|
||||
2,
|
||||
);
|
||||
let chrome = listed(
|
||||
"LazyBoy Cua Smoke - Chromium",
|
||||
"Chromium",
|
||||
0,
|
||||
0,
|
||||
1280,
|
||||
759,
|
||||
4,
|
||||
3,
|
||||
);
|
||||
let windows = [chrome, gtk];
|
||||
let hit = window_matching_title(&windows, "LazyBoy Cua Smoke").unwrap();
|
||||
assert_eq!(hit.app_name, "lazyboy-cua-smoke-gtk");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,146 @@
|
|||
use std::collections::HashMap;
|
||||
|
||||
use lazyboy_contracts::{RefVerb, UiElement};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::{ListedWindow, client::CuaClient};
|
||||
use crate::ControlError;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct NativeTarget {
|
||||
pid: u64,
|
||||
window_id: u64,
|
||||
token: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct NativeObservation {
|
||||
pub(super) elements: Vec<UiElement>,
|
||||
pub(super) targets: HashMap<String, NativeTarget>,
|
||||
/// False when any window answered with a degraded tree, which means the
|
||||
/// element list is partial and a second opinion is still worth taking.
|
||||
pub(super) complete: bool,
|
||||
}
|
||||
|
||||
pub(super) async fn observe(
|
||||
client: &CuaClient,
|
||||
display: &str,
|
||||
windows: &[ListedWindow],
|
||||
snapshot_id: &str,
|
||||
) -> NativeObservation {
|
||||
let mut observed = NativeObservation {
|
||||
complete: true,
|
||||
..NativeObservation::default()
|
||||
};
|
||||
for window in windows
|
||||
.iter()
|
||||
.filter(|window| !window.app_name.to_ascii_lowercase().contains("chrom"))
|
||||
{
|
||||
let state = match client
|
||||
.call(
|
||||
display,
|
||||
"get_window_state",
|
||||
&json!({
|
||||
"pid": window.pid, "window_id": window.id, "include_screenshot": false,
|
||||
"max_elements": 200, "max_depth": 16,
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(state) => state,
|
||||
Err(_) => {
|
||||
observed.complete = false;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
// A degraded reply is window metadata plus a root node: usable for
|
||||
// discovery, not as an element tree, and never as proof of coverage.
|
||||
if state.get("degraded").and_then(Value::as_bool) == Some(true) {
|
||||
observed.complete = false;
|
||||
continue;
|
||||
}
|
||||
for item in state
|
||||
.get("elements")
|
||||
.and_then(Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
let Some(token) = item.get("element_token").and_then(Value::as_str) else {
|
||||
continue;
|
||||
};
|
||||
if item.get("enabled").and_then(Value::as_bool) == Some(false) {
|
||||
continue;
|
||||
}
|
||||
let selector = format!("cua:{snapshot_id}:{}:{token}", window.id);
|
||||
let title = item.get("label").and_then(Value::as_str).unwrap_or("");
|
||||
let role = item.get("role").and_then(Value::as_str).unwrap_or("");
|
||||
let frame = item.get("frame").unwrap_or(&Value::Null);
|
||||
let coordinate = |key| {
|
||||
frame
|
||||
.get(key)
|
||||
.and_then(Value::as_f64)
|
||||
.unwrap_or(0.0)
|
||||
.clamp(0.0, u32::MAX as f64) as u32
|
||||
};
|
||||
observed.elements.push(UiElement {
|
||||
id: 0,
|
||||
title: if title.is_empty() {
|
||||
role.into()
|
||||
} else {
|
||||
title.into()
|
||||
},
|
||||
x: coordinate("x"),
|
||||
y: coordinate("y"),
|
||||
w: coordinate("w"),
|
||||
h: coordinate("h"),
|
||||
selector: Some(selector.clone()),
|
||||
kind: Some("a11y".into()),
|
||||
role: Some(role.into()),
|
||||
});
|
||||
observed.targets.insert(
|
||||
selector,
|
||||
NativeTarget {
|
||||
pid: window.pid,
|
||||
window_id: window.id,
|
||||
token: token.into(),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
observed
|
||||
}
|
||||
|
||||
pub(super) async fn act(
|
||||
client: &CuaClient,
|
||||
display: &str,
|
||||
target: NativeTarget,
|
||||
verb: RefVerb,
|
||||
text: Option<&str>,
|
||||
) -> Result<(), ControlError> {
|
||||
client
|
||||
.call(
|
||||
display,
|
||||
"bring_to_front",
|
||||
&json!({
|
||||
"pid": target.pid, "window_id": target.window_id,
|
||||
}),
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
let mut payload =
|
||||
json!({"pid": target.pid, "window_id": target.window_id, "element_token": target.token});
|
||||
let tool = match verb {
|
||||
RefVerb::Click => {
|
||||
payload["delivery_mode"] = json!("foreground");
|
||||
"click"
|
||||
}
|
||||
RefVerb::SetValue => {
|
||||
payload["value"] = json!(text.unwrap_or(""));
|
||||
"set_value"
|
||||
}
|
||||
RefVerb::Focus => return Err(ControlError::Unsupported),
|
||||
};
|
||||
client.call(display, tool, &payload, &[]).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -0,0 +1,283 @@
|
|||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
||||
use chrono::DateTime;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
pub fn events_from_dir(dir: &Path) -> Vec<Value> {
|
||||
let mut turns = match fs::read_dir(dir) {
|
||||
Ok(entries) => entries
|
||||
.filter_map(Result::ok)
|
||||
.map(|entry| entry.path())
|
||||
.filter(|path| path.join("action.json").is_file())
|
||||
.collect::<Vec<PathBuf>>(),
|
||||
Err(_) => return Vec::new(),
|
||||
};
|
||||
turns.sort();
|
||||
turns
|
||||
.into_iter()
|
||||
.filter_map(|path| event_from_turn(&path))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn event_from_turn(dir: &Path) -> Option<Value> {
|
||||
let raw = fs::read_to_string(dir.join("action.json")).ok()?;
|
||||
let value: Value = serde_json::from_str(&raw).ok()?;
|
||||
let tool = tool_name(&value);
|
||||
if tool.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let args = arguments(&value);
|
||||
let at = timestamp_ms(&value).unwrap_or_else(|| file_time_ms(&dir.join("action.json")));
|
||||
let el = element_from_args(args);
|
||||
let mut event = match tool.as_str() {
|
||||
"click" | "right_click" | "double_click" | "browser_click" => {
|
||||
json!({ "t": "click", "el": el, "at": at })
|
||||
}
|
||||
"type_text" | "set_value" | "browser_type" => {
|
||||
let name = element_label(&el);
|
||||
let role = el.get("role").and_then(Value::as_str).unwrap_or("");
|
||||
let raw_text = args
|
||||
.get("text")
|
||||
.or_else(|| args.get("value"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("");
|
||||
let value = if looks_secret(&name) || looks_secret(role) {
|
||||
"[已遮罩]"
|
||||
} else {
|
||||
raw_text
|
||||
};
|
||||
json!({ "t": "input", "el": el, "value": value, "at": at })
|
||||
}
|
||||
"press_key" | "hotkey" => {
|
||||
let key = args
|
||||
.get("key")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string)
|
||||
.or_else(|| {
|
||||
args.get("keys").and_then(Value::as_array).map(|keys| {
|
||||
keys.iter()
|
||||
.filter_map(Value::as_str)
|
||||
.collect::<Vec<_>>()
|
||||
.join("+")
|
||||
})
|
||||
})
|
||||
.unwrap_or_default();
|
||||
json!({ "t": "key", "key": key, "el": el, "at": at })
|
||||
}
|
||||
"scroll" => json!({
|
||||
"t": "scroll",
|
||||
"y": args.get("amount").and_then(Value::as_i64).unwrap_or(0),
|
||||
"at": at
|
||||
}),
|
||||
"browser_navigate" => json!({
|
||||
"t": "navigate",
|
||||
"url": args.get("url").and_then(Value::as_str).unwrap_or(""),
|
||||
"at": at
|
||||
}),
|
||||
_ => return None,
|
||||
};
|
||||
if let Some(title) = window_title(dir) {
|
||||
event["window"] = json!(title);
|
||||
}
|
||||
Some(event)
|
||||
}
|
||||
|
||||
fn tool_name(value: &Value) -> String {
|
||||
value
|
||||
.get("tool")
|
||||
.or_else(|| value.get("name"))
|
||||
.or_else(|| value.get("action"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_ascii_lowercase()
|
||||
}
|
||||
|
||||
fn arguments(value: &Value) -> &Value {
|
||||
value
|
||||
.get("arguments")
|
||||
.or_else(|| value.get("input"))
|
||||
.or_else(|| value.get("args"))
|
||||
.unwrap_or(value)
|
||||
}
|
||||
|
||||
fn element_from_args(args: &Value) -> Value {
|
||||
let name = pick(
|
||||
args,
|
||||
&[
|
||||
"name",
|
||||
"label",
|
||||
"ref",
|
||||
"selector",
|
||||
"element_token",
|
||||
"element_index",
|
||||
],
|
||||
);
|
||||
let role = pick(args, &["role"]);
|
||||
json!({
|
||||
"name": name,
|
||||
"label": name,
|
||||
"role": role,
|
||||
"text": name,
|
||||
})
|
||||
}
|
||||
|
||||
fn element_label(el: &Value) -> String {
|
||||
el.get("name")
|
||||
.or_else(|| el.get("label"))
|
||||
.or_else(|| el.get("text"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn pick(value: &Value, keys: &[&str]) -> String {
|
||||
for key in keys {
|
||||
if let Some(text) = value.get(*key).and_then(Value::as_str).map(str::trim)
|
||||
&& !text.is_empty()
|
||||
{
|
||||
return text.to_string();
|
||||
}
|
||||
if let Some(number) = value.get(*key).and_then(Value::as_i64) {
|
||||
return number.to_string();
|
||||
}
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
fn timestamp_ms(value: &Value) -> Option<i64> {
|
||||
if let Some(ms) = value.get("at").and_then(Value::as_i64) {
|
||||
return Some(ms);
|
||||
}
|
||||
let stamp = value
|
||||
.get("timestamp")
|
||||
.or_else(|| value.get("ts"))
|
||||
.or_else(|| value.get("time"))
|
||||
.and_then(Value::as_str)?;
|
||||
DateTime::parse_from_rfc3339(stamp)
|
||||
.ok()
|
||||
.map(|time| time.timestamp_millis())
|
||||
}
|
||||
|
||||
fn file_time_ms(path: &Path) -> i64 {
|
||||
fs::metadata(path)
|
||||
.and_then(|meta| meta.modified())
|
||||
.ok()
|
||||
.and_then(|time| time.duration_since(UNIX_EPOCH).ok())
|
||||
.map(|duration| duration.as_millis() as i64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn window_title(dir: &Path) -> Option<String> {
|
||||
for name in ["after_state.json", "app_state.json", "before_state.json"] {
|
||||
let Ok(raw) = fs::read_to_string(dir.join(name)) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = serde_json::from_str::<Value>(&raw) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(title) = value
|
||||
.get("title")
|
||||
.or_else(|| value.get("window_title"))
|
||||
.or_else(|| value.pointer("/window/title"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|title| !title.is_empty())
|
||||
{
|
||||
return Some(title.to_string());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn looks_secret(text: &str) -> bool {
|
||||
let lower = text.to_lowercase();
|
||||
[
|
||||
"pass",
|
||||
"pwd",
|
||||
"密碼",
|
||||
"token",
|
||||
"otp",
|
||||
"驗證碼",
|
||||
"secret",
|
||||
"cvv",
|
||||
]
|
||||
.iter()
|
||||
.any(|needle| lower.contains(needle))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::SystemTime;
|
||||
|
||||
fn write_turn(root: &Path, name: &str, action: &Value, after: Option<&Value>) {
|
||||
let dir = root.join(name);
|
||||
fs::create_dir_all(&dir).unwrap();
|
||||
fs::write(dir.join("action.json"), action.to_string()).unwrap();
|
||||
if let Some(state) = after {
|
||||
fs::write(dir.join("after_state.json"), state.to_string()).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trajectory_turns_become_skill_events() {
|
||||
let root = std::env::temp_dir().join(format!(
|
||||
"lazyboy-traj-{}-{}",
|
||||
std::process::id(),
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
fs::create_dir_all(&root).unwrap();
|
||||
write_turn(
|
||||
&root,
|
||||
"turn-00001",
|
||||
&json!({
|
||||
"tool": "browser_click",
|
||||
"timestamp": "2026-09-07T00:00:01Z",
|
||||
"arguments": { "ref": "p1:1", "role": "button", "name": "Smoke Click" }
|
||||
}),
|
||||
Some(&json!({ "title": "LazyBoy Cua Smoke" })),
|
||||
);
|
||||
write_turn(
|
||||
&root,
|
||||
"turn-00002",
|
||||
&json!({
|
||||
"tool": "type_text",
|
||||
"timestamp": "2026-09-07T00:00:02Z",
|
||||
"arguments": { "text": "hunter2", "name": "Password", "role": "textbox" }
|
||||
}),
|
||||
None,
|
||||
);
|
||||
write_turn(
|
||||
&root,
|
||||
"turn-00003",
|
||||
&json!({
|
||||
"tool": "browser_navigate",
|
||||
"timestamp": "2026-09-07T00:00:03Z",
|
||||
"arguments": { "url": "https://example.com" }
|
||||
}),
|
||||
None,
|
||||
);
|
||||
let events = events_from_dir(&root);
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
assert_eq!(events.len(), 3);
|
||||
assert_eq!(events[0]["t"], "click");
|
||||
assert_eq!(events[0]["el"]["name"], "Smoke Click");
|
||||
assert_eq!(events[0]["window"], "LazyBoy Cua Smoke");
|
||||
assert_eq!(events[1]["t"], "input");
|
||||
assert_eq!(events[1]["value"], "[已遮罩]");
|
||||
assert_eq!(events[2]["t"], "navigate");
|
||||
assert_eq!(events[2]["url"], "https://example.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secret_labels_are_detected() {
|
||||
assert!(looks_secret("Password"));
|
||||
assert!(looks_secret("確認密碼"));
|
||||
assert!(!looks_secret("Search"));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,297 @@
|
|||
use lazyboy_contracts::{ComputerAction, PointerButton, PointerType, ScrollDirection};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::controller::ControlError;
|
||||
use crate::{launch_argv_on, open_argv_on};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum TranslatedAction {
|
||||
Cua { tool: &'static str, payload: Value },
|
||||
Sleep { ms: u64 },
|
||||
Launch { argv: Vec<String> },
|
||||
FocusTitle { title: String },
|
||||
}
|
||||
|
||||
pub fn translate_action(
|
||||
action: &ComputerAction,
|
||||
display: &str,
|
||||
profile: Option<&str>,
|
||||
) -> Result<TranslatedAction, ControlError> {
|
||||
match action {
|
||||
ComputerAction::Wait { ms } => Ok(TranslatedAction::Sleep { ms: u64::from(*ms) }),
|
||||
ComputerAction::Open { path } => Ok(TranslatedAction::Launch {
|
||||
argv: open_argv_on(display, profile, path),
|
||||
}),
|
||||
ComputerAction::Launch { application, uri } => {
|
||||
let argv = launch_argv_on(display, profile, application, uri.as_deref())
|
||||
.ok_or(ControlError::Unsupported)?;
|
||||
Ok(TranslatedAction::Launch { argv })
|
||||
}
|
||||
ComputerAction::CopySelection | ComputerAction::Ref { .. } => {
|
||||
Err(ControlError::Unsupported)
|
||||
}
|
||||
ComputerAction::Focus { title } => Ok(TranslatedAction::FocusTitle {
|
||||
title: title.clone(),
|
||||
}),
|
||||
ComputerAction::Pointer {
|
||||
x,
|
||||
y,
|
||||
pointer_type,
|
||||
button,
|
||||
} => translate_pointer(*x, *y, pointer_type, *button),
|
||||
ComputerAction::Clipboard { text } => {
|
||||
tracing::info!(backend = "cua", tool = "type_text", length = text.len());
|
||||
Ok(TranslatedAction::Cua {
|
||||
tool: "type_text",
|
||||
payload: json!({
|
||||
"text": text,
|
||||
"scope": "desktop",
|
||||
}),
|
||||
})
|
||||
}
|
||||
ComputerAction::Key { key, modifiers } => Ok(translate_key(key, modifiers.as_deref())),
|
||||
ComputerAction::Scroll { direction, amount } => Ok(TranslatedAction::Cua {
|
||||
tool: "scroll",
|
||||
payload: json!({
|
||||
"direction": match direction {
|
||||
ScrollDirection::Up => "up",
|
||||
ScrollDirection::Down => "down",
|
||||
},
|
||||
"amount": amount.unwrap_or(12).clamp(1, 50),
|
||||
"by": "line",
|
||||
"scope": "desktop",
|
||||
}),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn translate_pointer(
|
||||
x: u32,
|
||||
y: u32,
|
||||
pointer_type: &PointerType,
|
||||
button: Option<PointerButton>,
|
||||
) -> Result<TranslatedAction, ControlError> {
|
||||
let button = match button.unwrap_or(PointerButton::Left) {
|
||||
PointerButton::Left => "left",
|
||||
PointerButton::Middle => "middle",
|
||||
PointerButton::Right => "right",
|
||||
};
|
||||
match pointer_type {
|
||||
PointerType::Move => Ok(TranslatedAction::Cua {
|
||||
tool: "move_cursor",
|
||||
payload: json!({
|
||||
"x": x,
|
||||
"y": y,
|
||||
"scope": "desktop",
|
||||
}),
|
||||
}),
|
||||
PointerType::Click => Ok(TranslatedAction::Cua {
|
||||
tool: "click",
|
||||
payload: json!({
|
||||
"x": x,
|
||||
"y": y,
|
||||
"button": button,
|
||||
"scope": "desktop",
|
||||
}),
|
||||
}),
|
||||
// A held-button gesture collapses into one `drag` call before it gets
|
||||
// here; the CLI has no lease that keeps a button down between calls.
|
||||
PointerType::Down | PointerType::Up => Err(ControlError::Unsupported),
|
||||
}
|
||||
}
|
||||
|
||||
fn translate_key(key: &str, modifiers: Option<&[String]>) -> TranslatedAction {
|
||||
// The public action DSL and mobile keyboard send chords as "ctrl+a".
|
||||
// Cua requires separate keys passed to hotkey, not a literal press_key.
|
||||
let mut keys: Vec<String> = modifiers
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.map(|item| map_key(item))
|
||||
.collect();
|
||||
if key.contains('+') && key.split('+').all(|part| !part.is_empty()) {
|
||||
keys.extend(key.split('+').map(map_key));
|
||||
} else {
|
||||
keys.push(map_key(key));
|
||||
}
|
||||
if keys.len() > 1 {
|
||||
TranslatedAction::Cua {
|
||||
tool: "hotkey",
|
||||
payload: json!({ "keys": keys, "scope": "desktop" }),
|
||||
}
|
||||
} else {
|
||||
TranslatedAction::Cua {
|
||||
tool: "press_key",
|
||||
payload: json!({ "key": keys[0], "scope": "desktop" }),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn map_key(key: &str) -> String {
|
||||
match key.to_ascii_lowercase().as_str() {
|
||||
"enter" | "return" => "return".into(),
|
||||
"esc" | "escape" => "escape".into(),
|
||||
"cmd" | "command" | "super" | "meta" | "win" => "ctrl".into(),
|
||||
"control" | "ctl" => "ctrl".into(),
|
||||
"option" => "alt".into(),
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use lazyboy_contracts::RefVerb;
|
||||
|
||||
#[test]
|
||||
fn inline_shortcuts_use_hotkey_and_literal_plus_stays_a_key() {
|
||||
for (key, expected) in [
|
||||
("ctrl+a", json!(["ctrl", "a"])),
|
||||
("alt+Left", json!(["alt", "left"])),
|
||||
("Control+Shift+Tab", json!(["ctrl", "shift", "tab"])),
|
||||
] {
|
||||
let TranslatedAction::Cua { tool, payload } = translate_key(key, None) else {
|
||||
panic!("expected Cua")
|
||||
};
|
||||
assert_eq!(tool, "hotkey");
|
||||
assert_eq!(payload["keys"], expected);
|
||||
}
|
||||
let TranslatedAction::Cua { tool, payload } = translate_key("+", None) else {
|
||||
panic!("expected Cua")
|
||||
};
|
||||
assert_eq!(tool, "press_key");
|
||||
assert_eq!(payload["key"], "+");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pixel_click_is_desktop_cua_click() {
|
||||
let action = ComputerAction::Pointer {
|
||||
x: 40,
|
||||
y: 80,
|
||||
pointer_type: PointerType::Click,
|
||||
button: Some(PointerButton::Left),
|
||||
};
|
||||
let TranslatedAction::Cua { tool, payload } =
|
||||
translate_action(&action, ":1", None).unwrap()
|
||||
else {
|
||||
panic!("expected cua click");
|
||||
};
|
||||
assert_eq!(tool, "click");
|
||||
assert_eq!(payload["x"], 40);
|
||||
assert_eq!(payload["y"], 80);
|
||||
assert_eq!(payload["scope"], "desktop");
|
||||
// 0.23.2 rejects an explicit desktop target with invalid_action_target.
|
||||
assert!(payload.get("target").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn desktop_actions_omit_the_rejected_target() {
|
||||
for action in [
|
||||
ComputerAction::Clipboard { text: "hi".into() },
|
||||
ComputerAction::Key {
|
||||
key: "return".into(),
|
||||
modifiers: None,
|
||||
},
|
||||
ComputerAction::Key {
|
||||
key: "c".into(),
|
||||
modifiers: Some(vec!["ctrl".into()]),
|
||||
},
|
||||
ComputerAction::Pointer {
|
||||
x: 1,
|
||||
y: 2,
|
||||
pointer_type: PointerType::Move,
|
||||
button: None,
|
||||
},
|
||||
ComputerAction::Scroll {
|
||||
direction: ScrollDirection::Down,
|
||||
amount: None,
|
||||
},
|
||||
] {
|
||||
let TranslatedAction::Cua { payload, .. } =
|
||||
translate_action(&action, ":1", None).unwrap()
|
||||
else {
|
||||
panic!("every desktop action translates to a Cua tool");
|
||||
};
|
||||
assert!(payload.get("target").is_none(), "{action:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scroll_amount_stays_inside_the_driver_window() {
|
||||
let scroll = |amount| match translate_action(
|
||||
&ComputerAction::Scroll {
|
||||
direction: ScrollDirection::Down,
|
||||
amount: Some(amount),
|
||||
},
|
||||
":1",
|
||||
None,
|
||||
)
|
||||
.unwrap()
|
||||
{
|
||||
TranslatedAction::Cua { payload, .. } => payload["amount"].as_i64().unwrap(),
|
||||
other => panic!("expected scroll, got {other:?}"),
|
||||
};
|
||||
assert_eq!(scroll(0), 1);
|
||||
assert_eq!(scroll(12), 12);
|
||||
assert_eq!(scroll(200), 50);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn held_buttons_are_not_translated_one_by_one() {
|
||||
for pointer_type in [PointerType::Down, PointerType::Up] {
|
||||
let action = ComputerAction::Pointer {
|
||||
x: 1,
|
||||
y: 2,
|
||||
pointer_type,
|
||||
button: Some(PointerButton::Left),
|
||||
};
|
||||
assert!(matches!(
|
||||
translate_action(&action, ":1", None),
|
||||
Err(ControlError::Unsupported)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn typed_text_is_redacted_from_tool_name_only() {
|
||||
let action = ComputerAction::Clipboard {
|
||||
text: "secret-password".into(),
|
||||
};
|
||||
let TranslatedAction::Cua { tool, payload } =
|
||||
translate_action(&action, ":1", None).unwrap()
|
||||
else {
|
||||
panic!("expected type_text");
|
||||
};
|
||||
assert_eq!(tool, "type_text");
|
||||
assert_eq!(payload["text"], "secret-password");
|
||||
assert_eq!(payload["scope"], "desktop");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chord_uses_hotkey_and_maps_cmd_to_ctrl() {
|
||||
let action = ComputerAction::Key {
|
||||
key: "c".into(),
|
||||
modifiers: Some(vec!["cmd".into()]),
|
||||
};
|
||||
let TranslatedAction::Cua { tool, payload } =
|
||||
translate_action(&action, ":1", None).unwrap()
|
||||
else {
|
||||
panic!("expected hotkey");
|
||||
};
|
||||
assert_eq!(tool, "hotkey");
|
||||
assert_eq!(payload["keys"], json!(["ctrl", "c"]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn semantic_ref_is_not_translated_here() {
|
||||
let action = ComputerAction::Ref {
|
||||
verb: RefVerb::Click,
|
||||
target: "#go".into(),
|
||||
ref_kind: "dom".into(),
|
||||
text: None,
|
||||
};
|
||||
assert!(matches!(
|
||||
translate_action(&action, ":1", None),
|
||||
Err(ControlError::Unsupported)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,15 +1,9 @@
|
|||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ScreenLease {
|
||||
pub owner_id: String,
|
||||
pub fence: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Error, Clone, PartialEq, Eq)]
|
||||
#[error("computer is busy")]
|
||||
pub struct ComputerBusyError;
|
||||
|
||||
pub fn screen_lease_id(run_id: &str, fence: u32) -> String {
|
||||
format!("{run_id}:{fence}")
|
||||
}
|
||||
|
|
@ -57,10 +51,6 @@ pub fn can_release_screen_lease(existing: Option<&str>, incoming: Option<&str>)
|
|||
}
|
||||
}
|
||||
|
||||
pub fn next_fence(current: u32) -> u32 {
|
||||
current.saturating_add(1)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
mod a11y;
|
||||
mod actions;
|
||||
mod cdp;
|
||||
mod browser_page;
|
||||
mod controller;
|
||||
mod cua;
|
||||
mod lease;
|
||||
mod observe;
|
||||
mod overlay;
|
||||
|
|
@ -12,7 +14,9 @@ mod x11;
|
|||
|
||||
pub use a11y::*;
|
||||
pub use actions::*;
|
||||
pub use cdp::*;
|
||||
pub use browser_page::*;
|
||||
pub use controller::*;
|
||||
pub use cua::{CuaClient, CuaController, TranslatedAction, translate_action};
|
||||
pub use lease::*;
|
||||
pub use observe::*;
|
||||
pub use overlay::*;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
use base64::Engine;
|
||||
use chrono::Utc;
|
||||
use lazyboy_contracts::{ActiveWindow, ComputerObservation, CursorPosition, UiElement};
|
||||
use serde_json::{Value, json};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub fn observation_from_png(
|
||||
|
|
@ -10,6 +12,7 @@ pub fn observation_from_png(
|
|||
active_window: Option<ActiveWindow>,
|
||||
) -> ComputerObservation {
|
||||
ComputerObservation {
|
||||
native_observation_complete: false,
|
||||
frame_id: hex::encode(Sha256::digest(&image)),
|
||||
captured_at: Utc::now().to_rfc3339(),
|
||||
mime_type: sniff_image_mime(&image).to_string(),
|
||||
|
|
@ -22,6 +25,39 @@ pub fn observation_from_png(
|
|||
}
|
||||
}
|
||||
|
||||
/// Pixel size of a captured frame, or `None` when the bytes are not decodable.
|
||||
/// Capture inputs may use different codecs (imported frames may be JPEG
|
||||
/// from `xwd | convert`, the Cua driver writes PNG), so the dimensions have to
|
||||
/// come from the frame itself rather than from a configured constant.
|
||||
pub fn image_dimensions(bytes: &[u8]) -> Option<(u32, u32)> {
|
||||
use image::ImageDecoder;
|
||||
|
||||
let reader = image::ImageReader::new(std::io::Cursor::new(bytes))
|
||||
.with_guessed_format()
|
||||
.ok()?;
|
||||
let decoder = reader.into_decoder().ok()?;
|
||||
Some(decoder.dimensions())
|
||||
}
|
||||
|
||||
pub fn observation_to_control_json(observation: &ComputerObservation) -> Value {
|
||||
let mut body = json!({
|
||||
"png_base64": base64::engine::general_purpose::STANDARD.encode(&observation.image),
|
||||
"native_observation_complete": observation.native_observation_complete,
|
||||
});
|
||||
if let Some(cursor) = &observation.cursor {
|
||||
body["cursor"] = json!({ "x": cursor.x, "y": cursor.y });
|
||||
}
|
||||
if let Some(window) = &observation.active_window {
|
||||
body["activeWindow"] = json!({ "id": window.id, "title": window.title });
|
||||
}
|
||||
if !observation.elements.is_empty()
|
||||
&& let Ok(value) = serde_json::to_value(&observation.elements)
|
||||
{
|
||||
body["elements"] = value;
|
||||
}
|
||||
body
|
||||
}
|
||||
|
||||
pub fn observation_with_elements(
|
||||
mut observation: ComputerObservation,
|
||||
elements: Vec<UiElement>,
|
||||
|
|
@ -62,7 +98,11 @@ const SIGNATURE_H: u32 = 18;
|
|||
pub fn frame_signature(image: &[u8]) -> Option<Vec<u8>> {
|
||||
let dynamic = image::load_from_memory(image).ok()?;
|
||||
let thumb = dynamic
|
||||
.resize_exact(SIGNATURE_W, SIGNATURE_H, image::imageops::FilterType::Triangle)
|
||||
.resize_exact(
|
||||
SIGNATURE_W,
|
||||
SIGNATURE_H,
|
||||
image::imageops::FilterType::Triangle,
|
||||
)
|
||||
.to_luma8();
|
||||
Some(thumb.into_raw())
|
||||
}
|
||||
|
|
@ -83,6 +123,33 @@ pub fn signatures_similar(a: &[u8], b: &[u8]) -> bool {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use image::codecs::jpeg::JpegEncoder;
|
||||
use image::codecs::png::PngEncoder;
|
||||
use image::{ExtendedColorType, ImageEncoder, Rgb, RgbImage};
|
||||
use std::io::Cursor;
|
||||
|
||||
#[test]
|
||||
fn dimensions_come_from_the_frame_for_both_codecs() {
|
||||
let frame = RgbImage::from_pixel(96, 48, Rgb([10, 20, 30]));
|
||||
|
||||
let mut png = Cursor::new(Vec::new());
|
||||
PngEncoder::new(&mut png)
|
||||
.write_image(frame.as_raw(), 96, 48, ExtendedColorType::Rgb8)
|
||||
.unwrap();
|
||||
assert_eq!(image_dimensions(&png.into_inner()), Some((96, 48)));
|
||||
|
||||
// Imported captures can use JPEG, so a hardcoded size would drift the
|
||||
// moment the Xvfb geometry changes.
|
||||
let mut jpeg = Cursor::new(Vec::new());
|
||||
JpegEncoder::new_with_quality(&mut jpeg, 60)
|
||||
.encode(frame.as_raw(), 96, 48, ExtendedColorType::Rgb8)
|
||||
.unwrap();
|
||||
assert_eq!(image_dimensions(&jpeg.into_inner()), Some((96, 48)));
|
||||
|
||||
// A truncated capture must not invent a size.
|
||||
assert_eq!(image_dimensions(&[0xFF, 0xD8, 0xFF]), None);
|
||||
assert_eq!(image_dimensions(&[]), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identical_bytes_share_a_frame_id() {
|
||||
|
|
@ -108,12 +175,20 @@ mod tests {
|
|||
let base = frame_signature(&png(|_, _| Rgb([240, 240, 240]))).unwrap();
|
||||
// A panel clock flipping digits touches a couple of pixels only.
|
||||
let clock = frame_signature(&png(|x, y| {
|
||||
if x < 6 && y < 6 { Rgb([0, 0, 0]) } else { Rgb([240, 240, 240]) }
|
||||
if x < 6 && y < 6 {
|
||||
Rgb([0, 0, 0])
|
||||
} else {
|
||||
Rgb([240, 240, 240])
|
||||
}
|
||||
}))
|
||||
.unwrap();
|
||||
// A dialog covering a quarter of the screen.
|
||||
let dialog = frame_signature(&png(|x, y| {
|
||||
if x < 160 && y < 90 { Rgb([20, 20, 20]) } else { Rgb([240, 240, 240]) }
|
||||
if x < 160 && y < 90 {
|
||||
Rgb([20, 20, 20])
|
||||
} else {
|
||||
Rgb([240, 240, 240])
|
||||
}
|
||||
}))
|
||||
.unwrap();
|
||||
assert!(signatures_similar(&base, &clock));
|
||||
|
|
|
|||
|
|
@ -150,7 +150,7 @@ fn draw_badge(img: &mut RgbImage, x: u32, y: u32, id: u32) {
|
|||
2,
|
||||
);
|
||||
for (i, ch) in label.bytes().enumerate() {
|
||||
if !(b'0'..=b'9').contains(&ch) {
|
||||
if !ch.is_ascii_digit() {
|
||||
continue;
|
||||
}
|
||||
let dx = bx + pad + i as u32 * digit_w;
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ use lazyboy_contracts::{ComputerAction, ComputerCapabilities, ComputerObservatio
|
|||
use serde::{Deserialize, Serialize};
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
pub struct AdapterContext {
|
||||
pub operation_id: String,
|
||||
pub space_id: String,
|
||||
|
|
@ -20,23 +20,6 @@ pub struct AdapterContext {
|
|||
pub profile_path: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for AdapterContext {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
operation_id: String::new(),
|
||||
space_id: String::new(),
|
||||
user_id: String::new(),
|
||||
bot_id: None,
|
||||
run_id: None,
|
||||
screen_lease_id: None,
|
||||
screen_id: None,
|
||||
screen_slot: None,
|
||||
display: None,
|
||||
profile_path: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ComputerRef {
|
||||
pub id: String,
|
||||
|
|
@ -53,7 +36,7 @@ pub struct ProvisionRequest {
|
|||
pub provider_ref: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
pub struct CommandRequest {
|
||||
pub argv: Vec<String>,
|
||||
pub cwd: Option<String>,
|
||||
|
|
@ -64,17 +47,6 @@ pub struct CommandRequest {
|
|||
pub stdin: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for CommandRequest {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
argv: Vec::new(),
|
||||
cwd: None,
|
||||
timeout_ms: None,
|
||||
stdin: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct CommandResult {
|
||||
pub stdout: String,
|
||||
|
|
@ -105,11 +77,62 @@ impl ActionRequest {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BrowserRequest {
|
||||
pub action: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub url: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub text: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub key: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub ms: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selector: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub wait_ms: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub ensure: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub display: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub profile_path: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RecordingRequest {
|
||||
pub skill_id: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub display: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub profile_path: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RecordingSession {
|
||||
pub skill_id: String,
|
||||
pub output_dir: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RecordingResult {
|
||||
pub events: Vec<serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct EnsureScreenRequest {
|
||||
pub slot: u32,
|
||||
pub profile_path: String,
|
||||
pub bot_id: String,
|
||||
#[serde(default)]
|
||||
pub bot_name: String,
|
||||
#[serde(default)]
|
||||
pub bot_color: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
|
|
@ -121,6 +144,8 @@ pub struct EnsureScreenResult {
|
|||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ActionResult {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub clipboard_text: Option<String>,
|
||||
pub completed: usize,
|
||||
pub observation: Option<ComputerObservation>,
|
||||
}
|
||||
|
|
@ -236,6 +261,46 @@ pub trait SandboxProvider: Send + Sync {
|
|||
context: &AdapterContext,
|
||||
) -> Result<ActionResult, SandboxError>;
|
||||
|
||||
async fn browser(
|
||||
&self,
|
||||
computer: &ComputerRef,
|
||||
request: BrowserRequest,
|
||||
context: &AdapterContext,
|
||||
) -> Result<crate::BrowserPage, SandboxError> {
|
||||
let _ = (computer, request, context);
|
||||
Err(SandboxError::message("browser is unavailable"))
|
||||
}
|
||||
|
||||
async fn start_recording(
|
||||
&self,
|
||||
computer: &ComputerRef,
|
||||
request: RecordingRequest,
|
||||
context: &AdapterContext,
|
||||
) -> Result<RecordingSession, SandboxError> {
|
||||
let _ = (computer, request, context);
|
||||
Err(SandboxError::message("recording is unavailable"))
|
||||
}
|
||||
|
||||
async fn stop_recording(
|
||||
&self,
|
||||
computer: &ComputerRef,
|
||||
request: RecordingRequest,
|
||||
context: &AdapterContext,
|
||||
) -> Result<(), SandboxError> {
|
||||
let _ = (computer, request, context);
|
||||
Err(SandboxError::message("recording is unavailable"))
|
||||
}
|
||||
|
||||
async fn collect_recording(
|
||||
&self,
|
||||
computer: &ComputerRef,
|
||||
request: RecordingRequest,
|
||||
context: &AdapterContext,
|
||||
) -> Result<RecordingResult, SandboxError> {
|
||||
let _ = (computer, request, context);
|
||||
Err(SandboxError::message("recording is unavailable"))
|
||||
}
|
||||
|
||||
async fn connect_screen(
|
||||
&self,
|
||||
computer: &ComputerRef,
|
||||
|
|
@ -277,3 +342,35 @@ pub trait SandboxProvider: Send + Sync {
|
|||
context: &AdapterContext,
|
||||
) -> Result<(), SandboxError>;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn browser_request_reads_camel_case_wait_ms() {
|
||||
let request: BrowserRequest = serde_json::from_value(json!({
|
||||
"action": "click",
|
||||
"waitMs": 12000,
|
||||
"ensure": true,
|
||||
"selector": "p1:1"
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(request.action, "click");
|
||||
assert_eq!(request.wait_ms, Some(12_000));
|
||||
assert!(request.ensure);
|
||||
assert_eq!(request.selector.as_deref(), Some("p1:1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recording_request_reads_camel_case_skill_id() {
|
||||
let request: RecordingRequest = serde_json::from_value(json!({
|
||||
"skillId": "abc-1",
|
||||
"display": ":2"
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(request.skill_id, "abc-1");
|
||||
assert_eq!(request.display.as_deref(), Some(":2"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,151 +1,18 @@
|
|||
use lazyboy_contracts::{ComputerAction, PointerButton, PointerType, ScrollDirection};
|
||||
use lazyboy_contracts::{ComputerAction, PointerType};
|
||||
|
||||
pub fn is_browser_title(title: &str) -> bool {
|
||||
let title = title.to_lowercase();
|
||||
title.contains("chromium") || title.contains("chrome")
|
||||
}
|
||||
|
||||
use crate::screen::{PRIMARY_DISPLAY, normalize_display};
|
||||
use crate::screen::normalize_display;
|
||||
|
||||
pub const DISPLAY: &str = PRIMARY_DISPLAY;
|
||||
pub const HOME: &str = "/home/lazyboy";
|
||||
|
||||
fn display_env(display: &str) -> String {
|
||||
format!("DISPLAY={}", normalize_display(display))
|
||||
}
|
||||
|
||||
pub fn xdotool_argv(action: &ComputerAction) -> Option<Vec<String>> {
|
||||
xdotool_argv_on(PRIMARY_DISPLAY, action)
|
||||
}
|
||||
|
||||
pub fn xdotool_argv_on(display: &str, action: &ComputerAction) -> Option<Vec<String>> {
|
||||
let mut argv = vec!["env".into(), display_env(display), "xdotool".into()];
|
||||
match action {
|
||||
ComputerAction::Pointer {
|
||||
x,
|
||||
y,
|
||||
pointer_type,
|
||||
button,
|
||||
} => {
|
||||
let button_n = match button.unwrap_or(PointerButton::Left) {
|
||||
PointerButton::Left => "1",
|
||||
PointerButton::Middle => "2",
|
||||
PointerButton::Right => "3",
|
||||
};
|
||||
match pointer_type {
|
||||
PointerType::Move => {
|
||||
argv.extend([
|
||||
"mousemove".into(),
|
||||
"--sync".into(),
|
||||
"--".into(),
|
||||
x.to_string(),
|
||||
y.to_string(),
|
||||
]);
|
||||
}
|
||||
PointerType::Click => {
|
||||
argv.extend([
|
||||
"mousemove".into(),
|
||||
"--sync".into(),
|
||||
"--".into(),
|
||||
x.to_string(),
|
||||
y.to_string(),
|
||||
"click".into(),
|
||||
"--delay".into(),
|
||||
"40".into(),
|
||||
button_n.into(),
|
||||
]);
|
||||
}
|
||||
PointerType::Down => {
|
||||
argv.extend([
|
||||
"mousemove".into(),
|
||||
"--".into(),
|
||||
x.to_string(),
|
||||
y.to_string(),
|
||||
"mousedown".into(),
|
||||
button_n.into(),
|
||||
]);
|
||||
}
|
||||
PointerType::Up => {
|
||||
argv.extend([
|
||||
"mousemove".into(),
|
||||
"--".into(),
|
||||
x.to_string(),
|
||||
y.to_string(),
|
||||
"mouseup".into(),
|
||||
button_n.into(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
ComputerAction::Key { key, modifiers } => {
|
||||
let combo = match modifiers {
|
||||
Some(items) if !items.is_empty() => format!("{}+{key}", items.join("+")),
|
||||
_ => key.clone(),
|
||||
};
|
||||
argv.extend(["key".into(), "--clearmodifiers".into(), combo]);
|
||||
}
|
||||
ComputerAction::Clipboard { text } => {
|
||||
let quoted = shell_single_quote(text);
|
||||
if looks_like_typed_ascii(text) {
|
||||
argv.extend([
|
||||
"type".into(),
|
||||
"--delay".into(),
|
||||
"16".into(),
|
||||
"--".into(),
|
||||
text.clone(),
|
||||
]);
|
||||
} else {
|
||||
return Some(vec![
|
||||
"env".into(),
|
||||
display_env(display),
|
||||
"bash".into(),
|
||||
"-lc".into(),
|
||||
format!(
|
||||
"printf %s {quoted} | xclip -selection clipboard && xdotool key --clearmodifiers ctrl+v"
|
||||
),
|
||||
]);
|
||||
}
|
||||
}
|
||||
ComputerAction::Scroll { direction, amount } => {
|
||||
let button = match direction {
|
||||
ScrollDirection::Up => "4",
|
||||
ScrollDirection::Down => "5",
|
||||
};
|
||||
argv.extend([
|
||||
"click".into(),
|
||||
"--repeat".into(),
|
||||
amount.unwrap_or(12).to_string(),
|
||||
"--delay".into(),
|
||||
"15".into(),
|
||||
button.into(),
|
||||
]);
|
||||
}
|
||||
ComputerAction::Focus { title } => {
|
||||
let quoted = shell_single_quote(title);
|
||||
return Some(vec![
|
||||
"env".into(),
|
||||
display_env(display),
|
||||
"bash".into(),
|
||||
"-lc".into(),
|
||||
format!(
|
||||
"wmctrl -a {quoted} || xdotool search --name {quoted} windowactivate --sync windowfocus"
|
||||
),
|
||||
]);
|
||||
}
|
||||
ComputerAction::Wait { .. }
|
||||
| ComputerAction::Open { .. }
|
||||
| ComputerAction::Launch { .. }
|
||||
| ComputerAction::Ref { .. } => {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
Some(argv)
|
||||
}
|
||||
|
||||
fn shell_single_quote(value: &str) -> String {
|
||||
format!("'{}'", value.replace('\'', "'\\''"))
|
||||
}
|
||||
|
||||
fn looks_like_typed_ascii(text: &str) -> bool {
|
||||
text.len() <= 48
|
||||
&& text
|
||||
|
|
@ -165,81 +32,11 @@ pub fn action_pause_ms(action: &ComputerAction) -> u64 {
|
|||
ComputerAction::Clipboard { .. } | ComputerAction::Key { .. } => 35,
|
||||
ComputerAction::Focus { .. } => 90,
|
||||
ComputerAction::Open { .. } | ComputerAction::Launch { .. } => 220,
|
||||
ComputerAction::Wait { .. } => 0,
|
||||
ComputerAction::Wait { .. } | ComputerAction::CopySelection => 0,
|
||||
ComputerAction::Ref { .. } => 55,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn pointer_state_command() -> Vec<String> {
|
||||
pointer_state_command_on(PRIMARY_DISPLAY)
|
||||
}
|
||||
|
||||
pub fn pointer_state_command_on(display: &str) -> Vec<String> {
|
||||
vec![
|
||||
"env".into(),
|
||||
display_env(display),
|
||||
"python3".into(),
|
||||
"-c".into(),
|
||||
r#"
|
||||
import json, subprocess
|
||||
def out(args):
|
||||
try:
|
||||
return subprocess.check_output(args, stderr=subprocess.DEVNULL, text=True).strip()
|
||||
except Exception:
|
||||
return ""
|
||||
vals = {}
|
||||
for line in out(["xdotool", "getmouselocation", "--shell"]).splitlines():
|
||||
if "=" in line:
|
||||
key, value = line.split("=", 1)
|
||||
vals[key] = value
|
||||
wid = out(["xdotool", "getactivewindow"])
|
||||
title = out(["xdotool", "getwindowname", wid]) if wid else ""
|
||||
print(json.dumps({"x": int(vals.get("X") or 0), "y": int(vals.get("Y") or 0), "id": wid, "title": title}))
|
||||
"#
|
||||
.into(),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn window_list_command() -> Vec<String> {
|
||||
window_list_command_on(PRIMARY_DISPLAY)
|
||||
}
|
||||
|
||||
pub fn window_list_command_on(display: &str) -> Vec<String> {
|
||||
vec![
|
||||
"env".into(),
|
||||
display_env(display),
|
||||
"python3".into(),
|
||||
"-c".into(),
|
||||
r#"
|
||||
import json, subprocess
|
||||
def out(args):
|
||||
try:
|
||||
return subprocess.check_output(args, stderr=subprocess.DEVNULL, text=True)
|
||||
except Exception:
|
||||
return ""
|
||||
els = []
|
||||
n = 1
|
||||
for line in out(["wmctrl", "-lG"]).splitlines():
|
||||
parts = line.split(None, 7)
|
||||
if len(parts) < 7:
|
||||
continue
|
||||
try:
|
||||
x, y, w, h = int(parts[2]), int(parts[3]), int(parts[4]), int(parts[5])
|
||||
except ValueError:
|
||||
continue
|
||||
if w < 32 or h < 16:
|
||||
continue
|
||||
title = parts[7].strip() if len(parts) > 7 else parts[6]
|
||||
if not title or title in ("Desktop", "xfce4-panel"):
|
||||
continue
|
||||
els.append({"id": n, "title": title[:80], "kind": "window", "x": max(0, x), "y": max(0, y), "w": w, "h": h})
|
||||
n += 1
|
||||
print(json.dumps(els))
|
||||
"#
|
||||
.into(),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn parse_ui_elements(raw: &str) -> Vec<lazyboy_contracts::UiElement> {
|
||||
let value: serde_json::Value =
|
||||
serde_json::from_str(raw.trim()).unwrap_or(serde_json::Value::Null);
|
||||
|
|
@ -276,43 +73,6 @@ pub fn parse_ui_elements(raw: &str) -> Vec<lazyboy_contracts::UiElement> {
|
|||
.collect()
|
||||
}
|
||||
|
||||
pub fn parse_pointer_state(
|
||||
raw: &str,
|
||||
) -> (
|
||||
Option<lazyboy_contracts::CursorPosition>,
|
||||
Option<lazyboy_contracts::ActiveWindow>,
|
||||
) {
|
||||
let value: serde_json::Value =
|
||||
serde_json::from_str(raw.trim()).unwrap_or(serde_json::Value::Null);
|
||||
let cursor = match (
|
||||
value.get("x").and_then(serde_json::Value::as_i64),
|
||||
value.get("y").and_then(serde_json::Value::as_i64),
|
||||
) {
|
||||
(Some(x), Some(y)) => Some(lazyboy_contracts::CursorPosition {
|
||||
x: x as i32,
|
||||
y: y as i32,
|
||||
}),
|
||||
_ => None,
|
||||
};
|
||||
let window = value
|
||||
.get("id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|id| !id.is_empty())
|
||||
.map(|id| lazyboy_contracts::ActiveWindow {
|
||||
id: id.to_string(),
|
||||
title: value
|
||||
.get("title")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|title| !title.is_empty())
|
||||
.map(str::to_string),
|
||||
});
|
||||
(cursor, window)
|
||||
}
|
||||
|
||||
pub fn open_argv(path: &str) -> Vec<String> {
|
||||
open_argv_on(PRIMARY_DISPLAY, None, path)
|
||||
}
|
||||
|
||||
pub fn open_argv_on(display: &str, profile: Option<&str>, path: &str) -> Vec<String> {
|
||||
if path.starts_with("http://") || path.starts_with("https://") {
|
||||
return browser_argv(display, profile, Some(path));
|
||||
|
|
@ -325,10 +85,6 @@ pub fn open_argv_on(display: &str, profile: Option<&str>, path: &str) -> Vec<Str
|
|||
]
|
||||
}
|
||||
|
||||
pub fn launch_argv(application: &str, uri: Option<&str>) -> Option<Vec<String>> {
|
||||
launch_argv_on(PRIMARY_DISPLAY, None, application, uri)
|
||||
}
|
||||
|
||||
pub fn launch_argv_on(
|
||||
display: &str,
|
||||
profile: Option<&str>,
|
||||
|
|
@ -360,91 +116,16 @@ fn browser_argv(display: &str, profile: Option<&str>, uri: Option<&str>) -> Vec<
|
|||
argv.push(format!("LAZYBOY_BROWSER_PROFILE={profile}"));
|
||||
}
|
||||
argv.push("lazyboy-browser".into());
|
||||
argv.push(format!(
|
||||
"--remote-debugging-port={}",
|
||||
crate::devtools_port(display)
|
||||
));
|
||||
argv.push("--remote-allow-origins=*".into());
|
||||
if let Some(uri) = uri {
|
||||
argv.push(uri.into());
|
||||
}
|
||||
argv
|
||||
}
|
||||
|
||||
pub fn screenshot_command() -> Vec<String> {
|
||||
screenshot_command_on(PRIMARY_DISPLAY)
|
||||
}
|
||||
|
||||
pub fn screenshot_command_on(display: &str) -> Vec<String> {
|
||||
vec![
|
||||
"bash".into(),
|
||||
"-lc".into(),
|
||||
format!(
|
||||
"DISPLAY={} xwd -root -silent | convert xwd:- -quality 60 jpeg:-",
|
||||
normalize_display(display)
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn click_maps_to_xdotool() {
|
||||
let argv = xdotool_argv(&ComputerAction::Pointer {
|
||||
x: 12,
|
||||
y: 40,
|
||||
pointer_type: PointerType::Click,
|
||||
button: Some(PointerButton::Left),
|
||||
})
|
||||
.unwrap();
|
||||
assert!(argv.contains(&"click".into()));
|
||||
assert!(argv.contains(&"12".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pointer_up_moves_before_release() {
|
||||
let argv = xdotool_argv(&ComputerAction::Pointer {
|
||||
x: 80,
|
||||
y: 90,
|
||||
pointer_type: PointerType::Up,
|
||||
button: Some(PointerButton::Left),
|
||||
})
|
||||
.unwrap();
|
||||
assert!(argv.contains(&"mousemove".into()));
|
||||
assert!(argv.contains(&"80".into()));
|
||||
assert!(argv.contains(&"mouseup".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extra_display_is_injected_into_input_commands() {
|
||||
let argv = xdotool_argv_on(
|
||||
":2",
|
||||
&ComputerAction::Pointer {
|
||||
x: 4,
|
||||
y: 8,
|
||||
pointer_type: PointerType::Move,
|
||||
button: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(argv.contains(&"DISPLAY=:2".into()));
|
||||
let shot = screenshot_command_on(":3");
|
||||
assert!(shot.last().unwrap().contains("DISPLAY=:3"));
|
||||
assert!(shot.last().unwrap().contains("jpeg:-"));
|
||||
let browser = launch_argv_on(
|
||||
":2",
|
||||
Some("/home/lazyboy/.browser-profiles/bots/a"),
|
||||
"browser",
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(browser.contains(&"DISPLAY=:2".into()));
|
||||
assert!(browser.iter().any(|item| item.contains("bots/a")));
|
||||
assert!(browser.contains(&"--remote-debugging-port=9223".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_window_list_elements() {
|
||||
let elements =
|
||||
|
|
@ -490,14 +171,3 @@ mod tests {
|
|||
assert!(parse_ui_elements("not-json").is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
/// Native clipboard path; text is supplied on stdin, never process arguments.
|
||||
pub fn paste_command_on(display: &str) -> Vec<String> {
|
||||
vec![
|
||||
"env".into(),
|
||||
display_env(display),
|
||||
"python3".into(),
|
||||
"-c".into(),
|
||||
include_str!("clipboard.py").into(),
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,17 +2,17 @@
|
|||
name = "lazyboy-controld"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
publish.workspace = true
|
||||
|
||||
[dependencies]
|
||||
lazyboy-contracts.workspace = true
|
||||
lazyboy-control.workspace = true
|
||||
axum.workspace = true
|
||||
tokio.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
base64.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
|
|
|||
|
|
@ -1,24 +1,20 @@
|
|||
use std::process::Stdio;
|
||||
use std::time::Duration;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::extract::State;
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Json, Router};
|
||||
use lazyboy_contracts::{ComputerAction, RefVerb};
|
||||
use lazyboy_control::{
|
||||
ActionRequest, PRIMARY_DISPLAY, a11y_command_on, action_pause_ms, cdp_command_on,
|
||||
launch_argv_on, normalize_display, open_argv_on, parse_a11y_page, parse_cdp_page,
|
||||
parse_pointer_state, parse_ui_elements, pointer_state_command_on, screenshot_command_on,
|
||||
window_list_command_on, xdotool_argv_on,
|
||||
ActionRequest, BrowserRequest, ComputerController, ComputerDriver, ControlContext,
|
||||
ControlError, PRIMARY_DISPLAY, RecordingRequest, normalize_display,
|
||||
observation_to_control_json,
|
||||
};
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::process::Command;
|
||||
use tokio::time::sleep;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct App {
|
||||
token: String,
|
||||
controller: Arc<dyn ComputerController>,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
|
|
@ -27,11 +23,21 @@ async fn main() {
|
|||
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
|
||||
.init();
|
||||
let token = std::env::var("LAZYBOY_CONTROL_TOKEN").unwrap_or_default();
|
||||
let driver = ComputerDriver::from_env();
|
||||
tracing::info!(backend = driver.as_str(), "computer controller");
|
||||
let app = Router::new()
|
||||
.route("/health", get(|| async { "ok" }))
|
||||
.route("/controller/health", get(controller_health))
|
||||
.route("/observe", post(observe))
|
||||
.route("/act", post(act))
|
||||
.with_state(App { token });
|
||||
.route("/browser", post(browser))
|
||||
.route("/recording/start", post(recording_start))
|
||||
.route("/recording/stop", post(recording_stop))
|
||||
.route("/recording/collect", post(recording_collect))
|
||||
.with_state(App {
|
||||
token,
|
||||
controller: driver.controller(),
|
||||
});
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:7070")
|
||||
.await
|
||||
.expect("bind control port");
|
||||
|
|
@ -72,227 +78,179 @@ fn profile_of(headers: &HeaderMap, fallback: Option<&str>) -> Option<String> {
|
|||
.map(str::to_string)
|
||||
}
|
||||
|
||||
struct ControlFailure(StatusCode, String);
|
||||
|
||||
impl IntoResponse for ControlFailure {
|
||||
fn into_response(self) -> Response {
|
||||
(
|
||||
self.0,
|
||||
Json(serde_json::json!({ "ok": false, "error": self.1 })),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StatusCode> for ControlFailure {
|
||||
fn from(status: StatusCode) -> Self {
|
||||
Self(status, status.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn status_for(error: &ControlError) -> ControlFailure {
|
||||
let status = if error.is_client_error() {
|
||||
StatusCode::BAD_REQUEST
|
||||
} else if matches!(error, ControlError::Timeout) {
|
||||
StatusCode::GATEWAY_TIMEOUT
|
||||
} else {
|
||||
tracing::error!(error = %error, "control failed");
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
};
|
||||
ControlFailure(status, error.to_string())
|
||||
}
|
||||
|
||||
async fn controller_health(
|
||||
State(app): State<App>,
|
||||
headers: HeaderMap,
|
||||
) -> Result<Json<serde_json::Value>, ControlFailure> {
|
||||
if !authorized(&headers, &app.token) {
|
||||
return Err(StatusCode::UNAUTHORIZED.into());
|
||||
}
|
||||
let ctx = ControlContext::new(display_of(&headers, None), None);
|
||||
let health = app
|
||||
.controller
|
||||
.health(&ctx)
|
||||
.await
|
||||
.map_err(|error| status_for(&error))?;
|
||||
Ok(Json(serde_json::json!({
|
||||
"backend": health.backend,
|
||||
"version": health.version,
|
||||
"healthy": health.healthy,
|
||||
"degraded": health.degraded,
|
||||
"details": health.details,
|
||||
})))
|
||||
}
|
||||
|
||||
async fn observe(
|
||||
State(app): State<App>,
|
||||
headers: HeaderMap,
|
||||
) -> Result<Json<serde_json::Value>, StatusCode> {
|
||||
) -> Result<Json<serde_json::Value>, ControlFailure> {
|
||||
if !authorized(&headers, &app.token) {
|
||||
return Err(StatusCode::UNAUTHORIZED);
|
||||
return Err(StatusCode::UNAUTHORIZED.into());
|
||||
}
|
||||
let ctx = ControlContext::new(display_of(&headers, None), None);
|
||||
match app.controller.observe(&ctx).await {
|
||||
Ok(observation) => Ok(Json(observation_to_control_json(&observation))),
|
||||
Err(error) => Err(status_for(&error)),
|
||||
}
|
||||
let display = display_of(&headers, None);
|
||||
let png = run_capture(&display)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
Ok(Json(observation_json(&display, png).await))
|
||||
}
|
||||
|
||||
async fn act(
|
||||
State(app): State<App>,
|
||||
headers: HeaderMap,
|
||||
Json(request): Json<ActionRequest>,
|
||||
) -> Result<Json<serde_json::Value>, StatusCode> {
|
||||
) -> Result<Json<serde_json::Value>, ControlFailure> {
|
||||
if !authorized(&headers, &app.token) {
|
||||
return Err(StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
let display = display_of(&headers, request.display.as_deref());
|
||||
let profile = profile_of(&headers, request.profile_path.as_deref());
|
||||
let mut completed = 0usize;
|
||||
for action in &request.actions {
|
||||
apply_action(&display, profile.as_deref(), action)
|
||||
.await
|
||||
.map_err(|_| StatusCode::BAD_REQUEST)?;
|
||||
let pause = action_pause_ms(action);
|
||||
if pause > 0 {
|
||||
sleep(Duration::from_millis(pause)).await;
|
||||
}
|
||||
completed += 1;
|
||||
}
|
||||
if request.settle_ms > 0 {
|
||||
sleep(Duration::from_millis(request.settle_ms as u64)).await;
|
||||
}
|
||||
let mut body = serde_json::json!({ "completed": completed });
|
||||
if request.observe {
|
||||
let png = run_capture(&display)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
if let serde_json::Value::Object(map) = observation_json(&display, png).await {
|
||||
body.as_object_mut().unwrap().extend(map);
|
||||
return Err(StatusCode::UNAUTHORIZED.into());
|
||||
}
|
||||
let ctx = ControlContext::new(
|
||||
display_of(&headers, request.display.as_deref()),
|
||||
profile_of(&headers, request.profile_path.as_deref()),
|
||||
);
|
||||
match app.controller.act(&request, &ctx).await {
|
||||
Ok(result) => {
|
||||
let mut body = serde_json::json!({ "completed": result.completed, "clipboardText": result.clipboard_text });
|
||||
if let Some(observation) = result.observation
|
||||
&& let serde_json::Value::Object(map) = observation_to_control_json(&observation)
|
||||
{
|
||||
body.as_object_mut().expect("object").extend(map);
|
||||
}
|
||||
Ok(Json(body))
|
||||
}
|
||||
Err(error) => Err(status_for(&error)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn apply_action(
|
||||
display: &str,
|
||||
profile: Option<&str>,
|
||||
action: &ComputerAction,
|
||||
) -> Result<(), String> {
|
||||
match action {
|
||||
ComputerAction::Wait { ms } => {
|
||||
sleep(Duration::from_millis(*ms as u64)).await;
|
||||
Ok(())
|
||||
async fn browser(
|
||||
State(app): State<App>,
|
||||
headers: HeaderMap,
|
||||
Json(request): Json<BrowserRequest>,
|
||||
) -> Result<Json<serde_json::Value>, ControlFailure> {
|
||||
if !authorized(&headers, &app.token) {
|
||||
return Err(StatusCode::UNAUTHORIZED.into());
|
||||
}
|
||||
ComputerAction::Open { path } => {
|
||||
spawn_detached(&open_argv_on(display, profile, path)).await
|
||||
let ctx = ControlContext::new(
|
||||
display_of(&headers, request.display.as_deref()),
|
||||
profile_of(&headers, request.profile_path.as_deref()),
|
||||
);
|
||||
match app.controller.browser(&request, &ctx).await {
|
||||
Ok(page) => Ok(Json(
|
||||
serde_json::to_value(&page).unwrap_or_else(|_| serde_json::json!({"ok": false})),
|
||||
)),
|
||||
Err(error) => Err(status_for(&error)),
|
||||
}
|
||||
ComputerAction::Focus { .. } => {
|
||||
let argv =
|
||||
xdotool_argv_on(display, action).ok_or_else(|| "unsupported action".to_string())?;
|
||||
let output = Command::new(&argv[0])
|
||||
.args(&argv[1..])
|
||||
.output()
|
||||
}
|
||||
|
||||
fn recording_ctx(headers: &HeaderMap, request: &RecordingRequest) -> ControlContext {
|
||||
ControlContext::new(
|
||||
display_of(headers, request.display.as_deref()),
|
||||
profile_of(headers, request.profile_path.as_deref()),
|
||||
)
|
||||
}
|
||||
|
||||
async fn recording_start(
|
||||
State(app): State<App>,
|
||||
headers: HeaderMap,
|
||||
Json(request): Json<RecordingRequest>,
|
||||
) -> Result<Json<serde_json::Value>, ControlFailure> {
|
||||
if !authorized(&headers, &app.token) {
|
||||
return Err(StatusCode::UNAUTHORIZED.into());
|
||||
}
|
||||
match app
|
||||
.controller
|
||||
.start_recording(&request, &recording_ctx(&headers, &request))
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
if output.status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(String::from_utf8_lossy(&output.stderr).into_owned())
|
||||
{
|
||||
Ok(session) => Ok(Json(
|
||||
serde_json::to_value(&session).unwrap_or_else(|_| serde_json::json!({"ok": false})),
|
||||
)),
|
||||
Err(error) => Err(status_for(&error)),
|
||||
}
|
||||
}
|
||||
ComputerAction::Launch { application, uri } => {
|
||||
let argv = launch_argv_on(display, profile, application, uri.as_deref())
|
||||
.ok_or_else(|| "unknown application".to_string())?;
|
||||
spawn_detached(&argv).await
|
||||
|
||||
async fn recording_stop(
|
||||
State(app): State<App>,
|
||||
headers: HeaderMap,
|
||||
Json(request): Json<RecordingRequest>,
|
||||
) -> Result<Json<serde_json::Value>, ControlFailure> {
|
||||
if !authorized(&headers, &app.token) {
|
||||
return Err(StatusCode::UNAUTHORIZED.into());
|
||||
}
|
||||
ComputerAction::Ref {
|
||||
verb,
|
||||
target,
|
||||
ref_kind,
|
||||
text,
|
||||
} => apply_ref(display, profile, *verb, target, ref_kind, text.as_deref()).await,
|
||||
other => {
|
||||
let argv =
|
||||
xdotool_argv_on(display, other).ok_or_else(|| "unsupported action".to_string())?;
|
||||
let output = Command::new(&argv[0])
|
||||
.args(&argv[1..])
|
||||
.output()
|
||||
match app
|
||||
.controller
|
||||
.stop_recording(&request, &recording_ctx(&headers, &request))
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
if output.status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(String::from_utf8_lossy(&output.stderr).into_owned())
|
||||
}
|
||||
}
|
||||
{
|
||||
Ok(()) => Ok(Json(serde_json::json!({ "ok": true }))),
|
||||
Err(error) => Err(status_for(&error)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn apply_ref(
|
||||
display: &str,
|
||||
profile: Option<&str>,
|
||||
verb: RefVerb,
|
||||
target: &str,
|
||||
kind: &str,
|
||||
text: Option<&str>,
|
||||
) -> Result<(), String> {
|
||||
let action = match verb {
|
||||
RefVerb::Click => "click",
|
||||
RefVerb::SetValue => "type",
|
||||
RefVerb::Focus => "focus",
|
||||
};
|
||||
let mut request = serde_json::json!({
|
||||
"action": action,
|
||||
"selector": target,
|
||||
"display": display,
|
||||
"ensure": false,
|
||||
});
|
||||
if let Some(text) = text {
|
||||
request["text"] = serde_json::json!(text);
|
||||
async fn recording_collect(
|
||||
State(app): State<App>,
|
||||
headers: HeaderMap,
|
||||
Json(request): Json<RecordingRequest>,
|
||||
) -> Result<Json<serde_json::Value>, ControlFailure> {
|
||||
if !authorized(&headers, &app.token) {
|
||||
return Err(StatusCode::UNAUTHORIZED.into());
|
||||
}
|
||||
let argv = if kind == "dom" {
|
||||
cdp_command_on(display, profile, &request)
|
||||
} else {
|
||||
a11y_command_on(display, &request)
|
||||
};
|
||||
let output = Command::new(&argv[0])
|
||||
.args(&argv[1..])
|
||||
.output()
|
||||
match app
|
||||
.controller
|
||||
.collect_recording(&request, &recording_ctx(&headers, &request))
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let raw = if output.stdout.is_empty() {
|
||||
String::from_utf8_lossy(&output.stderr).into_owned()
|
||||
} else {
|
||||
String::from_utf8_lossy(&output.stdout).into_owned()
|
||||
};
|
||||
let ok = if kind == "dom" {
|
||||
parse_cdp_page(&raw).ok
|
||||
} else {
|
||||
parse_a11y_page(&raw).ok
|
||||
};
|
||||
if ok { Ok(()) } else { Err(raw) }
|
||||
{
|
||||
Ok(result) => Ok(Json(
|
||||
serde_json::to_value(&result).unwrap_or_else(|_| serde_json::json!({"events": []})),
|
||||
)),
|
||||
Err(error) => Err(status_for(&error)),
|
||||
}
|
||||
|
||||
async fn spawn_detached(argv: &[String]) -> Result<(), String> {
|
||||
Command::new(&argv[0])
|
||||
.args(&argv[1..])
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.map_err(|error| error.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn observation_json(display: &str, png: Vec<u8>) -> serde_json::Value {
|
||||
use base64::Engine;
|
||||
let mut body = serde_json::json!({
|
||||
"png_base64": base64::engine::general_purpose::STANDARD.encode(png)
|
||||
});
|
||||
let ((cursor, window), elements) =
|
||||
tokio::join!(run_pointer_state(display), run_window_list(display));
|
||||
if let Some(cursor) = cursor {
|
||||
body["cursor"] = serde_json::json!({ "x": cursor.x, "y": cursor.y });
|
||||
}
|
||||
if let Some(window) = window {
|
||||
body["activeWindow"] = serde_json::json!({ "id": window.id, "title": window.title });
|
||||
}
|
||||
|
||||
if !elements.is_empty() {
|
||||
body["elements"] = serde_json::to_value(elements).unwrap_or(serde_json::json!([]));
|
||||
}
|
||||
body
|
||||
}
|
||||
|
||||
async fn run_window_list(display: &str) -> Vec<lazyboy_contracts::UiElement> {
|
||||
let argv = window_list_command_on(display);
|
||||
let output = Command::new(&argv[0]).args(&argv[1..]).output().await.ok();
|
||||
let Some(output) = output else {
|
||||
return Vec::new();
|
||||
};
|
||||
parse_ui_elements(&String::from_utf8_lossy(&output.stdout))
|
||||
}
|
||||
|
||||
async fn run_pointer_state(
|
||||
display: &str,
|
||||
) -> (
|
||||
Option<lazyboy_contracts::CursorPosition>,
|
||||
Option<lazyboy_contracts::ActiveWindow>,
|
||||
) {
|
||||
let argv = pointer_state_command_on(display);
|
||||
let output = Command::new(&argv[0]).args(&argv[1..]).output().await.ok();
|
||||
let Some(output) = output else {
|
||||
return (None, None);
|
||||
};
|
||||
parse_pointer_state(&String::from_utf8_lossy(&output.stdout))
|
||||
}
|
||||
|
||||
async fn run_capture(display: &str) -> Result<Vec<u8>, String> {
|
||||
let argv = screenshot_command_on(display);
|
||||
let mut child = Command::new(&argv[0])
|
||||
.args(&argv[1..])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.map_err(|error| error.to_string())?;
|
||||
let mut stdout = Vec::new();
|
||||
if let Some(mut pipe) = child.stdout.take() {
|
||||
pipe.read_to_end(&mut stdout)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
let status = child.wait().await.map_err(|error| error.to_string())?;
|
||||
if !status.success() || stdout.is_empty() {
|
||||
return Err("screenshot failed".into());
|
||||
}
|
||||
Ok(stdout)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,12 +2,12 @@
|
|||
name = "lazyboy-harness"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
publish.workspace = true
|
||||
|
||||
[dependencies]
|
||||
lazyboy-contracts.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
rig-core.workspace = true
|
||||
|
|
@ -21,3 +21,6 @@ http = "1"
|
|||
reqwest.workspace = true
|
||||
rustls.workspace = true
|
||||
rustls-native-certs = "0.8"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
|
|
|||
|
|
@ -28,7 +28,12 @@ impl ExecutionMode {
|
|||
|
||||
/// Only a standalone terminal marker is an outcome, not a quoted mention.
|
||||
pub fn goal_outcome(reply: &str) -> GoalOutcome {
|
||||
match reply.trim().lines().last().map(str::trim) {
|
||||
let reply = reply.trim();
|
||||
// A bare marker provides neither verification nor a reason to the human.
|
||||
if reply.lines().count() < 2 {
|
||||
return GoalOutcome::Continue;
|
||||
}
|
||||
match reply.lines().last().map(str::trim) {
|
||||
Some("[GOAL_COMPLETE]") => GoalOutcome::Complete,
|
||||
Some("[GOAL_BLOCKED]") => GoalOutcome::NeedsInput,
|
||||
_ => GoalOutcome::Continue,
|
||||
|
|
@ -38,10 +43,139 @@ pub fn goal_outcome(reply: &str) -> GoalOutcome {
|
|||
pub const GOAL_INSTRUCTIONS: &str = "Persistent goal execution: plan the requested work, execute it, and verify each requested outcome. Intermediate progress replies do not finish the run. Preserve completed work and incorporate user steering. End your final reply with a standalone [GOAL_COMPLETE] line only when all outcomes are verified; explain the verification. When required information or human action is missing, explain exactly what is needed and end with a standalone [GOAL_BLOCKED] line. For a simple Cloudflare connection-check checkbox, observe the current screen and try connection_check once, then verify the requested content. For other CAPTCHA, failed verification, login or 2FA use request_takeover. Never claim completion merely because you planned the work.";
|
||||
pub const GOAL_CONTINUE: &str = "The goal remains active. Continue the plan with tools and verify the outcome. Finish only with a standalone [GOAL_COMPLETE] line after verification, or [GOAL_BLOCKED] when required human input is missing.";
|
||||
|
||||
/// A run that stops in the middle must say so instead of going quiet. The
|
||||
/// model marks the moment; the loop turns the marker into a paused run the
|
||||
/// human can continue with one click.
|
||||
pub const NEEDS_INPUT_MARKER: &str = "[NEEDS_INPUT]";
|
||||
|
||||
/// Turn budgets for self-correction. A goal run is expected to fight through
|
||||
/// obstacles, so it gets more attempts than a plain task.
|
||||
pub const MAX_NUDGES_PLAIN: u32 = 6;
|
||||
pub const MAX_NUDGES_GOAL: u32 = 8;
|
||||
|
||||
/// Sent once per stop attempt: the cheapest way to tell "the work is done" from
|
||||
/// "the model just ran out of sentences".
|
||||
pub const VERIFY_BEFORE_DONE: &str = "Before you finish, verify the result against the CURRENT screen or file contents: say what you checked and what you still owe. If any requested outcome is missing, act on it now with a tool call. If the human must decide, supply something, or do a step you cannot do, say what you did so far and what the next step is, then end with a standalone [NEEDS_INPUT] line. Never end a half-finished task with a plain status sentence.";
|
||||
|
||||
/// Why the loop is handing the turn back to the human.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum StopReason {
|
||||
/// The model stopped mid-task and asked the human to take it from here.
|
||||
MidTaskText,
|
||||
/// The bounded turn budget is spent with work still outstanding.
|
||||
BudgetExhausted,
|
||||
/// The round policy saw the run repeating itself rather than moving. The
|
||||
/// difference from `BudgetExhausted` matters to the human: this is "it got
|
||||
/// stuck", not "it ran out of a quota", and the pause message carries the
|
||||
/// action it kept repeating.
|
||||
LoopDetected,
|
||||
}
|
||||
|
||||
impl StopReason {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::MidTaskText => "mid_task_text",
|
||||
Self::BudgetExhausted => "budget_exhausted",
|
||||
Self::LoopDetected => "loop_detected",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Only a standalone marker line asks for input, never a quoted mention.
|
||||
pub fn asks_for_input(reply: &str) -> bool {
|
||||
reply
|
||||
.trim()
|
||||
.lines()
|
||||
.last()
|
||||
.is_some_and(|line| line.trim() == NEEDS_INPUT_MARKER)
|
||||
}
|
||||
|
||||
/// A run that never touched a tool answered in prose, which is a complete
|
||||
/// reply. Once a run has done work, ending is a decision the human gets to
|
||||
/// make: report where the work stands, then ask before stopping.
|
||||
pub fn stop_reason(
|
||||
mode: ExecutionMode,
|
||||
turns: u32,
|
||||
reply: &str,
|
||||
did_work: bool,
|
||||
stalled: bool,
|
||||
) -> Option<StopReason> {
|
||||
if !did_work {
|
||||
return None;
|
||||
}
|
||||
if !mode.allows_turn(turns.saturating_add(1)) {
|
||||
return Some(StopReason::BudgetExhausted);
|
||||
}
|
||||
(stalled || asks_for_input(reply)).then_some(StopReason::MidTaskText)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_prose_answer_is_not_a_stop() {
|
||||
assert_eq!(
|
||||
stop_reason(ExecutionMode::Bounded(40), 3, "done", false, false),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_standalone_marker_asks_for_input() {
|
||||
assert!(asks_for_input("我做到一半。\n[NEEDS_INPUT]"));
|
||||
assert!(!asks_for_input("我不会用 [NEEDS_INPUT] 这种标记"));
|
||||
assert!(!asks_for_input("任务完成"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stalled_or_asking_run_hands_the_turn_to_the_human() {
|
||||
let mode = ExecutionMode::Bounded(40);
|
||||
assert_eq!(
|
||||
stop_reason(mode, 3, "需要密码\n[NEEDS_INPUT]", true, false),
|
||||
Some(StopReason::MidTaskText)
|
||||
);
|
||||
assert_eq!(
|
||||
stop_reason(mode, 3, "我先跳到下一步", true, true),
|
||||
Some(StopReason::MidTaskText)
|
||||
);
|
||||
assert_eq!(stop_reason(mode, 3, "看起来好了", true, false), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_spent_budget_reports_before_a_mid_task_stop() {
|
||||
assert_eq!(
|
||||
stop_reason(ExecutionMode::Bounded(40), 40, "", true, false),
|
||||
Some(StopReason::BudgetExhausted)
|
||||
);
|
||||
assert_eq!(
|
||||
stop_reason(
|
||||
ExecutionMode::Bounded(40),
|
||||
40,
|
||||
"需要密码\n[NEEDS_INPUT]",
|
||||
true,
|
||||
false
|
||||
),
|
||||
Some(StopReason::BudgetExhausted)
|
||||
);
|
||||
// A goal run that left the loop on its own terms reported its outcome;
|
||||
// the after-loop guard must not invent a second stop for it.
|
||||
assert_eq!(
|
||||
stop_reason(ExecutionMode::Goal, 4000, "", true, false),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
stop_reason(ExecutionMode::Goal, 4000, "我先跳过这一步", true, true),
|
||||
Some(StopReason::MidTaskText)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn goal_runs_get_more_self_correction_than_plain_runs() {
|
||||
assert_eq!(MAX_NUDGES_GOAL, 8);
|
||||
assert_eq!(MAX_NUDGES_PLAIN, 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn goals_are_unbounded_while_normal_runs_remain_bounded() {
|
||||
assert!(ExecutionMode::Goal.allows_turn(40));
|
||||
|
|
@ -57,8 +191,19 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn progress_and_quoted_markers_do_not_complete_a_goal() {
|
||||
assert_eq!(goal_outcome("Next I will use [GOAL_COMPLETE]."), GoalOutcome::Continue);
|
||||
assert_eq!(goal_outcome("Verified output.\n[GOAL_COMPLETE]"), GoalOutcome::Complete);
|
||||
assert_eq!(goal_outcome("Please supply the date.\n[GOAL_BLOCKED]"), GoalOutcome::NeedsInput);
|
||||
assert_eq!(goal_outcome("[GOAL_COMPLETE]"), GoalOutcome::Continue);
|
||||
assert_eq!(goal_outcome("\n[GOAL_BLOCKED]"), GoalOutcome::Continue);
|
||||
assert_eq!(
|
||||
goal_outcome("Next I will use [GOAL_COMPLETE]."),
|
||||
GoalOutcome::Continue
|
||||
);
|
||||
assert_eq!(
|
||||
goal_outcome("Verified output.\n[GOAL_COMPLETE]"),
|
||||
GoalOutcome::Complete
|
||||
);
|
||||
assert_eq!(
|
||||
goal_outcome("Please supply the date.\n[GOAL_BLOCKED]"),
|
||||
GoalOutcome::NeedsInput
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
mod resolve;
|
||||
pub mod execution;
|
||||
pub mod policy;
|
||||
mod resolve;
|
||||
mod voice;
|
||||
|
||||
pub use resolve::*;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,764 @@
|
|||
//! Round policy: how long a run may keep going, and when it must stop.
|
||||
//!
|
||||
//! Earlier every run got a fixed number of turns. A fixed quota punishes real
|
||||
//! work - a build, a 200-row sheet, a site that reloads slowly - and it teaches
|
||||
//! the agent to schedule its honesty around turn 39. Here turns are not the
|
||||
//! budget; evidence is. A run keeps going until the work is verified, the human
|
||||
//! says stop, or the guard sees the agent going in circles instead of moving.
|
||||
//! Only then, and with the evidence in hand, does the run park.
|
||||
//!
|
||||
//! Three layers, cheapest first:
|
||||
//! 1. Coaching (`Reflect`): the loop injects one pointed instruction and the run
|
||||
//! continues. Triggers are the same action again, the same failure again, no
|
||||
//! new success for a long time, or a soft checkpoint.
|
||||
//! 2. Derailment halt (`LoopDetected`): the same action keeps being repeated, or
|
||||
//! nothing has worked for a very long time. The run parks in `waiting_input`
|
||||
//! with the concrete evidence so the human can unblock it.
|
||||
//! 3. Circuit breaker (`BudgetExhausted`): a large turn and wall-clock ceiling
|
||||
//! that exists only so a bug cannot burn an API key overnight. It is not a
|
||||
//! task budget; reaching it is a bug report, not a result.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::execution::StopReason;
|
||||
|
||||
/// Turns before the first "where are you?" self-check. Generous on purpose:
|
||||
/// the first stretch of a hard task is normal, not suspicious.
|
||||
pub const SOFT_CHECKPOINT_TURNS: u32 = 60;
|
||||
/// Turns between later self-checks.
|
||||
pub const SOFT_CHECKPOINT_EVERY: u32 = 120;
|
||||
/// Circuit breaker. No honest task is 1000 model turns long; if it is, it
|
||||
/// belongs in a skill or a schedule, not in one run.
|
||||
pub const HARD_CAP_TURNS: u32 = 1_000;
|
||||
/// Wall clock before a run is asked to justify itself out loud.
|
||||
pub const SOFT_WALL_MINUTES: u64 = 75;
|
||||
/// Wall clock circuit breaker.
|
||||
pub const HARD_WALL_MINUTES: u64 = 240;
|
||||
|
||||
/// Same mutating action, in a row, before the run is coached.
|
||||
pub const REPEAT_WARN: u32 = 3;
|
||||
/// Same mutating action, in a row, before the run parks.
|
||||
pub const REPEAT_HALT: u32 = 6;
|
||||
/// Same mutating action, counted over the whole run.
|
||||
pub const REPEAT_HALT_TOTAL: u32 = 12;
|
||||
/// Same action failing, in a row.
|
||||
pub const FAILURE_WARN: u32 = 3;
|
||||
pub const FAILURE_HALT: u32 = 8;
|
||||
/// Anything failing, in a row, whatever the tool.
|
||||
pub const FAILURE_HALT_ANY: u32 = 14;
|
||||
/// Turns without a new success before a self-check.
|
||||
pub const STALE_REFLECT: u32 = 40;
|
||||
/// Turns without a new success before parking the run.
|
||||
pub const STALE_HALT: u32 = 150;
|
||||
/// Coaching has to stay rare enough that the model actually reads it.
|
||||
pub const MAX_REFLECTIONS: u32 = 8;
|
||||
|
||||
/// A run that repeats itself is not a run that waits or polls, so these tools
|
||||
/// reset the no-progress clock even when their arguments repeat.
|
||||
const PROGRESS_WHEN_REPEATED: [&str; 2] = ["shell", "wait"];
|
||||
|
||||
fn parse_u64(value: Option<String>) -> Option<u64> {
|
||||
value?.trim().parse::<u64>().ok().filter(|value| *value > 0)
|
||||
}
|
||||
|
||||
fn parse_u32(value: Option<String>) -> Option<u32> {
|
||||
parse_u64(value).and_then(|value| u32::try_from(value).ok())
|
||||
}
|
||||
|
||||
/// The knobs a deployment may want to move. Every one of them is a breaker,
|
||||
/// not a quota: the defaults sit far outside honest work.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct RunPolicy {
|
||||
pub soft_checkpoint_turns: u32,
|
||||
pub soft_checkpoint_every: u32,
|
||||
pub hard_cap_turns: u32,
|
||||
pub soft_wall: Duration,
|
||||
pub hard_wall: Duration,
|
||||
}
|
||||
|
||||
impl Default for RunPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
soft_checkpoint_turns: SOFT_CHECKPOINT_TURNS,
|
||||
soft_checkpoint_every: SOFT_CHECKPOINT_EVERY,
|
||||
hard_cap_turns: HARD_CAP_TURNS,
|
||||
soft_wall: Duration::from_secs(SOFT_WALL_MINUTES * 60),
|
||||
hard_wall: Duration::from_secs(HARD_WALL_MINUTES * 60),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RunPolicy {
|
||||
/// `LAZYBOY_RUN_SOFT_TURNS`, `LAZYBOY_RUN_SOFT_EVERY`,
|
||||
/// `LAZYBOY_RUN_CAP_TURNS`, `LAZYBOY_RUN_SOFT_MINUTES`,
|
||||
/// `LAZYBOY_RUN_HARD_MINUTES`. Unset or unparsable keeps the default.
|
||||
pub fn from_env() -> Self {
|
||||
Self::from_lookup(|name| std::env::var(name).ok())
|
||||
}
|
||||
|
||||
pub fn from_lookup(lookup: impl Fn(&str) -> Option<String>) -> Self {
|
||||
let mut policy = Self::default();
|
||||
if let Some(value) = parse_u32(lookup("LAZYBOY_RUN_SOFT_TURNS")) {
|
||||
policy.soft_checkpoint_turns = value;
|
||||
}
|
||||
if let Some(value) = parse_u32(lookup("LAZYBOY_RUN_SOFT_EVERY")) {
|
||||
policy.soft_checkpoint_every = value;
|
||||
}
|
||||
if let Some(value) = parse_u32(lookup("LAZYBOY_RUN_CAP_TURNS")) {
|
||||
policy.hard_cap_turns = value;
|
||||
}
|
||||
if let Some(value) = parse_u64(lookup("LAZYBOY_RUN_SOFT_MINUTES")) {
|
||||
policy.soft_wall = Duration::from_secs(value * 60);
|
||||
}
|
||||
if let Some(value) = parse_u64(lookup("LAZYBOY_RUN_HARD_MINUTES")) {
|
||||
policy.hard_wall = Duration::from_secs(value * 60);
|
||||
}
|
||||
policy
|
||||
}
|
||||
}
|
||||
|
||||
/// What the loop should do with a verdict.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Verdict {
|
||||
Continue,
|
||||
/// Inject one instruction into the next model turn and keep running.
|
||||
Reflect(String),
|
||||
/// Park the run and hand the reason, plus the evidence, to the human.
|
||||
Halt {
|
||||
reason: StopReason,
|
||||
note: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl Verdict {
|
||||
pub fn is_halt(&self) -> bool {
|
||||
matches!(self, Self::Halt { .. })
|
||||
}
|
||||
}
|
||||
|
||||
/// One finished tool call, as far as the policy is concerned.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct ActionObserved<'a> {
|
||||
pub name: &'a str,
|
||||
pub args: &'a Value,
|
||||
/// Human-readable rendering, reused verbatim in the pause message.
|
||||
pub label: &'a str,
|
||||
pub ok: bool,
|
||||
pub turn: u32,
|
||||
/// False for reads and observations: repeating those is how an agent looks
|
||||
/// at a page again, not how it gets stuck clicking.
|
||||
pub changes_state: bool,
|
||||
}
|
||||
|
||||
/// Key that identifies "the same action". Object keys are sorted so `{a,b}` and
|
||||
/// `{b,a}` match, while values are kept because `click #12` and `click #13` are
|
||||
/// genuinely different actions.
|
||||
fn signature(name: &str, args: &Value) -> String {
|
||||
let mut out = String::with_capacity(name.len() + 32);
|
||||
out.push_str(name);
|
||||
out.push('(');
|
||||
write_canonical(args, &mut out);
|
||||
out.push(')');
|
||||
const MAX_KEY_CHARS: usize = 400;
|
||||
let chars: Vec<char> = out.chars().collect();
|
||||
if chars.len() > MAX_KEY_CHARS {
|
||||
out = chars.iter().take(MAX_KEY_CHARS).collect();
|
||||
out.push('…');
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn write_canonical(value: &Value, out: &mut String) {
|
||||
match value {
|
||||
Value::Object(map) => {
|
||||
let mut keys: Vec<&String> = map.keys().collect();
|
||||
keys.sort();
|
||||
out.push('{');
|
||||
for (index, key) in keys.iter().enumerate() {
|
||||
if index > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
out.push_str(key);
|
||||
out.push(':');
|
||||
write_canonical(&map[*key], out);
|
||||
}
|
||||
out.push('}');
|
||||
}
|
||||
Value::Array(items) => {
|
||||
out.push('[');
|
||||
for (index, item) in items.iter().enumerate() {
|
||||
if index > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
write_canonical(item, out);
|
||||
}
|
||||
out.push(']');
|
||||
}
|
||||
other => out.push_str(other.to_string().as_str()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-run derailment detector. A resumed run starts with a clean slate, because
|
||||
/// the human has just told it to continue.
|
||||
pub struct LoopGuard {
|
||||
policy: RunPolicy,
|
||||
counts: HashMap<String, u32>,
|
||||
failures: HashMap<String, u32>,
|
||||
last_key: Option<String>,
|
||||
repeat_streak: u32,
|
||||
any_failure_streak: u32,
|
||||
warned_repeat: bool,
|
||||
warned_failure: bool,
|
||||
watched_since: u32,
|
||||
/// Set by the first successful tool call. Until a run has actually done
|
||||
/// something, "no progress" is meaningless: turns spent talking or thinking
|
||||
/// are handled by the nudge limit, not by this guard.
|
||||
acted: bool,
|
||||
last_progress_turn: u32,
|
||||
last_stale_reflect_turn: u32,
|
||||
checkpoint_turn: u32,
|
||||
soft_wall_spoken: bool,
|
||||
reflections: u32,
|
||||
}
|
||||
|
||||
impl LoopGuard {
|
||||
pub fn new(policy: RunPolicy) -> Self {
|
||||
Self::with_watch(policy, 0)
|
||||
}
|
||||
|
||||
/// `watched_from` is the turn number the guard starts counting at, so a
|
||||
/// resumed run is not judged for work it already finished.
|
||||
pub fn with_watch(policy: RunPolicy, watched_from: u32) -> Self {
|
||||
Self {
|
||||
policy,
|
||||
counts: HashMap::new(),
|
||||
failures: HashMap::new(),
|
||||
last_key: None,
|
||||
repeat_streak: 0,
|
||||
any_failure_streak: 0,
|
||||
warned_repeat: false,
|
||||
warned_failure: false,
|
||||
watched_since: watched_from,
|
||||
acted: false,
|
||||
last_progress_turn: watched_from,
|
||||
last_stale_reflect_turn: watched_from,
|
||||
checkpoint_turn: 0,
|
||||
soft_wall_spoken: false,
|
||||
reflections: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn policy(&self) -> RunPolicy {
|
||||
self.policy
|
||||
}
|
||||
|
||||
/// Called once per model turn, before the model is asked for anything.
|
||||
pub fn on_turn(&mut self, turns: u32, elapsed: Duration) -> Verdict {
|
||||
if turns >= self.policy.hard_cap_turns {
|
||||
return Verdict::Halt {
|
||||
reason: StopReason::BudgetExhausted,
|
||||
note: format!(
|
||||
"已達到保險上限 {cap} 輪（這是迴圈失控的保護，不是任務做完）",
|
||||
cap = self.policy.hard_cap_turns
|
||||
),
|
||||
};
|
||||
}
|
||||
let minutes = elapsed.as_secs() / 60;
|
||||
if elapsed >= self.policy.hard_wall {
|
||||
return Verdict::Halt {
|
||||
reason: StopReason::BudgetExhausted,
|
||||
note: format!(
|
||||
"已執行 {minutes} 分鐘，超過 {hard} 分鐘的最後保護上限",
|
||||
hard = self.policy.hard_wall.as_secs() / 60
|
||||
),
|
||||
};
|
||||
}
|
||||
if let Some(halt) = self.stale_check(turns) {
|
||||
return halt;
|
||||
}
|
||||
if self.is_checkpoint(turns) {
|
||||
self.checkpoint_turn = turns;
|
||||
return self.reflect(checkpoint_text(turns, minutes));
|
||||
}
|
||||
if !self.soft_wall_spoken && elapsed >= self.policy.soft_wall {
|
||||
self.soft_wall_spoken = true;
|
||||
return self.reflect(wall_text(minutes, self.policy.hard_wall.as_secs() / 60));
|
||||
}
|
||||
Verdict::Continue
|
||||
}
|
||||
|
||||
/// Called after every tool result.
|
||||
pub fn on_action(&mut self, action: &ActionObserved<'_>) -> Verdict {
|
||||
let key = signature(action.name, action.args);
|
||||
if self.last_key.as_deref() == Some(key.as_str()) {
|
||||
self.repeat_streak = self.repeat_streak.saturating_add(1);
|
||||
} else {
|
||||
self.repeat_streak = 1;
|
||||
self.warned_repeat = false;
|
||||
}
|
||||
let seen = self
|
||||
.counts
|
||||
.get(&key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
.saturating_add(1);
|
||||
self.counts.insert(key.clone(), seen);
|
||||
self.last_key = Some(key.clone());
|
||||
|
||||
if !action.ok {
|
||||
self.any_failure_streak = self.any_failure_streak.saturating_add(1);
|
||||
let failures = self
|
||||
.failures
|
||||
.get(&key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
.saturating_add(1);
|
||||
self.failures.insert(key.clone(), failures);
|
||||
if failures >= FAILURE_HALT {
|
||||
return Verdict::Halt {
|
||||
reason: StopReason::LoopDetected,
|
||||
note: format!(
|
||||
"同一個動作「{label}」連續失敗 {failures} 次",
|
||||
label = action.label
|
||||
),
|
||||
};
|
||||
}
|
||||
if self.any_failure_streak >= FAILURE_HALT_ANY {
|
||||
return Verdict::Halt {
|
||||
reason: StopReason::LoopDetected,
|
||||
note: format!(
|
||||
"連續 {count} 個動作都沒有成功（最後一個：{label}）",
|
||||
count = self.any_failure_streak,
|
||||
label = action.label,
|
||||
),
|
||||
};
|
||||
}
|
||||
if failures >= FAILURE_WARN && !self.warned_failure {
|
||||
self.warned_failure = true;
|
||||
return self.reflect(failure_text(action.label, failures));
|
||||
}
|
||||
return Verdict::Continue;
|
||||
}
|
||||
|
||||
self.any_failure_streak = 0;
|
||||
self.failures.remove(&key);
|
||||
self.warned_failure = false;
|
||||
self.acted = true;
|
||||
if seen == 1 || PROGRESS_WHEN_REPEATED.contains(&action.name) {
|
||||
self.last_progress_turn = action.turn;
|
||||
}
|
||||
|
||||
if action.changes_state {
|
||||
if self.repeat_streak >= REPEAT_HALT {
|
||||
return Verdict::Halt {
|
||||
reason: StopReason::LoopDetected,
|
||||
note: format!(
|
||||
"同一個動作「{label}」連續做了 {streak} 次，沒有新的進展",
|
||||
label = action.label,
|
||||
streak = self.repeat_streak,
|
||||
),
|
||||
};
|
||||
}
|
||||
if seen >= REPEAT_HALT_TOTAL {
|
||||
return Verdict::Halt {
|
||||
reason: StopReason::LoopDetected,
|
||||
note: format!(
|
||||
"同一個動作「{label}」在這輪任務裡已經做了 {seen} 次",
|
||||
label = action.label,
|
||||
),
|
||||
};
|
||||
}
|
||||
if self.repeat_streak >= REPEAT_WARN && !self.warned_repeat {
|
||||
self.warned_repeat = true;
|
||||
return self.reflect(repeat_text(action.label, self.repeat_streak));
|
||||
}
|
||||
}
|
||||
Verdict::Continue
|
||||
}
|
||||
|
||||
/// Nothing new has succeeded for a long stretch: either the agent is
|
||||
/// circling the same wall, or it is honestly waiting and should say so.
|
||||
fn stale_check(&mut self, turns: u32) -> Option<Verdict> {
|
||||
if !self.acted {
|
||||
return None;
|
||||
}
|
||||
let since = turns.saturating_sub(self.last_progress_turn);
|
||||
if since < STALE_REFLECT {
|
||||
return None;
|
||||
}
|
||||
if since >= STALE_HALT {
|
||||
return Some(Verdict::Halt {
|
||||
reason: StopReason::LoopDetected,
|
||||
note: format!(
|
||||
"從第 {watched} 輪起，連續 {since} 輪沒有任何新的成功動作，看起來在鬼打牆",
|
||||
watched = self.watched_since
|
||||
),
|
||||
});
|
||||
}
|
||||
if turns.saturating_sub(self.last_stale_reflect_turn) >= STALE_REFLECT {
|
||||
self.last_stale_reflect_turn = turns;
|
||||
return Some(self.reflect(stale_text(turns, since)));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn is_checkpoint(&self, turns: u32) -> bool {
|
||||
if turns < self.policy.soft_checkpoint_turns || turns <= self.checkpoint_turn {
|
||||
return false;
|
||||
}
|
||||
let every = self.policy.soft_checkpoint_every.max(1);
|
||||
turns == self.policy.soft_checkpoint_turns
|
||||
|| turns
|
||||
.saturating_sub(self.policy.soft_checkpoint_turns)
|
||||
.is_multiple_of(every)
|
||||
}
|
||||
|
||||
/// Coaching is capped so a confused model is never nagged forever; the
|
||||
/// breakers above still work after the cap is reached.
|
||||
fn reflect(&mut self, text: String) -> Verdict {
|
||||
if self.reflections >= MAX_REFLECTIONS {
|
||||
return Verdict::Continue;
|
||||
}
|
||||
self.reflections = self.reflections.saturating_add(1);
|
||||
Verdict::Reflect(text)
|
||||
}
|
||||
}
|
||||
|
||||
fn checkpoint_text(turns: u32, minutes: u64) -> String {
|
||||
format!(
|
||||
"Checkpoint: {turns} turns, {minutes} minutes in. Write exactly three short lines \
|
||||
(已完成 / 還缺 / 下一個動作) and then make that next action with a tool call in the \
|
||||
same reply. Do not repeat an action that already produced the same result; if the \
|
||||
current plan cannot work, replace it with a different route."
|
||||
)
|
||||
}
|
||||
|
||||
fn wall_text(minutes: u64, hard_minutes: u64) -> String {
|
||||
format!(
|
||||
"Time check: {minutes} minutes on this task. If the work is genuinely long (a build, \
|
||||
a download, a queue), say so in one line and continue with a tool. If you are stuck, \
|
||||
stop guessing: state what you have tried and what you need, then end with a \
|
||||
standalone [NEEDS_INPUT] line. Nothing stops at {hard_minutes} minutes except you."
|
||||
)
|
||||
}
|
||||
|
||||
fn repeat_text(label: &str, streak: u32) -> String {
|
||||
format!(
|
||||
"You have sent the same action {streak} times in a row: {label}. The result you already \
|
||||
have is the result you will get again. Do not send it again unchanged: read the last \
|
||||
tool result and change something - observe the screen again, scroll, target a selector \
|
||||
instead of an id, reload, or use the shell. If only the human can unblock this, say \
|
||||
what you need and end with a standalone [NEEDS_INPUT] line."
|
||||
)
|
||||
}
|
||||
|
||||
fn failure_text(label: &str, failures: u32) -> String {
|
||||
format!(
|
||||
"{label} has failed {failures} times in a row. Use the error text: different arguments, \
|
||||
a different tool, or a fresh observation of the current state before you retry. If the \
|
||||
blocker is outside your reach, say exactly what is missing and end with a standalone \
|
||||
[NEEDS_INPUT] line."
|
||||
)
|
||||
}
|
||||
|
||||
fn stale_text(turns: u32, since: u32) -> String {
|
||||
format!(
|
||||
"{turns} turns in, nothing new has succeeded for {since} turns. Step back before the \
|
||||
next call: state what you have actually verified, what is still missing, then take one \
|
||||
action you have not tried yet. If the task cannot move without the human, summarise the \
|
||||
state and end with a standalone [NEEDS_INPUT] line."
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn act<'a>(
|
||||
name: &'a str,
|
||||
args: &'a Value,
|
||||
turn: u32,
|
||||
ok: bool,
|
||||
changes_state: bool,
|
||||
) -> ActionObserved<'a> {
|
||||
ActionObserved {
|
||||
name,
|
||||
args,
|
||||
label: "browser: click #12",
|
||||
ok,
|
||||
turn,
|
||||
changes_state,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_ordinary_long_run_is_never_stopped() {
|
||||
let mut guard = LoopGuard::new(RunPolicy::default());
|
||||
for turn in 1..=39 {
|
||||
assert_eq!(
|
||||
guard.on_turn(turn, Duration::from_secs(60 * turn as u64)),
|
||||
Verdict::Continue,
|
||||
"turn {turn}"
|
||||
);
|
||||
let args = json!({"command": format!("step {turn}")});
|
||||
assert_eq!(
|
||||
guard.on_action(&act("shell", &args, turn, true, true)),
|
||||
Verdict::Continue,
|
||||
"action {turn}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_in_the_default_policy_stops_a_run_near_turn_forty() {
|
||||
// The complaint this replaces was "40 輪真的太少了".
|
||||
let policy = RunPolicy::default();
|
||||
assert!(policy.hard_cap_turns > 200);
|
||||
assert!(policy.hard_wall > Duration::from_secs(120 * 60));
|
||||
let mut guard = LoopGuard::new(policy);
|
||||
assert_eq!(
|
||||
guard.on_turn(40, Duration::from_secs(600)),
|
||||
Verdict::Continue
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn looking_at_the_screen_again_is_not_a_loop() {
|
||||
let mut guard = LoopGuard::new(RunPolicy::default());
|
||||
let args = json!({});
|
||||
for turn in 1..=20 {
|
||||
assert_eq!(
|
||||
guard.on_action(&act("computer_observe", &args, turn, true, false)),
|
||||
Verdict::Continue,
|
||||
"turn {turn}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clicking_the_same_control_learns_to_stop() {
|
||||
let mut guard = LoopGuard::new(RunPolicy::default());
|
||||
let args = json!({"action": "click", "element": 12});
|
||||
for turn in 1..REPEAT_WARN {
|
||||
assert_eq!(
|
||||
guard.on_action(&act("browser", &args, turn, true, true)),
|
||||
Verdict::Continue
|
||||
);
|
||||
}
|
||||
assert!(matches!(
|
||||
guard.on_action(&act("browser", &args, REPEAT_WARN, true, true)),
|
||||
Verdict::Reflect(_)
|
||||
));
|
||||
for turn in REPEAT_WARN + 1..REPEAT_HALT {
|
||||
assert_eq!(
|
||||
guard.on_action(&act("browser", &args, turn, true, true)),
|
||||
Verdict::Continue
|
||||
);
|
||||
}
|
||||
assert!(matches!(
|
||||
guard.on_action(&act("browser", &args, REPEAT_HALT, true, true)),
|
||||
Verdict::Halt {
|
||||
reason: StopReason::LoopDetected,
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn argument_order_does_not_make_the_same_action_look_new() {
|
||||
assert_eq!(
|
||||
signature("browser", &json!({"element": 12, "action": "click"})),
|
||||
signature("browser", &json!({"action": "click", "element": 12}))
|
||||
);
|
||||
assert_ne!(
|
||||
signature("browser", &json!({"action": "click", "element": 12})),
|
||||
signature("browser", &json!({"action": "click", "element": 13}))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_repeated_failure_is_coached_then_parked() {
|
||||
let mut guard = LoopGuard::new(RunPolicy::default());
|
||||
let args = json!({"command": "npm test"});
|
||||
for turn in 1..FAILURE_WARN {
|
||||
assert_eq!(
|
||||
guard.on_action(&act("shell", &args, turn, false, true)),
|
||||
Verdict::Continue
|
||||
);
|
||||
}
|
||||
assert!(matches!(
|
||||
guard.on_action(&act("shell", &args, FAILURE_WARN, false, true)),
|
||||
Verdict::Reflect(_)
|
||||
));
|
||||
for turn in FAILURE_WARN + 1..FAILURE_HALT {
|
||||
guard.on_action(&act("shell", &args, turn, false, true));
|
||||
}
|
||||
assert!(
|
||||
guard
|
||||
.on_action(&act("shell", &args, FAILURE_HALT, false, true))
|
||||
.is_halt()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_whole_batch_of_different_failures_also_stops() {
|
||||
let mut guard = LoopGuard::new(RunPolicy::default());
|
||||
for turn in 1..FAILURE_HALT_ANY {
|
||||
let args = json!({"command": format!("try {turn}")});
|
||||
let verdict = guard.on_action(&act("shell", &args, turn, false, true));
|
||||
assert!(!verdict.is_halt(), "halted too early at {turn}");
|
||||
}
|
||||
let args = json!({"command": "last try"});
|
||||
assert!(
|
||||
guard
|
||||
.on_action(&act("shell", &args, FAILURE_HALT_ANY, false, true))
|
||||
.is_halt()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn polling_a_build_or_a_queue_is_not_treated_as_spinning() {
|
||||
let mut guard = LoopGuard::new(RunPolicy::default());
|
||||
let args = json!({"ms": 30000});
|
||||
for turn in 1..=160 {
|
||||
assert_eq!(
|
||||
guard.on_action(&act("wait", &args, turn, true, false)),
|
||||
Verdict::Continue,
|
||||
"turn {turn}"
|
||||
);
|
||||
// A checkpoint question is fine; what a polling run must never get
|
||||
// is a halt.
|
||||
assert!(
|
||||
!guard.on_turn(turn, Duration::from_secs(30)).is_halt(),
|
||||
"a polling run was halted at turn {turn}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn re_observing_the_same_screen_eventually_stops_asking() {
|
||||
let mut guard = LoopGuard::new(RunPolicy::default());
|
||||
let first = json!({"command": "ls"});
|
||||
guard.on_action(&act("shell", &first, 1, true, true));
|
||||
let eye = json!({});
|
||||
let mut reflected = 0;
|
||||
for turn in 2..=150 {
|
||||
guard.on_action(&act("computer_observe", &eye, turn, true, false));
|
||||
if matches!(
|
||||
guard.on_turn(turn, Duration::from_secs(10)),
|
||||
Verdict::Reflect(_)
|
||||
) {
|
||||
reflected += 1;
|
||||
}
|
||||
}
|
||||
assert!(reflected >= 1, "the run should be asked to step back");
|
||||
assert!(guard.on_turn(152, Duration::from_secs(10)).is_halt());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoints_are_asks_not_stops() {
|
||||
let mut guard = LoopGuard::new(RunPolicy::default());
|
||||
assert!(matches!(
|
||||
guard.on_turn(SOFT_CHECKPOINT_TURNS, Duration::from_secs(600)),
|
||||
Verdict::Reflect(_)
|
||||
));
|
||||
assert_eq!(
|
||||
guard.on_turn(SOFT_CHECKPOINT_TURNS + 1, Duration::from_secs(601)),
|
||||
Verdict::Continue
|
||||
);
|
||||
let next = SOFT_CHECKPOINT_TURNS + SOFT_CHECKPOINT_EVERY;
|
||||
assert!(matches!(
|
||||
guard.on_turn(next, Duration::from_secs(700)),
|
||||
Verdict::Reflect(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coaching_is_rare_and_the_breaker_still_works_after_it() {
|
||||
let policy = RunPolicy {
|
||||
soft_checkpoint_turns: 10,
|
||||
soft_checkpoint_every: 10,
|
||||
..RunPolicy::default()
|
||||
};
|
||||
let mut guard = LoopGuard::new(policy);
|
||||
let mut asks = 0;
|
||||
for turn in 1..=120 {
|
||||
if matches!(
|
||||
guard.on_turn(turn, Duration::from_secs(2)),
|
||||
Verdict::Reflect(_)
|
||||
) {
|
||||
asks += 1;
|
||||
}
|
||||
let args = json!({"command": format!("step {turn}")});
|
||||
guard.on_action(&act("shell", &args, turn, true, true));
|
||||
}
|
||||
assert_eq!(asks, MAX_REFLECTIONS as usize, "coaching must stop nagging");
|
||||
assert_eq!(
|
||||
guard.on_turn(HARD_CAP_TURNS, Duration::from_secs(60)),
|
||||
Verdict::Halt {
|
||||
reason: StopReason::BudgetExhausted,
|
||||
note: format!(
|
||||
"已達到保險上限 {HARD_CAP_TURNS} 輪（這是迴圈失控的保護，不是任務做完）"
|
||||
),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wall_clock_breaks_last_of_all() {
|
||||
let mut guard = LoopGuard::new(RunPolicy::default());
|
||||
assert!(matches!(
|
||||
guard.on_turn(5, Duration::from_secs(SOFT_WALL_MINUTES * 60)),
|
||||
Verdict::Reflect(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
guard.on_turn(6, Duration::from_secs(HARD_WALL_MINUTES * 60 + 1)),
|
||||
Verdict::Halt {
|
||||
reason: StopReason::BudgetExhausted,
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_resumed_run_is_not_judged_for_the_turns_it_already_has() {
|
||||
let mut guard = LoopGuard::with_watch(RunPolicy::default(), 120);
|
||||
for turn in 120..150 {
|
||||
assert_eq!(
|
||||
guard.on_turn(turn, Duration::from_secs(60)),
|
||||
Verdict::Continue,
|
||||
"turn {turn}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overrides_are_opt_in_and_tolerant() {
|
||||
assert_eq!(RunPolicy::from_lookup(|_| None), RunPolicy::default());
|
||||
assert_eq!(
|
||||
RunPolicy::from_lookup(
|
||||
|name| (name == "LAZYBOY_RUN_CAP_TURNS").then(|| "not-a-number".to_string())
|
||||
)
|
||||
.hard_cap_turns,
|
||||
HARD_CAP_TURNS
|
||||
);
|
||||
assert_eq!(
|
||||
RunPolicy::from_lookup(
|
||||
|name| (name == "LAZYBOY_RUN_CAP_TURNS").then(|| "25".to_string())
|
||||
)
|
||||
.hard_cap_turns,
|
||||
25
|
||||
);
|
||||
assert_eq!(
|
||||
RunPolicy::from_lookup(
|
||||
|name| (name == "LAZYBOY_RUN_HARD_MINUTES").then(|| "30".to_string())
|
||||
)
|
||||
.hard_wall,
|
||||
Duration::from_secs(30 * 60)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -116,21 +116,36 @@ pub enum VoiceEvent {
|
|||
AudioPcm(Vec<u8>),
|
||||
SpeechStarted,
|
||||
SpeechStopped,
|
||||
InputTranscript { text: String, final_: bool },
|
||||
OutputTranscript { text: String, final_: bool },
|
||||
InputTranscript {
|
||||
text: String,
|
||||
final_: bool,
|
||||
},
|
||||
OutputTranscript {
|
||||
text: String,
|
||||
final_: bool,
|
||||
},
|
||||
FunctionCall {
|
||||
call_id: String,
|
||||
name: String,
|
||||
arguments: String,
|
||||
},
|
||||
FunctionCallOutput { call_id: String, output: String },
|
||||
SpeakNow { text: String },
|
||||
InjectContext { text: String },
|
||||
FunctionCallOutput {
|
||||
call_id: String,
|
||||
output: String,
|
||||
},
|
||||
SpeakNow {
|
||||
text: String,
|
||||
},
|
||||
InjectContext {
|
||||
text: String,
|
||||
},
|
||||
ResponseCreate,
|
||||
ResponseStarted,
|
||||
ResponseFinished,
|
||||
CancelResponse,
|
||||
Error { message: String },
|
||||
Error {
|
||||
message: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
|
|
@ -190,7 +205,9 @@ fn native_roots() -> Result<rustls::RootCertStore, VoiceError> {
|
|||
let certs = rustls_native_certs::load_native_certs();
|
||||
roots.add_parsable_certificates(certs.certs);
|
||||
if roots.is_empty() {
|
||||
return Err(VoiceError::Message("No trusted TLS certificates available".into()));
|
||||
return Err(VoiceError::Message(
|
||||
"No trusted TLS certificates available".into(),
|
||||
));
|
||||
}
|
||||
Ok(roots)
|
||||
}
|
||||
|
|
@ -222,15 +239,18 @@ impl VoiceRealtime for HostedVoice {
|
|||
);
|
||||
// Choose explicitly: the dependency graph enables both ring and aws-lc-rs.
|
||||
// Rustls's automatic provider selection panics in that configuration.
|
||||
let tls = rustls::ClientConfig::builder_with_provider(
|
||||
Arc::new(rustls::crypto::ring::default_provider()),
|
||||
)
|
||||
let tls = rustls::ClientConfig::builder_with_provider(Arc::new(
|
||||
rustls::crypto::ring::default_provider(),
|
||||
))
|
||||
.with_safe_default_protocol_versions()
|
||||
.map_err(|error| VoiceError::Message(error.to_string()))?
|
||||
.with_root_certificates(native_roots()?)
|
||||
.with_no_client_auth();
|
||||
let (stream, _) = tokio_tungstenite::connect_async_tls_with_config(
|
||||
http_request, None, false, Some(tokio_tungstenite::Connector::Rustls(Arc::new(tls))),
|
||||
http_request,
|
||||
None,
|
||||
false,
|
||||
Some(tokio_tungstenite::Connector::Rustls(Arc::new(tls))),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| VoiceError::Message(error.to_string()))?;
|
||||
|
|
@ -241,7 +261,9 @@ impl VoiceRealtime for HostedVoice {
|
|||
read,
|
||||
};
|
||||
socket
|
||||
.send_raw(Message::Text(session_update_json(self.provider, &request).into()))
|
||||
.send_raw(Message::Text(
|
||||
session_update_json(self.provider, &request).into(),
|
||||
))
|
||||
.await?;
|
||||
for (role, text) in &request.history {
|
||||
if text.trim().is_empty() {
|
||||
|
|
@ -266,10 +288,13 @@ impl VoiceRealtime for HostedVoice {
|
|||
}
|
||||
}
|
||||
|
||||
type HostedWrite =
|
||||
futures_util::stream::SplitSink<tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>, Message>;
|
||||
type HostedRead =
|
||||
futures_util::stream::SplitStream<tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>>;
|
||||
type HostedWrite = futures_util::stream::SplitSink<
|
||||
tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>,
|
||||
Message,
|
||||
>;
|
||||
type HostedRead = futures_util::stream::SplitStream<
|
||||
tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>,
|
||||
>;
|
||||
|
||||
struct HostedSocket {
|
||||
provider: VoiceProvider,
|
||||
|
|
@ -447,10 +472,7 @@ pub fn parse_provider_event(text: &str) -> Option<VoiceEvent> {
|
|||
.and_then(Value::as_str)
|
||||
.or_else(|| event.get("text").and_then(Value::as_str))?
|
||||
.to_string();
|
||||
Some(VoiceEvent::InputTranscript {
|
||||
text,
|
||||
final_: true,
|
||||
})
|
||||
Some(VoiceEvent::InputTranscript { text, final_: true })
|
||||
}
|
||||
"response.output_audio.delta" | "response.audio.delta" => {
|
||||
let encoded = event.get("delta").and_then(Value::as_str)?;
|
||||
|
|
@ -473,10 +495,7 @@ pub fn parse_provider_event(text: &str) -> Option<VoiceEvent> {
|
|||
.or_else(|| event.get("text").and_then(Value::as_str))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
Some(VoiceEvent::OutputTranscript {
|
||||
text,
|
||||
final_: true,
|
||||
})
|
||||
Some(VoiceEvent::OutputTranscript { text, final_: true })
|
||||
}
|
||||
"response.function_call_arguments.done" => {
|
||||
let call_id = event.get("call_id")?.as_str()?.to_string();
|
||||
|
|
@ -574,10 +593,7 @@ impl VoiceSocket for ScriptedSocket {
|
|||
}
|
||||
VoiceEvent::SpeakNow { text } | VoiceEvent::InjectContext { text } => {
|
||||
let tx = self.incoming.lock().await;
|
||||
let _ = tx.send(VoiceEvent::OutputTranscript {
|
||||
text,
|
||||
final_: true,
|
||||
});
|
||||
let _ = tx.send(VoiceEvent::OutputTranscript { text, final_: true });
|
||||
let _ = tx.send(VoiceEvent::AudioPcm(Self::beep()));
|
||||
}
|
||||
VoiceEvent::ResponseCreate => {}
|
||||
|
|
@ -597,31 +613,49 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn voice_tls_config_uses_an_explicit_provider() {
|
||||
let config = rustls::ClientConfig::builder_with_provider(
|
||||
Arc::new(rustls::crypto::ring::default_provider()),
|
||||
).with_safe_default_protocol_versions().unwrap()
|
||||
.with_root_certificates(native_roots().unwrap()).with_no_client_auth();
|
||||
let config = rustls::ClientConfig::builder_with_provider(Arc::new(
|
||||
rustls::crypto::ring::default_provider(),
|
||||
))
|
||||
.with_safe_default_protocol_versions()
|
||||
.unwrap()
|
||||
.with_root_certificates(native_roots().unwrap())
|
||||
.with_no_client_auth();
|
||||
assert!(!config.crypto_provider().cipher_suites.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_audio_uses_json_and_nested_session_settings() {
|
||||
let request = VoiceConnectRequest {
|
||||
api_key: String::new(), model_id: "gpt-realtime".into(),
|
||||
voice_id: "marin".into(), instructions: "test".into(),
|
||||
tools: vec![], history: vec![],
|
||||
api_key: String::new(),
|
||||
model_id: "gpt-realtime".into(),
|
||||
voice_id: "marin".into(),
|
||||
instructions: "test".into(),
|
||||
tools: vec![],
|
||||
history: vec![],
|
||||
};
|
||||
let session: Value = serde_json::from_str(&session_update_json(VoiceProvider::Openai, &request)).unwrap();
|
||||
let session: Value =
|
||||
serde_json::from_str(&session_update_json(VoiceProvider::Openai, &request)).unwrap();
|
||||
assert!(session["session"].get("voice").is_none());
|
||||
assert!(session["session"].get("turn_detection").is_none());
|
||||
assert_eq!(session.pointer("/session/audio/input/format/rate"), Some(&json!(24000)));
|
||||
assert_eq!(session.pointer("/session/audio/input/turn_detection/type"), Some(&json!("server_vad")));
|
||||
assert_eq!(
|
||||
session.pointer("/session/audio/input/format/rate"),
|
||||
Some(&json!(24000))
|
||||
);
|
||||
assert_eq!(
|
||||
session.pointer("/session/audio/input/turn_detection/type"),
|
||||
Some(&json!("server_vad"))
|
||||
);
|
||||
let event = VoiceEvent::AudioPcm(vec![0, 1, 2, 3]);
|
||||
let Some(Message::Text(text)) = encode_provider_event(VoiceProvider::Openai, &event) else { panic!("expected JSON audio") };
|
||||
let Some(Message::Text(text)) = encode_provider_event(VoiceProvider::Openai, &event) else {
|
||||
panic!("expected JSON audio")
|
||||
};
|
||||
let encoded: Value = serde_json::from_str(&text).unwrap();
|
||||
assert_eq!(encoded["type"], "input_audio_buffer.append");
|
||||
assert_eq!(encoded["audio"], "AAECAw==");
|
||||
assert!(matches!(encode_provider_event(VoiceProvider::Xai, &event), Some(Message::Binary(_))));
|
||||
assert!(matches!(
|
||||
encode_provider_event(VoiceProvider::Xai, &event),
|
||||
Some(Message::Binary(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -656,9 +690,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn parses_xai_and_openai_event_aliases() {
|
||||
let started = parse_provider_event(
|
||||
r#"{"type":"input_audio_buffer.speech_started"}"#,
|
||||
);
|
||||
let started = parse_provider_event(r#"{"type":"input_audio_buffer.speech_started"}"#);
|
||||
assert!(matches!(started, Some(VoiceEvent::SpeechStarted)));
|
||||
|
||||
let transcript = parse_provider_event(
|
||||
|
|
@ -672,9 +704,7 @@ mod tests {
|
|||
other => panic!("{other:?}"),
|
||||
}
|
||||
|
||||
let old_audio = parse_provider_event(
|
||||
r#"{"type":"response.audio.delta","delta":"AQID"}"#,
|
||||
);
|
||||
let old_audio = parse_provider_event(r#"{"type":"response.audio.delta","delta":"AQID"}"#);
|
||||
assert!(matches!(old_audio, Some(VoiceEvent::AudioPcm(_))));
|
||||
|
||||
let tool = parse_provider_event(
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
name = "lazyboy-sandbox"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
publish.workspace = true
|
||||
|
||||
|
|
@ -10,10 +11,8 @@ lazyboy-contracts.workspace = true
|
|||
lazyboy-control.workspace = true
|
||||
async-trait = "0.1"
|
||||
reqwest.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
tokio.workspace = true
|
||||
base64.workspace = true
|
||||
chrono.workspace = true
|
||||
sha2.workspace = true
|
||||
hex.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
|
|
|||
|
|
@ -1,12 +1,16 @@
|
|||
use async_trait::async_trait;
|
||||
use base64::Engine;
|
||||
use lazyboy_contracts::ComputerCapabilities;
|
||||
use lazyboy_contracts::{ActiveWindow, ComputerObservation, CursorPosition, SandboxKind};
|
||||
use lazyboy_contracts::{
|
||||
ActiveWindow, ComputerObservation, CursorPosition, DEFAULT_SCREEN_HEIGHT, DEFAULT_SCREEN_WIDTH,
|
||||
SandboxKind,
|
||||
};
|
||||
use lazyboy_control::{
|
||||
ActionRequest, ActionResult, AdapterContext, CommandRequest, CommandResult, ComputerRef,
|
||||
EnsureScreenRequest, EnsureScreenResult, FileEntry, ProvisionRequest, SandboxError,
|
||||
SandboxProvider, ScreenSession, observation_from_png, observation_with_elements,
|
||||
parse_ui_elements,
|
||||
ActionRequest, ActionResult, AdapterContext, BrowserPage, BrowserRequest, CommandRequest,
|
||||
CommandResult, ComputerRef, EnsureScreenRequest, EnsureScreenResult, FileEntry,
|
||||
ProvisionRequest, RecordingRequest, RecordingResult, RecordingSession, SandboxError,
|
||||
SandboxProvider, ScreenSession, image_dimensions, observation_from_png,
|
||||
observation_with_elements, parse_ui_elements,
|
||||
};
|
||||
use reqwest::Client;
|
||||
use serde_json::Value;
|
||||
|
|
@ -36,21 +40,21 @@ impl DockerSandbox {
|
|||
if let Some(bot_id) = &context.bot_id {
|
||||
headers.insert("x-lazyboy-bot-id", bot_id.parse().unwrap());
|
||||
}
|
||||
if let Some(display) = &context.display {
|
||||
if let Ok(value) = display.parse() {
|
||||
if let Some(display) = &context.display
|
||||
&& let Ok(value) = display.parse()
|
||||
{
|
||||
headers.insert("x-lazyboy-display", value);
|
||||
}
|
||||
}
|
||||
if let Some(profile) = &context.profile_path {
|
||||
if let Ok(value) = profile.parse() {
|
||||
if let Some(profile) = &context.profile_path
|
||||
&& let Ok(value) = profile.parse()
|
||||
{
|
||||
headers.insert("x-lazyboy-profile", value);
|
||||
}
|
||||
}
|
||||
if let Some(slot) = context.screen_slot {
|
||||
if let Ok(value) = slot.to_string().parse() {
|
||||
if let Some(slot) = context.screen_slot
|
||||
&& let Ok(value) = slot.to_string().parse()
|
||||
{
|
||||
headers.insert("x-lazyboy-screen-slot", value);
|
||||
}
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
|
|
@ -237,11 +241,18 @@ impl SandboxProvider for DockerSandbox {
|
|||
.send()
|
||||
.await
|
||||
.map_err(|error| SandboxError::message(error.to_string()))?;
|
||||
let response = response
|
||||
.error_for_status()
|
||||
.map_err(|error| SandboxError::message(error.to_string()))?;
|
||||
let body: Value = response
|
||||
.json()
|
||||
.await
|
||||
.map_err(|error| SandboxError::message(error.to_string()))?;
|
||||
Ok(ActionResult {
|
||||
clipboard_text: body
|
||||
.get("clipboardText")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string),
|
||||
completed: body.get("completed").and_then(Value::as_u64).unwrap_or(0) as usize,
|
||||
observation: if body.get("png_base64").is_some() {
|
||||
Some(decode_observation(&body)?)
|
||||
|
|
@ -251,6 +262,93 @@ impl SandboxProvider for DockerSandbox {
|
|||
})
|
||||
}
|
||||
|
||||
async fn browser(
|
||||
&self,
|
||||
computer: &ComputerRef,
|
||||
request: BrowserRequest,
|
||||
context: &AdapterContext,
|
||||
) -> Result<BrowserPage, SandboxError> {
|
||||
let response = self
|
||||
.client
|
||||
.post(self.url(&format!("/computers/{}/browser", computer.id)))
|
||||
.headers(self.headers(context))
|
||||
.json(&request)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| SandboxError::message(error.to_string()))?;
|
||||
let body: Value = response
|
||||
.json()
|
||||
.await
|
||||
.map_err(|error| SandboxError::message(error.to_string()))?;
|
||||
serde_json::from_value(body).map_err(|error| SandboxError::message(error.to_string()))
|
||||
}
|
||||
|
||||
async fn start_recording(
|
||||
&self,
|
||||
computer: &ComputerRef,
|
||||
request: RecordingRequest,
|
||||
context: &AdapterContext,
|
||||
) -> Result<RecordingSession, SandboxError> {
|
||||
let response = self
|
||||
.client
|
||||
.post(self.url(&format!("/computers/{}/recording/start", computer.id)))
|
||||
.headers(self.headers(context))
|
||||
.json(&request)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| SandboxError::message(error.to_string()))?;
|
||||
let body: Value = response
|
||||
.json()
|
||||
.await
|
||||
.map_err(|error| SandboxError::message(error.to_string()))?;
|
||||
serde_json::from_value(body).map_err(|error| SandboxError::message(error.to_string()))
|
||||
}
|
||||
|
||||
async fn stop_recording(
|
||||
&self,
|
||||
computer: &ComputerRef,
|
||||
request: RecordingRequest,
|
||||
context: &AdapterContext,
|
||||
) -> Result<(), SandboxError> {
|
||||
let response = self
|
||||
.client
|
||||
.post(self.url(&format!("/computers/{}/recording/stop", computer.id)))
|
||||
.headers(self.headers(context))
|
||||
.json(&request)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| SandboxError::message(error.to_string()))?;
|
||||
if response.status().is_success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(SandboxError::message(format!(
|
||||
"stop recording failed: {}",
|
||||
response.status()
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_recording(
|
||||
&self,
|
||||
computer: &ComputerRef,
|
||||
request: RecordingRequest,
|
||||
context: &AdapterContext,
|
||||
) -> Result<RecordingResult, SandboxError> {
|
||||
let response = self
|
||||
.client
|
||||
.post(self.url(&format!("/computers/{}/recording/collect", computer.id)))
|
||||
.headers(self.headers(context))
|
||||
.json(&request)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| SandboxError::message(error.to_string()))?;
|
||||
let body: Value = response
|
||||
.json()
|
||||
.await
|
||||
.map_err(|error| SandboxError::message(error.to_string()))?;
|
||||
serde_json::from_value(body).map_err(|error| SandboxError::message(error.to_string()))
|
||||
}
|
||||
|
||||
async fn connect_screen(
|
||||
&self,
|
||||
computer: &ComputerRef,
|
||||
|
|
@ -458,8 +556,45 @@ fn decode_observation(body: &Value) -> Result<ComputerObservation, SandboxError>
|
|||
.get("elements")
|
||||
.map(|value| parse_ui_elements(&value.to_string()))
|
||||
.unwrap_or_default();
|
||||
Ok(observation_with_elements(
|
||||
observation_from_png(png, 1280, 800, cursor, window),
|
||||
// The control endpoint does not put the mode on the wire, so the frame
|
||||
// itself decides: a desktop that is not 1280x800 must not be reported to
|
||||
// the model as 1280x800.
|
||||
let (width, height) =
|
||||
image_dimensions(&png).unwrap_or((DEFAULT_SCREEN_WIDTH, DEFAULT_SCREEN_HEIGHT));
|
||||
let mut observation = observation_with_elements(
|
||||
observation_from_png(png, width, height, cursor, window),
|
||||
elements,
|
||||
))
|
||||
);
|
||||
observation.native_observation_complete = body
|
||||
.get("native_observation_complete")
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false);
|
||||
Ok(observation)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::decode_observation;
|
||||
use lazyboy_contracts::{DEFAULT_SCREEN_HEIGHT, DEFAULT_SCREEN_WIDTH};
|
||||
use serde_json::json;
|
||||
|
||||
/// A 3x2 solid-red PNG: small enough to inline, real enough to decode.
|
||||
const FRAME_3X2: &str = "iVBORw0KGgoAAAANSUhEUgAAAAMAAAACCAIAAAASFvFNAAAAEElEQVR4nGP4z8AAQQxwFgBB\
|
||||
0gX7h/C5SAAAAABJRU5ErkJggg==";
|
||||
|
||||
#[test]
|
||||
fn the_mode_is_read_from_the_frame() {
|
||||
let observation = decode_observation(&json!({ "png_base64": FRAME_3X2 }))
|
||||
.expect("a decodable frame makes an observation");
|
||||
assert_eq!(observation.width, 3);
|
||||
assert_eq!(observation.height, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_undecodable_frame_falls_back_to_the_default_mode() {
|
||||
let observation =
|
||||
decode_observation(&json!({ "png_base64": "AAAAAAAAAAA=" })).expect("base64 decodes");
|
||||
assert_eq!(observation.width, DEFAULT_SCREEN_WIDTH);
|
||||
assert_eq!(observation.height, DEFAULT_SCREEN_HEIGHT);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ use std::sync::Mutex;
|
|||
use async_trait::async_trait;
|
||||
use lazyboy_contracts::{ComputerObservation, SandboxKind};
|
||||
use lazyboy_control::{
|
||||
ActionRequest, ActionResult, AdapterContext, CommandRequest, CommandResult, ComputerRef,
|
||||
FileEntry, ProvisionRequest, SandboxError, SandboxProvider, ScreenSession,
|
||||
observation_from_png,
|
||||
ActionRequest, ActionResult, AdapterContext, BrowserPage, BrowserRequest, CommandRequest,
|
||||
CommandResult, ComputerRef, FileEntry, ProvisionRequest, RecordingRequest, RecordingResult,
|
||||
RecordingSession, SandboxError, SandboxProvider, ScreenSession, observation_from_png,
|
||||
};
|
||||
|
||||
const EMPTY_PNG: &[u8] = &[
|
||||
|
|
@ -55,15 +55,16 @@ impl SandboxProvider for FakeSandbox {
|
|||
request: CommandRequest,
|
||||
_context: &AdapterContext,
|
||||
) -> Result<CommandResult, SandboxError> {
|
||||
if request.argv.get(0).map(String::as_str) == Some("mkdir") {
|
||||
if request.argv.first().map(String::as_str) == Some("mkdir") {
|
||||
return Ok(CommandResult {
|
||||
stdout: String::new(),
|
||||
stderr: String::new(),
|
||||
code: 0,
|
||||
});
|
||||
}
|
||||
if request.argv.get(0).map(String::as_str) == Some("touch") {
|
||||
if let Some(path) = request.argv.get(1) {
|
||||
if request.argv.first().map(String::as_str) == Some("touch")
|
||||
&& let Some(path) = request.argv.get(1)
|
||||
{
|
||||
self.files
|
||||
.lock()
|
||||
.unwrap()
|
||||
|
|
@ -71,7 +72,6 @@ impl SandboxProvider for FakeSandbox {
|
|||
.or_default()
|
||||
.insert(path.clone(), Vec::new());
|
||||
}
|
||||
}
|
||||
Ok(CommandResult {
|
||||
stdout: request.argv.join(" "),
|
||||
stderr: String::new(),
|
||||
|
|
@ -94,6 +94,7 @@ impl SandboxProvider for FakeSandbox {
|
|||
context: &AdapterContext,
|
||||
) -> Result<ActionResult, SandboxError> {
|
||||
Ok(ActionResult {
|
||||
clipboard_text: None,
|
||||
completed: request.actions.len(),
|
||||
observation: if request.observe {
|
||||
Some(self.observe(computer, context).await?)
|
||||
|
|
@ -103,6 +104,50 @@ impl SandboxProvider for FakeSandbox {
|
|||
})
|
||||
}
|
||||
|
||||
async fn browser(
|
||||
&self,
|
||||
_computer: &ComputerRef,
|
||||
_request: BrowserRequest,
|
||||
_context: &AdapterContext,
|
||||
) -> Result<BrowserPage, SandboxError> {
|
||||
Ok(BrowserPage {
|
||||
ok: true,
|
||||
url: "about:blank".into(),
|
||||
title: "fake".into(),
|
||||
..BrowserPage::default()
|
||||
})
|
||||
}
|
||||
|
||||
async fn start_recording(
|
||||
&self,
|
||||
_computer: &ComputerRef,
|
||||
request: RecordingRequest,
|
||||
_context: &AdapterContext,
|
||||
) -> Result<RecordingSession, SandboxError> {
|
||||
Ok(RecordingSession {
|
||||
skill_id: request.skill_id,
|
||||
output_dir: "/tmp/lazyboy/teach-fake".into(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn stop_recording(
|
||||
&self,
|
||||
_computer: &ComputerRef,
|
||||
_request: RecordingRequest,
|
||||
_context: &AdapterContext,
|
||||
) -> Result<(), SandboxError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn collect_recording(
|
||||
&self,
|
||||
_computer: &ComputerRef,
|
||||
_request: RecordingRequest,
|
||||
_context: &AdapterContext,
|
||||
) -> Result<RecordingResult, SandboxError> {
|
||||
Ok(RecordingResult { events: Vec::new() })
|
||||
}
|
||||
|
||||
async fn connect_screen(
|
||||
&self,
|
||||
computer: &ComputerRef,
|
||||
|
|
|
|||
|
|
@ -2,26 +2,26 @@
|
|||
name = "lazyboy-supervisor"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
publish.workspace = true
|
||||
|
||||
[dependencies]
|
||||
lazyboy-contracts.workspace = true
|
||||
lazyboy-control.workspace = true
|
||||
axum.workspace = true
|
||||
tokio.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
bollard.workspace = true
|
||||
uuid.workspace = true
|
||||
base64.workspace = true
|
||||
reqwest.workspace = true
|
||||
async-trait = "0.1"
|
||||
futures-util = "0.3"
|
||||
|
||||
hmac.workspace = true
|
||||
sha2.workspace = true
|
||||
hex.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
|
|
|||
|
|
@ -9,13 +9,13 @@ use bollard::container::{
|
|||
StartContainerOptions, StopContainerOptions,
|
||||
};
|
||||
use bollard::exec::{CreateExecOptions, StartExecOptions, StartExecResults};
|
||||
use bollard::models::{HostConfig, HostConfigLogConfig, PortBinding};
|
||||
use bollard::network::CreateNetworkOptions;
|
||||
use bollard::models::{EndpointSettings, HostConfig, HostConfigLogConfig, PortBinding};
|
||||
use bollard::network::{ConnectNetworkOptions, CreateNetworkOptions};
|
||||
use futures_util::StreamExt;
|
||||
use lazyboy_control::{
|
||||
ActionRequest, CommandRequest, CommandResult, EnsureScreenRequest, EnsureScreenResult, HOME,
|
||||
ScreenTarget, TEAM_SCREEN_LIMIT, normalize_display, normalize_workspace_path,
|
||||
pointer_state_command_on, screen_layout, screenshot_command_on, window_list_command_on,
|
||||
ActionRequest, BrowserRequest, CommandRequest, CommandResult, EnsureScreenRequest,
|
||||
EnsureScreenResult, HOME, RecordingRequest, ScreenTarget, TEAM_SCREEN_LIMIT, normalize_display,
|
||||
normalize_workspace_path, screen_layout,
|
||||
};
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
|
|
@ -34,20 +34,20 @@ pub struct Provisioned {
|
|||
}
|
||||
|
||||
pub struct ObservePayload {
|
||||
pub png: Vec<u8>,
|
||||
pub json: serde_json::Value,
|
||||
}
|
||||
|
||||
impl ObservePayload {
|
||||
fn from_json(value: serde_json::Value) -> Result<Self, String> {
|
||||
let encoded = value
|
||||
// The screenshot travels inside `json` as base64; only its presence is
|
||||
// checked here so a broken capture fails fast instead of returning an
|
||||
// empty observation.
|
||||
value
|
||||
.get("png_base64")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|encoded| !encoded.is_empty())
|
||||
.ok_or_else(|| "missing png".to_string())?;
|
||||
let png = base64::engine::general_purpose::STANDARD
|
||||
.decode(encoded)
|
||||
.map_err(|error| error.to_string())?;
|
||||
Ok(Self { png, json: value })
|
||||
Ok(Self { json: value })
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -76,7 +76,7 @@ impl DockerHost {
|
|||
return Err("invalid home key".into());
|
||||
}
|
||||
let expected = PathBuf::from(&data_dir).join("homes").join(home_key);
|
||||
if PathBuf::from(home_path) != expected {
|
||||
if *home_path != expected {
|
||||
return Err("home path is outside managed homes".into());
|
||||
}
|
||||
// Work on the container-local path, not the host daemon's bind path.
|
||||
|
|
@ -187,7 +187,15 @@ impl DockerHost {
|
|||
),
|
||||
format!(
|
||||
"LAZYBOY_COMPUTER_SUDO={}",
|
||||
if computer_sudo_enabled() { "true" } else { "false" }
|
||||
if computer_sudo_enabled() {
|
||||
"true"
|
||||
} else {
|
||||
"false"
|
||||
}
|
||||
),
|
||||
format!(
|
||||
"LAZYBOY_COMPUTER_DRIVER={}",
|
||||
lazyboy_control::ComputerDriver::from_env().as_str()
|
||||
),
|
||||
]),
|
||||
labels: Some(labels),
|
||||
|
|
@ -251,42 +259,6 @@ impl DockerHost {
|
|||
})
|
||||
}
|
||||
|
||||
pub async fn exec(&self, id: &str, request: CommandRequest) -> Result<CommandResult, String> {
|
||||
let cwd = match request.cwd {
|
||||
Some(cwd) if PathBuf::from(&cwd).is_absolute() => cwd,
|
||||
Some(cwd) => {
|
||||
let relative = normalize_workspace_path(&cwd).map_err(|error| error.to_string())?;
|
||||
if relative.is_empty() {
|
||||
HOME.to_string()
|
||||
} else {
|
||||
format!("{HOME}/{relative}")
|
||||
}
|
||||
}
|
||||
None => HOME.to_string(),
|
||||
};
|
||||
let timeout_ms = request.timeout_ms.unwrap_or(30_000).clamp(100, 120_000);
|
||||
let argv = if request.argv.is_empty() {
|
||||
vec!["/bin/echo".into(), "ready".into()]
|
||||
} else {
|
||||
request.argv
|
||||
};
|
||||
let mut bounded = vec![
|
||||
"timeout".into(),
|
||||
"--signal=TERM".into(),
|
||||
"--kill-after=2s".into(),
|
||||
format!("{}s", timeout_ms as f64 / 1000.0),
|
||||
];
|
||||
bounded.extend(argv);
|
||||
self.exec_raw_cmd(
|
||||
id,
|
||||
&bounded,
|
||||
Some(&cwd),
|
||||
&ScreenTarget::default(),
|
||||
request.stdin,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn exec_on(
|
||||
&self,
|
||||
id: &str,
|
||||
|
|
@ -322,100 +294,12 @@ impl DockerHost {
|
|||
.await
|
||||
}
|
||||
|
||||
pub async fn observe(&self, id: &str) -> Result<Vec<u8>, String> {
|
||||
Ok(self
|
||||
.observe_payload(id, &ScreenTarget::default())
|
||||
.await?
|
||||
.png)
|
||||
}
|
||||
|
||||
pub async fn observe_payload(
|
||||
&self,
|
||||
id: &str,
|
||||
target: &ScreenTarget,
|
||||
) -> Result<ObservePayload, String> {
|
||||
let mut body = if let Ok(value) = self.control_observe_json(id, target).await {
|
||||
value
|
||||
} else {
|
||||
let (stdout, stderr, code) = self
|
||||
.exec_raw(
|
||||
id,
|
||||
&screenshot_command_on(&target.display),
|
||||
None,
|
||||
target,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
if code != 0 {
|
||||
return Err(String::from_utf8_lossy(&stderr).into_owned());
|
||||
}
|
||||
let mut body = serde_json::json!({
|
||||
"png_base64": base64::engine::general_purpose::STANDARD.encode(&stdout)
|
||||
});
|
||||
if let Ok(meta) = self.pointer_state(id, target).await {
|
||||
if let serde_json::Value::Object(map) = meta {
|
||||
if let Some(obj) = body.as_object_mut() {
|
||||
if let (Some(x), Some(y)) = (map.get("x"), map.get("y")) {
|
||||
obj.insert("cursor".into(), serde_json::json!({ "x": x, "y": y }));
|
||||
}
|
||||
if map
|
||||
.get("id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.is_some_and(|id| !id.is_empty())
|
||||
{
|
||||
obj.insert(
|
||||
"activeWindow".into(),
|
||||
serde_json::json!({ "id": map.get("id"), "title": map.get("title") }),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
body
|
||||
};
|
||||
self.attach_window_elements(id, target, &mut body).await;
|
||||
ObservePayload::from_json(body)
|
||||
}
|
||||
|
||||
async fn attach_window_elements(
|
||||
&self,
|
||||
id: &str,
|
||||
target: &ScreenTarget,
|
||||
body: &mut serde_json::Value,
|
||||
) {
|
||||
if body
|
||||
.get("elements")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.is_some_and(|items| !items.is_empty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
let Ok(result) = self
|
||||
.exec_argv(id, &window_list_command_on(&target.display), None, target)
|
||||
.await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if result.code != 0 {
|
||||
return;
|
||||
}
|
||||
if let Ok(elements) = serde_json::from_str::<serde_json::Value>(&result.stdout) {
|
||||
body["elements"] = elements;
|
||||
}
|
||||
}
|
||||
|
||||
async fn pointer_state(
|
||||
&self,
|
||||
id: &str,
|
||||
target: &ScreenTarget,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
let result = self
|
||||
.exec_argv(id, &pointer_state_command_on(&target.display), None, target)
|
||||
.await?;
|
||||
if result.code != 0 {
|
||||
return Err(result.stderr);
|
||||
}
|
||||
serde_json::from_str(&result.stdout).map_err(|error| error.to_string())
|
||||
ObservePayload::from_json(self.control_observe_json(id, target).await?)
|
||||
}
|
||||
|
||||
pub async fn act(&self, id: &str, request: ActionRequest) -> Result<serde_json::Value, String> {
|
||||
|
|
@ -428,6 +312,25 @@ impl DockerHost {
|
|||
self.control_act(id, &request, &target).await
|
||||
}
|
||||
|
||||
pub async fn browser(
|
||||
&self,
|
||||
id: &str,
|
||||
request: BrowserRequest,
|
||||
target: &ScreenTarget,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
self.control_browser(id, &request, target).await
|
||||
}
|
||||
|
||||
pub async fn recording(
|
||||
&self,
|
||||
id: &str,
|
||||
action: &str,
|
||||
request: RecordingRequest,
|
||||
target: &ScreenTarget,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
self.control_recording(id, action, &request, target).await
|
||||
}
|
||||
|
||||
pub async fn screen_url(&self, id: &str, interactive: bool) -> Result<String, String> {
|
||||
self.screen_url_for(id, 0, interactive).await
|
||||
}
|
||||
|
|
@ -439,11 +342,75 @@ impl DockerHost {
|
|||
interactive: bool,
|
||||
) -> Result<String, String> {
|
||||
let layout = screen_layout(slot).map_err(|error| error.to_string())?;
|
||||
if let Some(network) = screen_network() {
|
||||
match self.screen_container_name(id, &network).await {
|
||||
Ok(name) => {
|
||||
let authority = format!("{name}:{}", layout.view_port);
|
||||
return Ok(view_url(&authority, interactive));
|
||||
}
|
||||
Err(error) => tracing::warn!(
|
||||
"screen network {network} unusable for {id}: {error}; using host ports"
|
||||
),
|
||||
}
|
||||
}
|
||||
let port = self.published_host_port(id, layout.view_port).await?;
|
||||
let view = if interactive { "false" } else { "true" };
|
||||
Ok(format!(
|
||||
"http://127.0.0.1:{port}/vnc_lite.html?resize=scale&view_only={view}"
|
||||
))
|
||||
Ok(view_url(&format!("127.0.0.1:{port}"), interactive))
|
||||
}
|
||||
|
||||
/// Resolves the computer's container name and joins it to the shared screen
|
||||
/// network on demand, so computers started before that network existed keep
|
||||
/// working without reprovisioning.
|
||||
async fn screen_container_name(&self, id: &str, network: &str) -> Result<String, String> {
|
||||
for _ in 0..20 {
|
||||
let info = self
|
||||
.docker
|
||||
.inspect_container(id, None)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
if info.state.as_ref().and_then(|state| state.running) == Some(true) {
|
||||
let name = info.name.unwrap_or_default().trim_matches('/').to_string();
|
||||
if name.is_empty() {
|
||||
return Err("computer container has no name".into());
|
||||
}
|
||||
let attached = info
|
||||
.network_settings
|
||||
.as_ref()
|
||||
.and_then(|settings| settings.networks.as_ref())
|
||||
.is_some_and(|networks| networks.contains_key(network));
|
||||
if !attached {
|
||||
self.attach_screen_network(&name, network).await?;
|
||||
}
|
||||
return Ok(name);
|
||||
}
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
Err("computer is not running".into())
|
||||
}
|
||||
|
||||
async fn attach_screen_network(&self, name: &str, network: &str) -> Result<(), String> {
|
||||
// Refuse a missing network before connect: Docker can otherwise retain
|
||||
// a broken attachment that prevents the container's next restart.
|
||||
self.docker
|
||||
.inspect_network::<String>(network, None)
|
||||
.await
|
||||
.map_err(|error| format!("screen network {network} is unavailable: {error}"))?;
|
||||
let result = self
|
||||
.docker
|
||||
.connect_network(
|
||||
network,
|
||||
ConnectNetworkOptions {
|
||||
container: name.to_string(),
|
||||
endpoint_config: EndpointSettings::default(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
if let Err(error) = result {
|
||||
let text = error.to_string();
|
||||
if !text.to_lowercase().contains("already") {
|
||||
return Err(format!("attach {name} to screen network {network}: {text}"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn published_host_port(&self, id: &str, view_port: u16) -> Result<String, String> {
|
||||
|
|
@ -482,9 +449,11 @@ impl DockerHost {
|
|||
) -> Result<EnsureScreenResult, String> {
|
||||
let layout = screen_layout(request.slot).map_err(|error| error.to_string())?;
|
||||
let script = format!(
|
||||
"lazyboy-screen ensure {} {}",
|
||||
"lazyboy-screen ensure {} {} {} {}",
|
||||
request.slot,
|
||||
shell_single_quote(&request.profile_path)
|
||||
shell_single_quote(&request.profile_path),
|
||||
shell_single_quote(&request.bot_name),
|
||||
shell_single_quote(&request.bot_color)
|
||||
);
|
||||
let result = self
|
||||
.exec_argv(
|
||||
|
|
@ -984,6 +953,87 @@ PY"#,
|
|||
}
|
||||
serde_json::from_str(&result.stdout).map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
async fn control_browser(
|
||||
&self,
|
||||
id: &str,
|
||||
request: &BrowserRequest,
|
||||
target: &ScreenTarget,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
let payload = serde_json::to_string(request).map_err(|error| error.to_string())?;
|
||||
let token = self.container_control_token(id).await?;
|
||||
let timeout = if request.action == "click" {
|
||||
"140"
|
||||
} else {
|
||||
"30"
|
||||
};
|
||||
let mut argv = vec![
|
||||
"curl".into(),
|
||||
"-fsS".into(),
|
||||
"--max-time".into(),
|
||||
timeout.into(),
|
||||
"-H".into(),
|
||||
format!("Authorization: Bearer {token}"),
|
||||
"-H".into(),
|
||||
format!("x-lazyboy-display: {}", target.display),
|
||||
"-H".into(),
|
||||
"content-type: application/json".into(),
|
||||
];
|
||||
if let Some(profile) = &target.profile_path {
|
||||
argv.extend(["-H".into(), format!("x-lazyboy-profile: {profile}")]);
|
||||
}
|
||||
argv.extend([
|
||||
"--data-binary".into(),
|
||||
"@-".into(),
|
||||
"http://127.0.0.1:7070/browser".into(),
|
||||
]);
|
||||
let result = self
|
||||
.exec_raw_cmd(id, &argv, None, target, Some(payload))
|
||||
.await?;
|
||||
if result.code != 0 {
|
||||
return Err(result.stderr);
|
||||
}
|
||||
serde_json::from_str(&result.stdout).map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
async fn control_recording(
|
||||
&self,
|
||||
id: &str,
|
||||
action: &str,
|
||||
request: &RecordingRequest,
|
||||
target: &ScreenTarget,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
let payload = serde_json::to_string(request).map_err(|error| error.to_string())?;
|
||||
let token = self.container_control_token(id).await?;
|
||||
let timeout = if action == "collect" { "30" } else { "20" };
|
||||
let mut argv = vec![
|
||||
"curl".into(),
|
||||
"-fsS".into(),
|
||||
"--max-time".into(),
|
||||
timeout.into(),
|
||||
"-H".into(),
|
||||
format!("Authorization: Bearer {token}"),
|
||||
"-H".into(),
|
||||
format!("x-lazyboy-display: {}", target.display),
|
||||
"-H".into(),
|
||||
"content-type: application/json".into(),
|
||||
];
|
||||
if let Some(profile) = &target.profile_path {
|
||||
argv.extend(["-H".into(), format!("x-lazyboy-profile: {profile}")]);
|
||||
}
|
||||
argv.extend([
|
||||
"--data-binary".into(),
|
||||
"@-".into(),
|
||||
format!("http://127.0.0.1:7070/recording/{action}"),
|
||||
]);
|
||||
let result = self
|
||||
.exec_raw_cmd(id, &argv, None, target, Some(payload))
|
||||
.await?;
|
||||
if result.code != 0 {
|
||||
return Err(result.stderr);
|
||||
}
|
||||
serde_json::from_str(&result.stdout).map_err(|error| error.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn image_ids_match(wanted: &str, have: &str) -> bool {
|
||||
|
|
@ -1031,7 +1081,10 @@ fn computer_pids_limit() -> i64 {
|
|||
}
|
||||
|
||||
fn computer_sudo_enabled() -> bool {
|
||||
matches!(std::env::var("LAZYBOY_COMPUTER_SUDO").as_deref(), Ok("1" | "true" | "yes"))
|
||||
matches!(
|
||||
std::env::var("LAZYBOY_COMPUTER_SUDO").as_deref(),
|
||||
Ok("1" | "true" | "yes")
|
||||
)
|
||||
}
|
||||
|
||||
/// LXCFS supplies cgroup-aware /proc views so tools such as htop and free
|
||||
|
|
@ -1071,6 +1124,21 @@ fn network_name(home_key: &str) -> String {
|
|||
format!("lbnet-{}", container_name(home_key))
|
||||
}
|
||||
|
||||
/// `LAZYBOY_SCREEN_NETWORK` places the API and the computer containers on one
|
||||
/// shared Docker network. Without it the desktop proxy must reach published host
|
||||
/// ports, which bind the host loopback and are unreachable from another container.
|
||||
fn screen_network() -> Option<String> {
|
||||
std::env::var("LAZYBOY_SCREEN_NETWORK")
|
||||
.ok()
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn view_url(authority: &str, interactive: bool) -> String {
|
||||
let view = if interactive { "false" } else { "true" };
|
||||
format!("http://{authority}/vnc_lite.html?resize=scale&view_only={view}")
|
||||
}
|
||||
|
||||
fn shell_single_quote(value: &str) -> String {
|
||||
format!("'{}'", value.replace('\'', r#"'"'"'"#))
|
||||
}
|
||||
|
|
@ -1097,3 +1165,20 @@ mod credential_tests {
|
|||
assert_eq!(a, scoped_control_token(master, "a"));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod screen_url_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn desktop_urls_keep_authority_and_view_mode() {
|
||||
assert_eq!(
|
||||
view_url("lb-team-local-space:6080", false),
|
||||
"http://lb-team-local-space:6080/vnc_lite.html?resize=scale&view_only=true"
|
||||
);
|
||||
assert_eq!(
|
||||
view_url("127.0.0.1:32905", true),
|
||||
"http://127.0.0.1:32905/vnc_lite.html?resize=scale&view_only=false"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ use axum::routing::{delete, get, post};
|
|||
use axum::{Json, Router};
|
||||
use docker::DockerHost;
|
||||
use lazyboy_control::{
|
||||
ActionRequest, CommandRequest, EnsureScreenRequest, HOME, ScreenTarget,
|
||||
normalize_workspace_path,
|
||||
ActionRequest, BrowserRequest, CommandRequest, EnsureScreenRequest, HOME, RecordingRequest,
|
||||
ScreenTarget, normalize_workspace_path,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
|
@ -77,6 +77,10 @@ async fn main() {
|
|||
.route("/computers/{id}/exec", post(exec))
|
||||
.route("/computers/{id}/observe", post(observe))
|
||||
.route("/computers/{id}/act", post(act))
|
||||
.route("/computers/{id}/browser", post(browser))
|
||||
.route("/computers/{id}/recording/start", post(recording_start))
|
||||
.route("/computers/{id}/recording/stop", post(recording_stop))
|
||||
.route("/computers/{id}/recording/collect", post(recording_collect))
|
||||
.route("/computers/{id}/screens", post(ensure_screen))
|
||||
.route("/computers/{id}/screen-mode", post(screen_mode))
|
||||
.route("/computers/{id}/files", get(list_files).post(write_file))
|
||||
|
|
@ -205,6 +209,99 @@ async fn act(
|
|||
Ok(Json(result))
|
||||
}
|
||||
|
||||
async fn browser(
|
||||
State(app): State<App>,
|
||||
headers: HeaderMap,
|
||||
Path(id): Path<String>,
|
||||
Json(body): Json<BrowserRequest>,
|
||||
) -> Result<Json<serde_json::Value>, StatusCode> {
|
||||
require_token(&headers, &app.token)?;
|
||||
let mut body = body;
|
||||
let target = screen_target(&headers);
|
||||
if body.display.is_none() {
|
||||
body.display = Some(target.display.clone());
|
||||
}
|
||||
if body.profile_path.is_none() {
|
||||
body.profile_path = target.profile_path.clone();
|
||||
}
|
||||
let result = app
|
||||
.docker
|
||||
.browser(&id, body, &target)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
tracing::error!("browser: {error}");
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
})?;
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
fn with_screen_target(mut body: RecordingRequest, target: &ScreenTarget) -> RecordingRequest {
|
||||
if body.display.is_none() {
|
||||
body.display = Some(target.display.clone());
|
||||
}
|
||||
if body.profile_path.is_none() {
|
||||
body.profile_path = target.profile_path.clone();
|
||||
}
|
||||
body
|
||||
}
|
||||
|
||||
async fn recording_start(
|
||||
State(app): State<App>,
|
||||
headers: HeaderMap,
|
||||
Path(id): Path<String>,
|
||||
Json(body): Json<RecordingRequest>,
|
||||
) -> Result<Json<serde_json::Value>, StatusCode> {
|
||||
require_token(&headers, &app.token)?;
|
||||
let target = screen_target(&headers);
|
||||
let result = app
|
||||
.docker
|
||||
.recording(&id, "start", with_screen_target(body, &target), &target)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
tracing::error!("recording start: {error}");
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
})?;
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
async fn recording_stop(
|
||||
State(app): State<App>,
|
||||
headers: HeaderMap,
|
||||
Path(id): Path<String>,
|
||||
Json(body): Json<RecordingRequest>,
|
||||
) -> Result<Json<serde_json::Value>, StatusCode> {
|
||||
require_token(&headers, &app.token)?;
|
||||
let target = screen_target(&headers);
|
||||
let result = app
|
||||
.docker
|
||||
.recording(&id, "stop", with_screen_target(body, &target), &target)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
tracing::error!("recording stop: {error}");
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
})?;
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
async fn recording_collect(
|
||||
State(app): State<App>,
|
||||
headers: HeaderMap,
|
||||
Path(id): Path<String>,
|
||||
Json(body): Json<RecordingRequest>,
|
||||
) -> Result<Json<serde_json::Value>, StatusCode> {
|
||||
require_token(&headers, &app.token)?;
|
||||
let target = screen_target(&headers);
|
||||
let result = app
|
||||
.docker
|
||||
.recording(&id, "collect", with_screen_target(body, &target), &target)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
tracing::error!("recording collect: {error}");
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
})?;
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ScreenModeBody {
|
||||
interactive: bool,
|
||||
|
|
@ -387,11 +484,10 @@ async fn managed_boundary(
|
|||
.path()
|
||||
.strip_prefix("/computers/")
|
||||
.and_then(|p| p.split('/').next())
|
||||
&& app.docker.container_control_token(id).await.is_err()
|
||||
{
|
||||
if app.docker.container_control_token(id).await.is_err() {
|
||||
return StatusCode::NOT_FOUND.into_response();
|
||||
}
|
||||
}
|
||||
}
|
||||
next.run(req).await
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
# Supply-chain policy for the LazyBoy workspace, checked by `make audit`.
|
||||
# cargo-deny is not part of the Rust toolchain:
|
||||
# cargo install --locked cargo-deny (or grab a release binary)
|
||||
#
|
||||
# Three things are enforced: known vulnerabilities (RustSec), the license set we
|
||||
# are willing to ship, and where dependencies may come from.
|
||||
|
||||
[graph]
|
||||
targets = [
|
||||
{ triple = "x86_64-unknown-linux-gnu" },
|
||||
{ triple = "aarch64-unknown-linux-gnu" },
|
||||
]
|
||||
|
||||
[advisories]
|
||||
yanked = "deny"
|
||||
ignore = [
|
||||
# `paste` is an eager-macro helper pulled in by fastembed -> tokenizers. It is
|
||||
# archived upstream (no safe upgrade), only used while compiling macros, and we
|
||||
# keep embeddings optional. Re-check on the next fastembed/tokenizers bump.
|
||||
{ id = "RUSTSEC-2024-0436", reason = "transitive proc-macro helper behind fastembed; unmaintained, no upgrade available" },
|
||||
]
|
||||
|
||||
[licenses]
|
||||
# Permissive set only; copyleft candidates (MPL-2.0) stay file-level.
|
||||
confidence-threshold = 0.93
|
||||
allow = [
|
||||
"0BSD",
|
||||
"Apache-2.0",
|
||||
"Apache-2.0 WITH LLVM-exception",
|
||||
"BSD-2-Clause",
|
||||
"BSD-3-Clause",
|
||||
"BSL-1.0",
|
||||
"CC0-1.0",
|
||||
"CDLA-Permissive-2.0",
|
||||
"ISC",
|
||||
"MIT",
|
||||
"MIT-0",
|
||||
"MPL-2.0",
|
||||
"Unicode-3.0",
|
||||
"Unlicense",
|
||||
"Zlib",
|
||||
]
|
||||
|
||||
[bans]
|
||||
# Duplicate major/minor versions are normal in an application graph (they are
|
||||
# compiled once each); we only want to know when a dependency disappears.
|
||||
multiple-versions = "allow"
|
||||
wildcards = "deny"
|
||||
# Workspace members are path dependencies and carry no version requirement.
|
||||
allow-workspace = true
|
||||
allow-wildcard-paths = true
|
||||
highlight = "all"
|
||||
|
||||
[sources]
|
||||
unknown-registry = "deny"
|
||||
unknown-git = "deny"
|
||||
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
|
||||
allow-git = []
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
# Optional separate Cua image tag; the base stack also uses Cua exclusively.
|
||||
# docker compose -f docker-compose.yml -f docker-compose.cua.yml up -d --build
|
||||
services:
|
||||
computer:
|
||||
image: lazyboy/computer:cua
|
||||
supervisor:
|
||||
environment:
|
||||
LAZYBOY_COMPUTER_DRIVER: cua
|
||||
LAZYBOY_COMPUTER_IMAGE: lazyboy/computer:cua
|
||||
|
|
@ -1,5 +1,14 @@
|
|||
# Opt-in host access for local Rust development only.
|
||||
# The project `database` network is internal, and Docker silently ignores published
|
||||
# ports for containers attached only to internal networks. Postgres therefore also
|
||||
# joins this host bridge, so 127.0.0.1:5434 really listens for `cargo test`.
|
||||
services:
|
||||
postgres:
|
||||
ports:
|
||||
- "127.0.0.1:5434:5432"
|
||||
networks:
|
||||
- database
|
||||
- dev_host
|
||||
networks:
|
||||
dev_host:
|
||||
name: lazyboy_dev_host
|
||||
|
|
|
|||
|
|
@ -51,10 +51,12 @@ services:
|
|||
LAZYBOY_COMPUTER_MEMORY_MB: ${LAZYBOY_COMPUTER_MEMORY_MB:-2048}
|
||||
LAZYBOY_COMPUTER_PIDS: ${LAZYBOY_COMPUTER_PIDS:-2048}
|
||||
LAZYBOY_COMPUTER_SUDO: ${LAZYBOY_COMPUTER_SUDO:-false}
|
||||
LAZYBOY_COMPUTER_DRIVER: cua
|
||||
LAZYBOY_LXCFS_ROOT: /var/lib/lxcfs
|
||||
SUPERVISOR_BIND: 0.0.0.0:7091
|
||||
DATA_DIR: /data
|
||||
HOST_DATA_DIR: ${LAZYBOY_HOST_DATA_DIR:-${PWD}/data}
|
||||
LAZYBOY_SCREEN_NETWORK: ${LAZYBOY_SCREEN_NETWORK:-lazyboy_screen}
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- ./data:/data
|
||||
|
|
@ -68,7 +70,7 @@ services:
|
|||
api:
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
networks: [database, control, egress]
|
||||
networks: [database, control, egress, screen]
|
||||
logging: *bounded-logs
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
cap_drop: [ALL]
|
||||
|
|
@ -100,6 +102,11 @@ services:
|
|||
LAZYBOY_RECORDING_RETENTION_DAYS: ${LAZYBOY_RECORDING_RETENTION_DAYS:-30}
|
||||
LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS: ${LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS:-90}
|
||||
LAZYBOY_DB_WARN_MB: ${LAZYBOY_DB_WARN_MB:-1024}
|
||||
LAZYBOY_RUN_SOFT_TURNS: ${LAZYBOY_RUN_SOFT_TURNS:-60}
|
||||
LAZYBOY_RUN_SOFT_EVERY: ${LAZYBOY_RUN_SOFT_EVERY:-120}
|
||||
LAZYBOY_RUN_CAP_TURNS: ${LAZYBOY_RUN_CAP_TURNS:-1000}
|
||||
LAZYBOY_RUN_SOFT_MINUTES: ${LAZYBOY_RUN_SOFT_MINUTES:-75}
|
||||
LAZYBOY_RUN_HARD_MINUTES: ${LAZYBOY_RUN_HARD_MINUTES:-240}
|
||||
LAZYBOY_WEB_DIR: /web
|
||||
LAZYBOY_SCREEN_UPSTREAM: host.docker.internal
|
||||
ORT_DYLIB_PATH: /usr/local/lib/libonnxruntime.so
|
||||
|
|
@ -124,3 +131,7 @@ networks:
|
|||
control:
|
||||
internal: true
|
||||
egress: {}
|
||||
# API ↔ 電腦 noVNC 專用。桌面代理因此不必繞經只綁主機 loopback 的發布埠。
|
||||
screen:
|
||||
name: ${LAZYBOY_SCREEN_NETWORK:-lazyboy_screen}
|
||||
internal: true
|
||||
|
|
|
|||
|
|
@ -0,0 +1,85 @@
|
|||
# Agent cursor on the shared desktop
|
||||
|
||||
The computer image enables Cua's native cursor overlay. Cua draws it in X11, so
|
||||
both the embedded viewer and an enlarged/direct VNC viewer receive the same
|
||||
cursor and name. There is no separate frontend cursor or coordinate replay.
|
||||
|
||||
The API supplies the bot's current name and `avatarColor` when ensuring/restoring its screen. The
|
||||
screen launcher atomically writes these settings in the display's temporary runtime
|
||||
directory. The Cua client uses that name as its public session label, keeping it
|
||||
consistent across observation, native/browser input, and session revival.
|
||||
Display sockets provide isolation even when two agents have the same name.
|
||||
Renaming starts a new public session: take a fresh browser snapshot before using
|
||||
browser references again. Mutations using stale references are not replayed.
|
||||
Calls that explicitly provide a session retain their explicit label; unnamed
|
||||
screens retain `lazyboy-N` for compatibility.
|
||||
|
||||
Cua owns the cursor animation, action feedback, idle hiding, and label truncation.
|
||||
Long names may be shortened in its badge. The cursor is a synthetic agent cursor,
|
||||
independent of the viewer's local mouse pointer. No extra input is injected to
|
||||
animate it.
|
||||
|
||||
## Selected agent colors
|
||||
|
||||
The cursor fill, glow, and name badge use the agent's existing `#RRGGBB` color.
|
||||
Saving the agent's appearance also refreshes the color on an existing running
|
||||
screen. This cosmetic command does not start a stopped computer, focus an app,
|
||||
rename the Cua session, or replay input. Ensure/restore reapplies the persisted
|
||||
setting if the running desktop could not be reached during a save.
|
||||
|
||||
`image/computer/cua-color.patch` adds a small override to Cua's shared color
|
||||
function. The launcher sets `LAZYBOY_CURSOR_COLOR_FILE` separately for each display
|
||||
daemon. `cua-color.rs` reads and caches that bounded hex value for 100 ms; missing
|
||||
or invalid values retain Cua's original palette. Both cursor artwork and its
|
||||
badge already consume the shared color function, so their styling stays aligned.
|
||||
Names/session identities do not change when the color changes.
|
||||
|
||||
## Chinese names
|
||||
|
||||
Official Cua Driver 0.23.2 binaries embed Inter in their session badge renderer.
|
||||
Inter has no Chinese glyphs, and the renderer does not consult fontconfig. Merely
|
||||
installing system CJK fonts does not fix this.
|
||||
|
||||
The computer Dockerfile builds the pinned 0.23.2 source with the existing Huninn
|
||||
2.1 UI font in that embedded asset. Both downloads have SHA-256 checks. Apart from
|
||||
the cosmetic color override, driver code and its locked dependency graph remain unchanged. The build enables
|
||||
`portal-input` and retains the release's Rust toolchain. Cua's MIT and the font's
|
||||
OFL notices ship in `/usr/share/licenses/cua-driver/`.
|
||||
|
||||
The initial image build now also compiles Cua; subsequent builds reuse its own
|
||||
Cargo cache. Existing containers require recreation from the new image to enable
|
||||
the overlay and use the embedded Chinese font. This change does not deploy or
|
||||
replace production computers automatically.
|
||||
|
||||
## Verification
|
||||
|
||||
- Workspace tests: 205 passed; the separate saved-login integration test is ignored
|
||||
by the unit-test suite.
|
||||
- Workspace Clippy with warnings denied, Rust formatting, and launcher shell syntax.
|
||||
- Two standalone color-module tests passed (hex validation, black/white, live
|
||||
refresh, missing-file fallback, and isolation between display files).
|
||||
- Complete desktop smoke passed, including native/browser input, terminal and
|
||||
clipboard Unicode, dual-display isolation, session recovery, browser-cookie
|
||||
persistence through restart, and the named-cursor test.
|
||||
- Saved-login integration passed against real HTTPS fields without submission.
|
||||
- English and Chinese labels were visually verified in desktop screenshots.
|
||||
A separate read-only VNC framebuffer capture also showed the Chinese name and
|
||||
synthetic pointer in the actual VNC stream, separate from the OS pointer.
|
||||
- VNC pixel assertions found the exact selected RGB values `#8B5CF6`, `#22C55E`,
|
||||
and `#E11D48` in the cursor while preserving the daemon PID and session name.
|
||||
- An isolated API/supervisor/database integration test verified that persisted
|
||||
`avatarColor` reaches a newly booted screen, PATCH updates the running cursor,
|
||||
and editing a stopped agent does not start its computer. Test resources were removed.
|
||||
|
||||
Verified locally on 2026-09-08, Linux arm64. Image:
|
||||
`lazyboy/computer:cua-cursor-color`,
|
||||
`sha256:5c6e7f6befa7d9b69d6b6ff56a29692dd08854ad1273e939020ac06370ee85e6`.
|
||||
Production services and existing application computers have not been replaced.
|
||||
Deploy the updated API/supervisor and recreate desktops from this image together.
|
||||
|
||||
Color verification: `/tmp/lazyboy-cursor-color-tests.log`,
|
||||
`/tmp/lazyboy-cursor-color-clippy.log`, `/tmp/lazyboy-cursor-color-smoke.log`,
|
||||
`/tmp/lazyboy-cursor-color-api-test.log`, and
|
||||
`/tmp/lazyboy-cursor-color-smoke/cua-cursor-green.png`.
|
||||
The preceding name/font verification and saved-login run remain in
|
||||
`/tmp/lazyboy-cursor-smoke.log` and `/tmp/lazyboy-cursor-login.log`.
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
# AI 使用體驗
|
||||
|
||||
[← 回到 README](../README.zh-TW.md)
|
||||
|
||||
這份文件記錄 LazyBoy 怎麼讓「AI 用電腦」這件事變得跟人一樣順，以及背後的取捨。
|
||||
三個主題其實是同一件事：系統如果把 AI 使用電腦當成一系列無狀態的請求，AI 就必須每輪重新
|
||||
交代自己在哪裡、在做什麼，界面也只能用輪詢去猜現在發生了什麼。
|
||||
|
||||
| 主題 | 一句話 | 細節 |
|
||||
| --- | --- | --- |
|
||||
| 任務長度 | 沒有輪數額度，只有偵測鬼打牆的保險絲 | [輪次政策](./operations.md#任務跑多久輪次政策) |
|
||||
| 容器內終端機 | 一台活的 tmux，人和 AI 看同一個 shell | [終端機](./operations.md#容器內終端機) |
|
||||
| 聊天即時性 | 事件推送，輪詢只是兜底 | 本文件下面 |
|
||||
|
||||
## 輪數不是額度，是偵錯工具
|
||||
|
||||
以前一個 run 固定 40 輪，等於假設所有任務一樣長：洗資料這種事情做不完，但也不會因此變成
|
||||
錯誤，只是被腰斬。現在的正常結局是**把事情做完**（模型提出驗證，或明白說它卡在哪裡），會
|
||||
被停下來的只有鬼打牆：同一個會改變狀態的動作連做六次、同一個錯誤連錯八次、連續十四個動作
|
||||
全部失敗。觀察、等待、輪詢不算重複動作，所以等一個長 build、等下載、輪詢佇列都不會被打斷。
|
||||
|
||||
設計上要記住兩件事：
|
||||
|
||||
- 提示（「你已經做同一件事三次了」）永远不會停掉任務，只有重複到像故障才會暫停，暫停也是停在
|
||||
現況、按「繼續」接下去，不重來。
|
||||
- 保險絲（`budget_exhausted`）代表迴圈失控，是故障不是成績；它照樣可以續跑，但值得去看執行記錄。
|
||||
|
||||
純聊天仍然是 4 輪上限：那是避免模型在閒聊裡燒額度，跟任務長度無關。
|
||||
|
||||
## 終端機是一台活的 tmux
|
||||
|
||||
`shell` 透過 Cua 在共用 VNC 桌面上開啟有名稱的終端機。相同 `session` 保留工作目錄、
|
||||
環境變數與背景工作；`keys: "C-c"` 會在該終端機送出中斷，`reset` 會換成乾淨的登入 shell。
|
||||
|
||||
輸出以桌面截圖回傳。模型必須確認提示字元已回來才能輸入下一條命令；省略 `command`
|
||||
即可再次查看，長輸出可以透過 Cua 捲動終端機。`wait_ms` 預設 1000、上限 10000，
|
||||
等待結束不會終止命令。終端機若被關閉，下次呼叫會重新開啟，已關閉 shell 的環境不會還原。
|
||||
|
||||
檔案列出、分頁讀取與寫入也使用這個可見終端機。所有這些工具都需要視覺模型與桌面控制鎖。
|
||||
|
||||
## 聊天是推送，不是輪詢
|
||||
|
||||
事件早就寫進 `events` 表，也有 SSE 端點，但兩端都在猜：伺服器每 750 ms 掃一次資料庫，瀏覽器
|
||||
每 2 秒重刷一次全部狀態（訊息、電腦狀態、螢幕、技能、心跳一起打）。所以一個回覆最快也要兩輪
|
||||
輪詢才被看見。
|
||||
|
||||
現在的管線：
|
||||
|
||||
1. 寫事件的事實不變——`events` 表仍然是順序與重放的依據（游標是 `threads.next_event_seq`，
|
||||
斷線重連用 `Last-Event-ID` 從資料庫重放，不會漏也不會重複）。
|
||||
2. 提交之後往進程內的 wake channel 敲一下（`WakeBus`）。SSE 端點不再固定間隔輪詢，而是等敲；
|
||||
敲不到時（多副本、讀者落後）以 5 秒兜底輪詢補上。
|
||||
3. 瀏覽器用 `EventSource` 訂閱 `/api/sessions/{id}/events`，事件在 120 ms 的窗口裡合流，
|
||||
一回合變動只刷新一次。
|
||||
|
||||
實測（本機 loopback、丟棄式資料庫、同一台機器）：
|
||||
|
||||
| 事件 | 以前 | 現在 |
|
||||
| --- | --- | --- |
|
||||
| 自己發的訊息出現在畫面 | 最多 2 秒 | 送出去後約 25 ms |
|
||||
| 「思考中」指示出現 | 最多 2 秒 | 約 32 ms（worker 也被敲醒，不再等 200 ms 定時器） |
|
||||
| 模型回覆出現 | 寫入後最多 2.75 秒 | 寫入後約 25 ms（模型本身要幾秒是另一回事） |
|
||||
|
||||
順帶把浪費掉的工作拿掉：分頁藏在後面時不再刷新（回到前景立刻補一次），心跳從 2 秒改成
|
||||
60 秒——一次心跳買的是 15 分鐘控制租約、避免 10 分鐘閒置暫停，60 秒是很寬的餘裕。
|
||||
|
||||
**壞了會怎樣。** SSE 被 proxy 擋住或斷線，瀏覽器察覺後自動回到 2 秒輪詢，等於退回以前的行為，
|
||||
不會變成不更新；伺服器端事件順序仍然只以資料庫為準，wake 不見得可靠，只是慢。
|
||||
|
||||
自己量一次：
|
||||
|
||||
```bash
|
||||
curl -N -b cookies.txt http://127.0.0.1:3101/api/sessions/<id>/events
|
||||
# 另開一個終端機送訊息，看事件幾毫秒後出現在這條串流裡
|
||||
```
|
||||
|
||||
## 已知取捨
|
||||
|
||||
- 模型回覆還是「整個完成」才出現，沒有串流 token。思考中的狀態有顯示，但要像 ChatGPT 那樣逐字
|
||||
冒出，需要把 completion 換成串流並處理中斷/接續，這是下一件事。
|
||||
- 電腦狀態（執行中／暫停、誰在控制）沒有自己的事件，只能靠兜底輪詢更新，`LIVE_POLL_MS` 目前是 4 秒；
|
||||
桌面畫面本身是即時串流，只有狀態列會慢幾秒。要真正事件化得把 `computers` 的狀態變更也寫進
|
||||
`events`（狀態是 bot 層、事件是執行緒層，要先決定寫到哪條執行緒）。
|
||||
- 滑鼠停在頭像上看到的執行記錄泡泡，開啟時仍是輪詢（它只在滑鼠停留時開啟，量很小）。
|
||||
- wake channel 是單進程的。多副本部署時，其他副本靠 5 秒安全輪詢追上；真要横向擴充應改用
|
||||
Postgres `LISTEN/NOTIFY`。
|
||||
- 持久終端機需要桌面映像檔裡的 `tmux`；舊映像檔會退回一次性 shell，`docker compose build computer`
|
||||
之後就有了。
|
||||
- 容器執行檔仍以 uid 1000、限制環境變數的方式進入容器，持久化的是 shell 狀態，不是憑證。
|
||||
|
|
@ -30,15 +30,18 @@ flowchart LR
|
|||
|
||||
</div>
|
||||
|
||||
主流程之外還有兩個重要迴圈：
|
||||
主流程之外還有三個重要迴圈：
|
||||
|
||||
1. **接管迴圈**：使用者接管時，進行中的 run 進入等待；釋放後從目前畫面重新排隊執行。
|
||||
2. **技能迴圈**：示範期間記錄控制項與頁面情境，模型整理成 playbook；往後仍在當下畫面重新尋找元素，不重播舊座標。
|
||||
3. **暫停迴圈**：動過工具的 run 不準用一句狀態結尾。模型第一次想停，先被要求對著當下畫面自我驗證；真的缺決定、缺資料時改附中斷，run 進入 `waiting_input` 並在對話留下「繼續」按鈕。使用者的下一則訊息直接接回同一個 run（`checkpoint.awaitResume`），不另開新任務；輪次預算用盡也走同一條路回報，不會送出空訊息。
|
||||
|
||||
## 系統架構
|
||||
|
||||
LazyBoy 的公開入口只有 API。Supervisor 位於 Compose 內部 control network，不直接對主機開埠；Agent 桌面也不掛載主機 Docker socket。
|
||||
|
||||
桌面畫面同樣不發布主機埠：API 透過一條 internal 的 `lazyboy_screen` 網路，直接用容器名稱連到該電腦的 websockify，再由 `/view/<bot>/` 轉給瀏覽器。這樣 API 無論跑在主機或容器內都走同一條路，也不會把沒有密碼的 VNC 暴露到主機網路。
|
||||
|
||||
```text
|
||||
Browser
|
||||
│ HTTP / WebSocket / authenticated screen proxy
|
||||
|
|
@ -56,8 +59,8 @@ lazyboy-supervisor (:7091, internal only)
|
|||
├── provision / pause / resume / stop
|
||||
├── CPU / memory / PID limits
|
||||
└── isolated computer containers
|
||||
├── Chromium + CDP
|
||||
├── XFCE + AT-SPI
|
||||
├── Cua Driver → Chromium / XFCE
|
||||
├── visible terminal + clipboard editor
|
||||
├── Xvfb + x11vnc + websockify
|
||||
└── per-computer persisted home
|
||||
```
|
||||
|
|
@ -70,7 +73,7 @@ lazyboy-supervisor (:7091, internal only)
|
|||
| `crates/api` | 對外 Axum API、Agent run、Session、排程、記憶、MCP、保險箱 |
|
||||
| `crates/harness` | 模型供應商、憑證解析與語音契約 |
|
||||
| `crates/supervisor` | Docker 電腦生命週期、隔離與資源上限 |
|
||||
| `crates/control` | CDP、AT-SPI、X11 與畫面觀察操作 |
|
||||
| `crates/control` | Cua 瀏覽器、原生視窗與共用桌面觀察；終端機與剪貼簿操作皆經 Cua |
|
||||
| `crates/controld` | 電腦容器內部的 localhost 控制服務 |
|
||||
| `crates/contracts` | 跨 crate 的 Bot、Run、Computer、Voice 資料契約 |
|
||||
| `PostgreSQL` | 對話、run、記憶、排程、憑證與保留政策 |
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
# Cua adapter timings (2026-09-07)
|
||||
|
||||
Measured on Apple Silicon, `lazyboy/computer:local`, Cua Driver 0.23.2, inside `make cua-smoke` (`--repeat 10`). These are LazyBoy `controld` HTTP calls, not raw `cua-driver` CLI.
|
||||
|
||||
| Call | n | median ms | max ms |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| `GET /controller/health` | 1 | 11 | 11 |
|
||||
| `POST /observe` | 30 | 101 | 125 |
|
||||
| `POST /act` | 40 | 403 | 744 |
|
||||
| `POST /browser` snapshot | 10 | 48 | 51 |
|
||||
| `POST /browser` click | 20 | 332 | 337 |
|
||||
| `POST /browser` type | 10 | 391 | 399 |
|
||||
| `POST /browser` navigate | 10 | 927 | 941 |
|
||||
|
||||
`/act` includes native click, batched setvalue+click, focus, and one Cua `drag`. Navigate includes the 800 ms settle in the adapter.
|
||||
|
||||
There is no legacy control-plane comparison in this run. Do not treat these numbers as a ship gate against CDP/AT-SPI.
|
||||
|
|
@ -0,0 +1,159 @@
|
|||
# Cua Driver compatibility (LazyBoy desktop)
|
||||
|
||||
> 歷史紀錄：本文描述 2026-09-07 的雙後端驗證，已非現況。目前只保留 Cua；操作方式見 [operations.md](operations.md)，最新驗證狀態見 [cua-migration-progress.md](cua-migration-progress.md)。
|
||||
|
||||
This report answers one question, from a real `make cua-smoke` run on 2026-09-07:
|
||||
|
||||
> Can Cua Driver reliably control the existing LazyBoy XFCE + Xvfb desktop container?
|
||||
|
||||
**Yes, as an opt-in backend.** A disposable `lazyboy/computer:local` desktop on 2026-09-07 (linux/arm64, Cua Driver 0.23.2) passed raw Driver smoke 10/10, the LazyBoy adapter 10/10, dual-display isolation, and Chromium cookie persistence across `docker pause` and `docker restart`. Production still defaults to `legacy`. See [cua-review.md](cua-review.md) and [cua-benchmark.md](cua-benchmark.md).
|
||||
|
||||
## Environment
|
||||
|
||||
- Image: `lazyboy/computer:local` (`image/computer/Dockerfile`)
|
||||
- Distro: Debian bookworm; Driver binary follows `TARGETARCH` (`linux-arm64` or `linux-x86_64`)
|
||||
- Display: Xvfb `DISPLAY=:1` at 1280×800, XFCE (`xfwm4` compositor off, `xfce4-panel`, `xfdesktop`)
|
||||
- Accessibility: AT-SPI 2 per screen (`at-spi-bus-launcher` + `at-spi2-registryd`)
|
||||
- Browser: Debian `chromium` via `lazyboy-browser` (persistent profile, `--remote-debugging-port=9221+display`, `--force-renderer-accessibility`, `--lang=zh-TW`)
|
||||
- Cua Driver: **0.23.2** (`cua-driver-rs-v0.23.2`; linux-arm64 SHA256 `be22768a207796a4bc1de50c52f32f9ef680b5e86e58c059e02eec2caba2e7bb`, linux-x86_64 SHA256 `01bf8339ec129cc00f4b4b2c6056ef1a7c5b52df39ff83ad17c9b16818aec500`)
|
||||
- Install path: `/usr/local/lib/cua-driver` + `/usr/local/bin/cua-driver` (not under the persisted `/home/lazyboy` bind)
|
||||
- Daemon: `cua-driver serve --grant existing-profile --socket /tmp/lazyboy/cua.sock --no-overlay` on the primary display only
|
||||
- Telemetry: disabled
|
||||
- How to reproduce: `make cua-smoke` (artifacts in `/tmp/lazyboy-cua-smoke-last/`)
|
||||
|
||||
## Doctor
|
||||
|
||||
`cua-driver doctor --json` exit 0, `ok: true`.
|
||||
|
||||
| Probe | Status | Note |
|
||||
| --- | --- | --- |
|
||||
| binary | ok | `cua-driver 0.23.2 (x86_64-linux)` |
|
||||
| install dir | ok | `/usr/local/lib/cua-driver/cua-driver` |
|
||||
| telemetry | ok | disabled |
|
||||
| display server | ok | X11 `DISPLAY=:1` |
|
||||
| X11 connection | ok | connected, visible top-level windows |
|
||||
| AT-SPI | **warn** | CLI `doctor` (docker exec) does not always see the XFCE session bus. The **daemon** started from `lazyboy-screen` does: native `get_window_state` + AT-SPI click/type worked 10/10. |
|
||||
|
||||
`cua-driver status --socket /tmp/lazyboy/cua.sock`: daemon running, permission mode `standard`. Unix socket rejects uid 0; smoke and future controld calls must run as uid 1000 (`lazyboy`).
|
||||
|
||||
## Results (10 consecutive iterations)
|
||||
|
||||
Independent application state, not Cua `"ok"`:
|
||||
|
||||
| Check | Result |
|
||||
| --- | --- |
|
||||
| `cua-driver --version` | `cua-driver 0.23.2` |
|
||||
| screenshot (`get_desktop_state`) | 10/10 PNG of the XFCE desktop |
|
||||
| window / accessibility observation | 10/10 `list_windows` + GTK `get_window_state` |
|
||||
| native click | 10/10 GTK `Smoke Click` wrote `/tmp/lazyboy/cua-smoke-clicked` |
|
||||
| native type | 10/10 GTK entry + `Smoke Save` wrote `hello-cua` |
|
||||
| Chromium attach (existing window/profile) | 10/10 `browser_prepare` `attached_existing_profile` |
|
||||
| browser semantic click / type | 10/10 local `http://127.0.0.1:8765/cua-smoke.html`; DOM became `clicked-ok` then `typed:hello-cua` |
|
||||
| noVNC `:6080` still up | 10/10 |
|
||||
| leftover Cua processes | none (only `cua-driver serve`) |
|
||||
|
||||
Same Chromium **pid 334** / **window_id 29360131** across all ten iterations. `browser_prepare` side effects were all false: no isolated profile, no copy, no restart, no extra remote-debugging toggle (LazyBoy already exposes loopback CDP).
|
||||
|
||||
Element refs are snapshot-scoped (`p1:1`, `p4:1`, … `p28:1`). Reusing an old ref would be wrong; the smoke re-snapshots every action.
|
||||
|
||||
## Relevant tools (0.23.2 `list-tools`)
|
||||
|
||||
Observation / native input used by `controld`: `get_desktop_state`, `list_windows`,
|
||||
`get_window_state`, `click`, `type_text`, `press_key`, `hotkey`, `scroll`, `drag`,
|
||||
`move_cursor`, `set_value`, `bring_to_front`, `get_cursor_position`, `get_screen_size`.
|
||||
|
||||
Browser (attach only): `browser_prepare` (`strategy.kind=existing_profile`,
|
||||
`allow_launch=false`), `get_browser_state` (`semantic_v2`), `browser_navigate`
|
||||
(http/https/about only), `browser_click`, `browser_type`.
|
||||
|
||||
Lifecycle / diagnostics: `start_session`, `end_session`, `health_report`. Skill
|
||||
teaching also uses `start_recording` / `stop_recording`.
|
||||
|
||||
Not used: `mouse_button_down`, `mouse_button_up`, `mouse_drag` (held-button
|
||||
background X11 tools). LazyBoy's action DSL has no partial-pointer state, so
|
||||
`Pointer{Down}` / `Pointer{Up}` translate to `ControlError::Unsupported` instead
|
||||
of a half-pressed button nobody releases. Also unused: isolated `launch_app`
|
||||
browsers, Wayland helpers, and the deprecated `get_session_state` /
|
||||
`escalate_session` aliases. None of these names may be exposed to the LLM.
|
||||
|
||||
## Driver contract rules (enforced in `crates/control/src/cua`)
|
||||
|
||||
Each of these was confirmed against a real 0.23.2 daemon, and each one fails
|
||||
silently (exit 0) if violated:
|
||||
|
||||
1. **Repeat the `session` label on every call.** `CuaClient::call` injects
|
||||
`lazyboy-<display>` unless the caller already set one. Without it each CLI
|
||||
process gets an ephemeral `cli-<uuid>` session, so trajectory turns,
|
||||
snapshots, and browser binds never line up, and every call also emits a bogus
|
||||
`end_session` turn.
|
||||
2. **Never send `target: {kind: "desktop", display_id: "primary"}`.** The Linux
|
||||
driver rejects it with `invalid_action_target` (exit 0). Omit `target` to use
|
||||
the global input route.
|
||||
3. **Desktop `scroll` needs a point.** With `scope: "desktop"`, `x`/`y` are
|
||||
required (`missing field x`); `dispatch` aims at the pointer and falls back to
|
||||
the screen centre. `amount` is clamped to the schema range `1..=50`.
|
||||
4. **Only a JSON object proves the tool ran.** A refusal or prose banner that
|
||||
arrives with exit 0 is a failure (`decode_stdout`), never an empty success.
|
||||
5. **One escalation retry.** `background_unavailable` carries
|
||||
`escalation.recommended`; the client retries once with that `delivery_mode`
|
||||
and never loops.
|
||||
6. **`get_window_state` can be degraded.** AT-SPI intermittently answers with
|
||||
`degraded: true` and a root-only tree. Such windows are skipped and the
|
||||
observation reports `native_observation_complete: false` rather than failing
|
||||
the whole `observe`.
|
||||
7. **Nothing is validated for you.** The Linux schemas declare
|
||||
`additionalProperties: false` and numeric bounds, but 0.23.2 accepts unknown
|
||||
keys and out-of-range values anyway (a bogus key on `list_windows` and
|
||||
`scroll amount: 0` both return exit 0 with `effect: unverifiable`), so the
|
||||
bounds in `docs/cua-schemas/0.23.2/` are enforced here, by the client.
|
||||
`session` is accepted by every tool, including the ones whose own schema
|
||||
omits it (`list_windows`, `bring_to_front`, `health_report`,
|
||||
`start_recording`), which is what lets rule 1 be applied uniformly.
|
||||
8. **Socket peer uid.** `/tmp/lazyboy/cua*.sock` rejects uid 0; `controld` runs as
|
||||
the desktop user (uid 1000).
|
||||
|
||||
## Integration notes for the next PR
|
||||
|
||||
- Call Cua as uid 1000 via `cua-driver call --socket /tmp/lazyboy/cua.sock`. Root is rejected (`reject Unix peer uid 0 for runtime owned by uid 1000`).
|
||||
- `get_browser_state` on a live LazyBoy Chromium first returns `browser_consent_required` / `consumer_profile_endpoint_requires_grant`. Then `browser_prepare` with `existing_profile` attaches. Serve must keep `--grant existing-profile`. Never `allow_launch`.
|
||||
- Linux Chromium trusted CDP pointer is unavailable; smoke used `browser_click` `input_route=dom_event` and verified the DOM. Production adapter should prefer that route on this platform and treat `browser_input_trust_unavailable` as classified, not a silent xdotool fallback.
|
||||
- Extra Team screens are extra Xvfb `DISPLAY`s. This POC only runs a daemon on `:1`. Later: one socket per slot.
|
||||
- `browser_navigate` refuses `file://`; local fixtures need `http://127.0.0.1`.
|
||||
- `get_window_state` on Linux is `additionalProperties: false` — do not send macOS-only fields such as `include_accessibility_tree`.
|
||||
|
||||
## Known limits
|
||||
|
||||
- Doctor AT-SPI warn from a non-desktop D-Bus is not a daemon failure.
|
||||
- Overlay warnings (`X11 channel rejected command`) appeared in the daemon log with `--no-overlay`; they did not block actions.
|
||||
- Debian Chromium + zh-TW UI: existing-profile attach worked because CDP was already open, so Cua did not need the English setup-checkbox path.
|
||||
- Multi-screen, pause/resume, takeover, and skill recording were **not** in this POC; later PRs added controller routing, browser attach, takeover re-observe, and dual-source skill recording.
|
||||
|
||||
## Conclusion
|
||||
|
||||
Cua Driver 0.23.2 **can** control the existing LazyBoy XFCE + Xvfb container: screenshot, window/AT-SPI observation, native click/type, and Chromium semantic click/type, 10/10, without replacing the browser profile or breaking noVNC.
|
||||
|
||||
`ComputerController` is in place. Production defaults to `legacy` pending the full acceptance suite and benchmark. Set `LAZYBOY_COMPUTER_DRIVER=cua` only for explicit testing. Set `LAZYBOY_COMPUTER_DRIVER=legacy` on the supervisor (passed into each desktop container) to roll back to CDP/AT-SPI/xdotool. Recreate desktop containers after changing the flag.
|
||||
|
||||
With `cua`: `POST /observe`, `POST /act`, and `POST /browser` go through Cua Driver. The Agent-facing `browser` schema is unchanged (`snapshot` / `click` / `type` / `press` / `navigate` / `wait`); Cua attaches with `existing_profile` and maps `semantic_v2` refs (`pN:M`) onto the existing element list. After human takeover ends, the run forces a fresh `computer_observe` and drops pre-handoff ids/refs. Skill teaching starts Cua `start_recording` (no video) plus the existing CDP DOM recorder so a human noVNC demo still yields semantic click/type/navigate events; Cua trajectory turns are ingested as extra evidence and password-labelled typing is masked. `use_saved_login` still fills via CDP stdin so passwords never appear on argv. `cdp.py` / AT-SPI remain for login fill, human browser recording, and the `legacy` rollback.
|
||||
|
||||
|
||||
## Review of the current checkout (2026-09-07)
|
||||
|
||||
The locally tagged `lazyboy/computer:local` image now installs Cua Driver 0.23.2
|
||||
for the build architecture (`cua-driver 0.23.2` on linux/arm64 in this run).
|
||||
`make cua-smoke` is the acceptance entry: raw Driver smoke, adapter E2E,
|
||||
isolation, and pause/restart persistence. Production defaults remain `legacy`.
|
||||
See [the migration audit](cua-review.md).
|
||||
|
||||
One upstream caveat about that persistence claim: Chromium writes its cookie
|
||||
database on a ~30 s timer and does not flush on `SIGTERM`. A profile survives
|
||||
`docker pause` / `docker restart` once that write has landed; stopping a desktop
|
||||
seconds after a login can still lose the cookie, and no LazyBoy code controls
|
||||
the timer. `scripts/cua-smoke-test.sh` waits for the fixture cookie to reach the
|
||||
profile before it restarts, so the check measures profile persistence instead of
|
||||
the flush timer.
|
||||
|
||||
Image architectures are capped at `linux/amd64` and `linux/arm64` by upstream
|
||||
binaries: the Cua Driver ships only `linux-x86_64` / `linux-arm64` and ONNX
|
||||
Runtime only `linux-x64` / `linux-aarch64`. See
|
||||
[development.md](development.md#映像與-cpu-架構).
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
# Cua-only migration verification
|
||||
|
||||
Verified locally on 2026-09-08 (Linux arm64, Cua Driver 0.23.2). Source changes and local acceptance are complete. Production deployment is not part of this verification.
|
||||
|
||||
The subsequent named-cursor feature, Chinese badge font, and its separate image
|
||||
verification are documented in [Agent cursor on the shared desktop](agent-cursor.md).
|
||||
|
||||
## Requirements and evidence
|
||||
|
||||
| Requirement | Implementation | Verification |
|
||||
| --- | --- | --- |
|
||||
| Remove the old control implementation | Cua is the only ComputerDriver. Deleted LegacyController, direct CDP/AT-SPI Python controllers, clipboard.py, process helpers and the tmux shell. Removed xdotool/xclip packages. | Source audit; old backend names rejected; final running image contains neither binary. |
|
||||
| See what Cua is doing | Actions foreground the existing browser/native window on the bot's shared display. Shell/file tools use visible named terminals; clipboard operations use a visible GTK editor. | Native/browser/noVNC smoke; terminal and clipboard integration; actual VNC demonstration. |
|
||||
| All computer actions through Cua | Browser, native pointer/key/ref actions, launch/focus, shell/file tools, clipboard and saved-login use the Cua controller. Removed blanket browser-pixel blocking so canvas/unsupported controls can use fresh screenshot coordinates through Cua. | Adapter, terminal, clipboard, login and dual-display tests; API tool-path audit found no hidden shell execution in agent computer/file tools. |
|
||||
| Time on every conversation record | Every persisted message renders MessageTime, including attachment/chip messages. Date/time includes seconds, ISO datetime and full local-time tooltip. | Real user and assistant error messages inspected at desktop and 500px width; frontend tests/typecheck/build. |
|
||||
| Faster/reliable connections | Reuse browser bindings; start VNC before Cua; record the daemon PID; do not inherit startup locks; skip repeated network attachment; reject missing networks before connect and fall back to host ports; recover expired Cua sessions for reads without replaying mutations. | Warm ensure reuses one daemon with an available lock; missing-network fallback leaves no stale attachment; session-expiry observation/browser recovery and mutation refusal pass. |
|
||||
|
||||
Infrastructure provisioning/storage/database calls and independent connected-service MCP facilities remain. They are not alternate desktop controllers. Computer, browser, terminal and agent workspace-file interactions use Cua.
|
||||
|
||||
## Acceptance results
|
||||
|
||||
Final desktop image `lazyboy/computer:cua-work`:
|
||||
`sha256:4ad46205023694a26ff06f9e969d7c7e0ac43b98cae97a4483d2ba3379462117`
|
||||
|
||||
- `cargo test --workspace`: 204 passed; the environment-dependent login integration is ignored by default and separately passed.
|
||||
- `cargo clippy --workspace --all-targets -- -D warnings`: passed.
|
||||
- `cargo fmt --all --check` and `git diff --check`: passed.
|
||||
- Frontend: 44 tests passed; TypeScript/Vite build passed.
|
||||
- `scripts/cua-smoke-test.sh --docker --repeat 1 --image lazyboy/computer:cua-work`: passed on the final image. Covers native/browser/noVNC, terminal persistence/Unicode/interrupt/reset, clipboard Unicode/multiline/copy, two-display isolation, session expiry, and browser-cookie persistence across pause/restart.
|
||||
- `COMPUTER_IMAGE=lazyboy/computer:cua-work scripts/cua-login-test.sh`: passed. Uses a trusted local HTTPS fixture in a disposable container and the production field-filling function, checks both exact values and verifies no submission.
|
||||
- Manual VNC demonstration switched the target desktop from Chromium to its terminal. Teaching retained 2 window events and 3 screenshots, including the final frame. The target's recording was driven by human-style VNC input, not target-side Cua action calls.
|
||||
|
||||
Local measurements are samples, not production benchmarks: boot request 2.358s; warm screen URL request 0.100s; missing-network fallback 0.086s with unchanged Docker network attachments. A full container replacement/restart took 10.419s, including Docker shutdown.
|
||||
|
||||
## Driver compatibility fixes
|
||||
|
||||
- A zero CLI exit code is insufficient: `effect: refused` is treated as failure along with `status: refused`.
|
||||
- Email input can refuse Cua browser typing. Its fallback resolves one uniquely labelled visible native web entry through Cua, clicks the fresh observed bounds, selects all and pastes through Cua. Duplicate labels, browser chrome and zero-size fields are rejected.
|
||||
- Native type_text loses Unicode in terminals. Shell commands use ASCII Bash literals encoding UTF-8 bytes; general Unicode/multiline paste uses the Cua-operated clipboard editor. Zsh confirms multiline bracketed paste with another Enter.
|
||||
- The GTK helper exposes exact clipboard text through its accessibility label because the pinned driver does not return GTK entry values.
|
||||
- Expired driver sessions are revived for observations. Mutations rejected at expiry are not replayed. Expired browser bindings are classified as stale and re-bound for read requests.
|
||||
- Browser semantic clicks use `dom_event`; callers still inspect results. Unsupported controls can be operated with Cua coordinates from a fresh screenshot.
|
||||
|
||||
## Teaching and environment limits
|
||||
|
||||
Cua trajectories record driver invocations, not raw human VNC clicks/keys. Teaching retains window changes and visual keyframes. The start message and model prompt describe that accurately and require review of missing/ambiguous steps. Model failure no longer claims the skill was learned. The local environment has no real model key, so model-generated playbook quality was not tested; recording persistence and missing-key handling were verified.
|
||||
|
||||
Local API: `http://127.0.0.1:3111`; supervisor7191. The test bot `1ae00840-ceaf-4197-957d-661df677b015` is running the final image with one Cua daemon. Generated test credentials are in the local .env; no real provider credentials were used. API login sessions are in-memory, so restarting the API requires signing in again (existing behavior).
|
||||
|
||||
The disposable Postgres test database uses tmpfs on15434; the pre-existing compose database volume was left intact. The separate old `lazyboy-cua-verify` container is a UI-test viewer, not the final target desktop. Temporary smoke/login containers are removed by their scripts.
|
||||
|
||||
## Local evidence files
|
||||
|
||||
- `/tmp/lazyboy-workspace-tests.log`, `/tmp/lazyboy-workspace-clippy.log`
|
||||
- `/tmp/lazyboy-acceptance-smoke.log`, `/tmp/lazyboy-acceptance-login.log`
|
||||
- `/tmp/lazyboy-network-fallback-test.log`, `/tmp/lazyboy-session-recovery-test.log`
|
||||
- `/tmp/lazyboy-chat-time.png`, `/tmp/lazyboy-chat-mobile.png`, `/tmp/teach-vnc.png`
|
||||
- `/tmp/lazyboy-web-build-final.log`, `/tmp/lazyboy-final-frontend-tests.log`
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
# Cua 遷移檢查（2026-09-07）
|
||||
|
||||
> 歷史紀錄：本文描述 2026-09-07 的雙後端驗證，已非現況。目前只保留 Cua；操作方式見 [operations.md](operations.md)，最新驗證狀態見 [cua-migration-progress.md](cua-migration-progress.md)。
|
||||
|
||||
結論：`a.md` Phase 1 規格尚未全部勾完，但 **opt-in Cua 已可在現有 XFCE + Xvfb 桌面容器使用**。生產預設仍是 `legacy`。
|
||||
|
||||
本次在 Apple Silicon（linux/arm64）上以 `lazyboy/computer:local` + Cua Driver **0.23.2** 重跑隔離桌面驗收。
|
||||
|
||||
## 現況
|
||||
|
||||
| 規格 | 狀態 |
|
||||
| --- | --- |
|
||||
| 雙後端、Agent schema 不變 | 完成。`LAZYBOY_COMPUTER_DRIVER=legacy`（預設）或 `cua`。 |
|
||||
| Docker 安裝 | 完成。Dockerfile 依 `TARGETARCH` 安裝 linux-arm64 / linux-x86_64，checksum 固定。 |
|
||||
| `computer_observe` | 完成。截圖 + native AT-SPI 元素（`kind=a11y`、snapshot-scoped `cua:…` handle）+ 視窗列表。 |
|
||||
| `computer_act` | 完成。pointer / type / key / scroll / focus / 單次 `drag`；native ref 走 Cua `click` / `set_value`。不支援的動作明確 `Unsupported`，不再偷偷回退 legacy。 |
|
||||
| Browser | 完成。`existing_profile` attach、`semantic_v2` refs、navigate 限 http/https/about。 |
|
||||
| 多螢幕隔離 | 完成。display `:1` 的 handle 送到 `:2` 會被拒絕；各自點擊只改自己的測試程式。 |
|
||||
| pause / restart 後 Chromium cookie | 完成。adapter `--check-persistence` 在 `docker pause` 與 `docker restart` 後都通過。 |
|
||||
| noVNC | smoke 確認 `:6080` 仍可連；完整 human takeover 端到端未另開測試。 |
|
||||
| 示範錄製 | Cua `start_recording` + 既有 CDP recorder 仍在；未做真人 noVNC 示範驗收。 |
|
||||
| Benchmark 文件 | 見 [cua-benchmark.md](cua-benchmark.md)。尚未對 legacy 做對照。 |
|
||||
| 預設切到 Cua | **未做。** 完整 DoD（takeover、錄製、生產 metrics）未過前維持 legacy。 |
|
||||
|
||||
## 這輪修正
|
||||
|
||||
- Native 同一批 `computer_act` 共用一份觀察快照；先前每個動作都把 handle map 拿掉，導致 `wait` 後面的 ref 或連續兩個 ref 被當成過期。
|
||||
- 拒絕的過期 ref 不再清掉該螢幕上其他仍有效的 handle。
|
||||
- JSON `code != ok` 即使行程成功碼為 0 也當失敗（避免 drag 誤報成功）。
|
||||
- CLI JSON 改走 stdin，避免輸入文字出現在 argv。
|
||||
- 拖曳改打最小包含該點的視窗（避免點到覆蓋其上的 Chromium），focus 優先精確標題（避免 `LazyBoy Cua Smoke - Chromium` 搶走 GTK 視窗）。
|
||||
- GTK 測資把 drawing area 座標轉成螢幕座標。
|
||||
- Cua 選到但 binary 不在或 daemon 起不來會明確失敗。
|
||||
|
||||
## 驗證（本機 2026-09-07）
|
||||
|
||||
- `cargo test --locked -p lazyboy-control`：84 通過。
|
||||
- `cargo check --locked -p lazyboy-api -p lazyboy-sandbox -p lazyboy-controld`：通過。
|
||||
- `make cua-smoke`（`--repeat 10`）：
|
||||
- 原始 Driver smoke **10/10**（截圖、視窗、native click/type、Chromium attach、noVNC）
|
||||
- LazyBoy adapter **10/10**（GTK click / 中文 setvalue、過期 handle 拒絕、批次 native、drag、Chromium 表單）
|
||||
- 雙 display 隔離通過
|
||||
- pause/unpause 與 `docker restart` 後 cookie 仍在
|
||||
|
||||
## 如何啟用
|
||||
|
||||
預設不要改。要在本機明確跑 Cua：
|
||||
|
||||
```bash
|
||||
make cua-smoke
|
||||
# 或
|
||||
docker compose -f docker-compose.yml -f docker-compose.cua.yml up -d --build
|
||||
```
|
||||
|
||||
overlay 把 supervisor 的 `LAZYBOY_COMPUTER_DRIVER` 設成 `cua`，桌面映像標成 `lazyboy/computer:cua`，不會覆寫預設的 `lazyboy/computer:local` legacy 映像。改 flag 後必須重建桌面容器。
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
name: bring_to_front
|
||||
|
||||
description:
|
||||
Persistently activate a window so subsequent input lands on it. This deliberately breaks the no-foreground contract and is not part of the normal input ladder. For an ordinary `background_unavailable` response, retry only the refused action with `delivery_mode:"foreground"`; the input tool performs its own activate, act, and restore sequence. Use `bring_to_front` only for a focus-proxy surface that must remain foreground across multiple calls, such as a remote desktop session, or when repeated action-scoped activation prevents the remote surface from accepting input. X11: EWMH _NET_ACTIVE_WINDOW activation (the `wmctrl -a` equivalent, with proper timestamp handling to beat focus-stealing prevention). Wayland: activates through a target-addressable compositor adapter (wlroots foreign-toplevel or the GNOME Shell helper) and refuses when the compositor offers no safe adapter. Matches the macOS / Windows bring_to_front rung.
|
||||
|
||||
input_schema:
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"pid": {
|
||||
"type": "integer"
|
||||
},
|
||||
"window_id": {
|
||||
"description": "X11 window id (xid) to activate. If omitted, the first window of `pid` is used.",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"pid"
|
||||
],
|
||||
"type": "object"
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
name: browser_click
|
||||
|
||||
description:
|
||||
Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture. input_route="dom_event" (synthetic el.click(), ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. Refused for heuristic bindings.
|
||||
|
||||
input_schema:
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"input_route": {
|
||||
"description": "\"trusted\" (default): Input.dispatchMouseEvent. It refuses rather than foregrounding a standalone browser. \"dom_event\": synthetic full-background DOM click, only when explicitly requested. Dispatch does not prove the control activated; refresh page state and verify the expected postcondition.",
|
||||
"enum": [
|
||||
"trusted",
|
||||
"dom_event"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ref": {
|
||||
"description": "Page element ref in the p<snapshot>:<index> namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.",
|
||||
"type": "string"
|
||||
},
|
||||
"session": {
|
||||
"description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.",
|
||||
"type": "string"
|
||||
},
|
||||
"tab_id": {
|
||||
"description": "Opaque tab id from get_browser_state (session-scoped).",
|
||||
"type": "string"
|
||||
},
|
||||
"target_id": {
|
||||
"description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).",
|
||||
"type": "string"
|
||||
},
|
||||
"x": {
|
||||
"description": "Viewport x (CSS px) — alternative to ref.",
|
||||
"type": "number"
|
||||
},
|
||||
"y": {
|
||||
"description": "Viewport y (CSS px) — alternative to ref.",
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"target_id",
|
||||
"tab_id"
|
||||
],
|
||||
"type": "object"
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue