main function #1

Merged
daniel.w merged 14 commits from main into feat/test 2026-09-06 14:10:24 +00:00
127 changed files with 29736 additions and 1703 deletions

11
.cargo/config.toml Normal file
View File

@ -0,0 +1,11 @@
# Portable x86_64 codegen (SSE2 baseline). Do not set target-cpu=native.
# AVX kernels belong in CPU-dispatched libraries (Microsoft ONNX Runtime),
# not in the Rust binary itself.
[target.x86_64-unknown-linux-gnu]
rustflags = ["-C", "target-cpu=x86-64"]
[target.x86_64-apple-darwin]
rustflags = ["-C", "target-cpu=x86-64"]
[target.x86_64-pc-windows-msvc]
rustflags = ["-C", "target-cpu=x86-64"]

View File

@ -4,3 +4,13 @@ data
reference
**/*.md
apps/web/node_modules
.env
.env.*
!.env.example
**/*.pem
**/*.key
**/node_modules
apps/web/dist
**/__pycache__
.DS_Store

View File

@ -4,16 +4,33 @@ OPENAI_API_KEY=
# Generate independent values with: openssl rand -hex 32
LAZYBOY_APP_TOKEN=
SANDBOX_SUPERVISOR_TOKEN=
# Keep this key stable when rotating the app login token.
LAZYBOY_VAULT_KEY=
POSTGRES_PASSWORD=lazyboy
LAZYBOY_BIND_IP=127.0.0.1
SANDBOX_PROVIDER=docker
DATABASE_URL=postgres://lazyboy:lazyboy@127.0.0.1:5434/lazyboy
DATA_DIR=./data
API_BIND=0.0.0.0:3101
API_BIND=127.0.0.1:3101
SANDBOX_SUPERVISOR_URL=http://127.0.0.1:7091
LAZYBOY_SECURE_COOKIE=false
LAZYBOY_COMPUTER_MEMORY_MB=2048
LAZYBOY_COMPUTER_CPUS=2
LAZYBOY_COMPUTER_PIDS=2048
# Only affects the Agent desktop container. Disabled by default.
LAZYBOY_COMPUTER_SUDO=false
# Linux only (optional): point this at the host's LXCFS root to make htop/free
# report the per-Agent cgroup quota. Leave the default empty directory on macOS.
LAZYBOY_LXCFS_ROOT=./data/lxcfs
LAZYBOY_MEMORY_ENABLED=true
LAZYBOY_MEMORY_MODEL_CACHE=./data/fastembed
LAZYBOY_MEMORY_TOP_K=8
LAZYBOY_MEMORY_BYTE_BUDGET=6000
# Hourly cleanup of diagnostics; conversations and current memories are retained.
LAZYBOY_EVENT_RETENTION_DAYS=30
LAZYBOY_CHECKPOINT_RETENTION_DAYS=7
LAZYBOY_RUN_RETENTION_DAYS=90
LAZYBOY_RECORDING_RETENTION_DAYS=30
LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS=90
LAZYBOY_DB_WARN_MB=1024

3
.gitignore vendored
View File

@ -9,3 +9,6 @@ node_modules
dist
.DS_Store
.gstack/
__pycache__/
*.pyc

920
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -13,9 +13,14 @@ members = [
[workspace.package]
edition = "2024"
version = "0.1.0"
license = "MIT"
license = "Apache-2.0"
publish = false
[profile.release]
lto = true
codegen-units = 1
strip = "symbols"
[workspace.dependencies]
lazyboy-contracts = { path = "crates/contracts" }
lazyboy-control = { path = "crates/control" }
@ -36,7 +41,8 @@ tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
uuid = { version = "1", features = ["v4", "serde"] }
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] }
tokio-tungstenite = { version = "0.26", features = ["connect"] }
rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }
tokio-tungstenite = { version = "0.26", features = ["connect", "rustls-tls-native-roots"] }
futures-util = "0.3"
bollard = "0.18"
hmac = "0.12"

202
LICENSE Normal file
View File

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@ -53,26 +53,12 @@ help: ## Show this help
# --- Environment -----------------------------------------------------------
env: ## Create .env from example with fresh random tokens (no-op if .env exists)
@if [ -f .env ]; then \
echo ".env already exists — leaving it untouched (use 'make env-force' to regenerate tokens)."; \
else \
cp .env.example .env; \
APP=$$(openssl rand -hex 32); SUP=$$(openssl rand -hex 32); \
sed "s|^LAZYBOY_APP_TOKEN=.*|LAZYBOY_APP_TOKEN=$$APP|" .env > .env.tmp; \
sed "s|^SANDBOX_SUPERVISOR_TOKEN=.*|SANDBOX_SUPERVISOR_TOKEN=$$SUP|" .env.tmp > .env.tmp2; \
mv .env.tmp2 .env; rm -f .env.tmp; \
echo "created .env with generated LAZYBOY_APP_TOKEN / SANDBOX_SUPERVISOR_TOKEN (64 hex chars)."; \
echo "next: set XAI_API_KEY in .env, then run 'make up'."; \
fi
env: ## Create .env with independent random secrets (preserve existing .env)
@python3 scripts/init-env.py
env-force: ## Regenerate .env (wipes XAI_API_KEY — you will set it again)
@cp .env.example .env; \
APP=$$(openssl rand -hex 32); SUP=$$(openssl rand -hex 32); \
sed "s|^LAZYBOY_APP_TOKEN=.*|LAZYBOY_APP_TOKEN=$$APP|" .env > .env.tmp; \
sed "s|^SANDBOX_SUPERVISOR_TOKEN=.*|SANDBOX_SUPERVISOR_TOKEN=$$SUP|" .env.tmp > .env.tmp2; \
mv .env.tmp2 .env; rm -f .env.tmp; \
echo "regenerated .env tokens (XAI_API_KEY reset — set it again)."
env-force: ## Refuse destructive key regeneration; existing vault keys must be preserved
@echo "Refusing to overwrite .env: this can orphan saved passwords. See docs/security-and-harness-review.md for safe rotation."
@exit 1
# --- Full Docker stack -----------------------------------------------------
@ -104,7 +90,7 @@ computer: ## Build the Debian desktop image used to spawn bot computers
docker build -f image/computer/Dockerfile -t $(COMPUTER_IMAGE) .
postgres: ## Start only postgres and wait until it is ready
$(COMPOSE) up -d postgres
$(COMPOSE) -f docker-compose.yml -f docker-compose.dev.yml up -d postgres
@echo "waiting for postgres (127.0.0.1:5434) to be ready..."
@for i in $$(seq 1 40); do if $(COMPOSE) exec -T postgres pg_isready -U lazyboy >/dev/null 2>&1; then echo "postgres ready"; break; fi; sleep 0.5; done
@ -168,5 +154,6 @@ dev-api: ## Run lazyboy-api on the host (terminal 2)
export API_BIND="$${API_BIND:-0.0.0.0:3101}"; \
export LAZYBOY_WEB_DIR="$${LAZYBOY_WEB_DIR:-$$PWD/$(WEB_DIR)}"; \
mkdir -p "$$DATA_DIR"; \
export ORT_DYLIB_PATH="$${ORT_DYLIB_PATH:-$$(./scripts/fetch-onnxruntime.sh "$$DATA_DIR/onnxruntime")}"; \
echo "starting api on $$API_BIND (web dir $$LAZYBOY_WEB_DIR)"; \
exec cargo run -p lazyboy-api

165
README.md
View File

@ -1,72 +1,135 @@
<div align="center">
<img src="./apps/web/public/lazyboy-icon.png" width="96" height="96" alt="LazyBoy" />
**English** · [繁體中文](./README.zh-TW.md)
# LazyBoy
Create a bot in the browser, give it a Team or Private computer, and let it drive a Linux desktop.
**Give AI a computer so it can do the work.**
## Run (full Docker, recommended)
A self-hosted AI agent workspace. Assign tasks in text or voice, watch the desktop live, and take over whenever you need to.
Everything (postgres, desktop image build, supervisor, API + web UI) runs in
Docker Compose, driven by the Makefile:
[Quick start](#quick-start) · [Features](#features) · [Architecture](./docs/architecture.md) · [Operations](./docs/operations.md) · [Development](./docs/development.md)
</div>
![LazyBoy workspace: agent list, chat, and live desktop](./docs/readme-hero.png)
LazyBoy gives each agent its own Linux desktop in Docker — browser, terminal, and files. You can run several agents, put them in a group, turn a demonstration into a skill, and schedule it to run again.
This is an early `0.1.0` release with desktop and phone browser UIs. You bring your own model API key.
## Features
- **A lasting workspace**: each agent has its own chats, run history, and optional long-term memory.
- **A real computer**: open pages, use the terminal, organize files, drive the GUI — and watch it live.
- **Take over any time**: sign in, pass a check, or nudge things by hand on the same desktop, then hand it back.
- **Several agents and groups**: shared Team computers or private dedicated desktops.
- **Teach by demo, then schedule**: turn a walkthrough into a skill; use cron for repeat work.
- **Your models and tools**: xAI, OpenCode Go, OpenAI-compatible endpoints, MCP, and file skills.
- **Voice calls**: after you enable a voice provider, you can talk to the agent on a call.
- **Phone-friendly**: collapse the chat sidebar; the remote desktop has keyboard, trackpad, right-click, drag, and scroll.
- **English and Traditional Chinese**: switch the UI language in the app.
## Quick start
You need Docker and Compose, Git, Make, Python 3, and an API key for a supported model. On macOS use Docker Desktop or OrbStack; on Linux use Docker Engine.
From the repo root:
```bash
make env # creates .env with two generated 64-hex tokens (no-op if .env exists)
# then set XAI_API_KEY in .env
make up # builds all images and starts the stack
make env
```
Edit the generated `.env` and set one of `XAI_API_KEY`, `OPENCODE_GO_API_KEY`, or `OPENAI_API_KEY`. The init tool creates the login and service secrets; running it again keeps existing values.
```bash
make up
make health
```
Open `http://127.0.0.1:3101` and sign in with `LAZYBOY_APP_TOKEN`.
Open **[http://127.0.0.1:3101](http://127.0.0.1:3101)**, sign in with `LAZYBOY_APP_TOKEN` from `.env`, create an agent, and pick a model.
Useful targets (see `make help`):
The first run builds the API and Linux desktop images from source and takes a while. The build toolchain lives in the containers, so a full Docker deploy does not need Rust or Node.js on the host.
| target | what it does |
Try a concrete task:
> Open the site I name, summarize the page, and save the notes as a Markdown file in the workspace.
Status and logs:
```bash
make ps
make logs
make down # stop services, keep PostgreSQL data
```
For resource limits, environment variables, HTTPS, and in-container sudo, see [Operations](./docs/operations.md).
## On a phone
Desktop and phone share the same web UI. After you deploy on a host the phone can reach, open that URL in the phone browser. `127.0.0.1` is the phone itself — it will not reach another machine.
Tap outside the chat sidebar to collapse it. On the remote desktop you can switch between tap-to-click and trackpad, and use the toolbar for keyboard, right-click, or drag. Put the service behind HTTPS before you expose it; see [Operations](./docs/operations.md#安全模型).
## Stack
- **Frontend**: React 19, TypeScript, Vite, noVNC
- **Backend**: Rust 2024, Axum, Tokio
- **Data**: PostgreSQL, pgvector, SQLx
- **Desktop**: Docker, Debian, XFCE, Chromium, Xvfb
- **Computer control**: CDP, AT-SPI, X11
- **Extensions**: MCP, file skills, demonstration playbooks
Flow diagrams, handoff, component roles, and the computer lifecycle live in **[Architecture](./docs/architecture.md)**. The older **[interactive diagram](./docs/workflow.html)** is still there — download it and open it in a browser.
## Local development
Besides Docker, you need a Rust toolchain that supports the 2024 edition, plus Node.js / npm.
```bash
make dev
make dev-supervisor # terminal 1
make dev-api # terminal 2
```
Frontend hot reload in another terminal:
```bash
cd apps/web
npm install
npm run dev
```
Open [http://127.0.0.1:5173](http://127.0.0.1:5173). Tests and the tree layout are in the [development guide](./docs/development.md).
## Docs
The guides below are currently in Traditional Chinese.
| Doc | Contents |
| --- | --- |
| `make up` / `make down` / `make purge` | start / stop (keep data) / stop + delete postgres volume |
| `make logs`, `make ps`, `make health` | observe the stack |
| `make computer` | build only the heavy desktop image `lazyboy/computer:local` |
| `make postgres`, `make postgres-down` | start/stop just the database on `127.0.0.1:5434` |
| `make build`, `make fmt`, `make clippy`, `make test` | cargo workspace tasks |
| `make web` | build `apps/web` with npm (needs node) |
| `make dev`, `make dev-supervisor`, `make dev-api` | local dev: postgres in Docker, Rust services on the host in two terminals |
| [Architecture](./docs/architecture.md) | Task flow, system architecture, computer lifecycle |
| [Interactive diagram](./docs/workflow.html) | Zoomable, searchable HTML chart; download and open |
| [Operations](./docs/operations.md) | Resources, env vars, security, site checks, sudo |
| [Development](./docs/development.md) | Local dev, checks and tests, directory layout |
| [Env example](./.env.example) | Environment variables and defaults |
Without make, the full-stack equivalent is:
## Data
```bash
cp .env.example .env # fill in the two tokens + XAI_API_KEY
docker compose up -d --build
```
Chats, memory, browser profiles, and encrypted credentials stay on your host. When you use an external model, the prompts, tool results, and screenshots the task needs may still be sent to that provider.
## Run (local)
---
Postgres is on `127.0.0.1:5434` so it does not collide with other stacks.
<div align="center">
```bash
cp .env.example .env
# Set XAI_API_KEY, then generate two different secrets:
# openssl rand -hex 32
# openssl rand -hex 32
# Put them in LAZYBOY_APP_TOKEN and SANDBOX_SUPERVISOR_TOKEN.
docker compose up -d postgres
./scripts/build-computer-image.sh
SANDBOX_SUPERVISOR_TOKEN=<same-strong-supervisor-token> \
DATA_DIR=/root/LazyBoy/data cargo run -p lazyboy-supervisor
DATABASE_URL=postgres://lazyboy:lazyboy@127.0.0.1:5434/lazyboy \
SANDBOX_PROVIDER=docker \
SANDBOX_SUPERVISOR_URL=http://127.0.0.1:7091 \
SANDBOX_SUPERVISOR_TOKEN=<same-strong-supervisor-token> \
LAZYBOY_APP_TOKEN=<strong-app-token> \
DATA_DIR=/root/LazyBoy/data \
LAZYBOY_WEB_DIR=apps/web \
API_BIND=0.0.0.0:3101 \
cargo run -p lazyboy-api
```
**Daniel Wang** <img src="https://flagcdn.com/w20/tw.png" width="20" alt="Taiwan" />
Open `http://<host>:3101` and sign in with `LAZYBOY_APP_TOKEN`. The computer is a real Debian container: fluxbox toolbar, Chromium with tabs and URL bar, and xterm. The display is proxied through the authenticated API; the supervisor is internal-only in Docker Compose and must not be published to the LAN.
[igs170911@gmail.com](mailto:igs170911@gmail.com)
For LAN use, put LazyBoy behind HTTPS whenever possible. A shared token sent over plain HTTP can be observed by other devices on an untrusted network. Set `LAZYBOY_SECURE_COOKIE=true` when HTTPS terminates at LazyBoy or a trusted reverse proxy. The API refuses a non-loopback bind unless `LAZYBOY_APP_TOKEN` is at least 32 characters; the supervisor likewise rejects missing, short, or default tokens.
<a href="https://www.buymeacoffee.com/daniel.wang.1993"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me a Coffee" height="50" /></a>
For frontend development, run `npm install && npm run dev` in `apps/web`, then open
`http://127.0.0.1:5173`. Vite proxies API and computer-screen traffic to the Rust API on port 3101.
[Apache License 2.0](./LICENSE)
The standalone supervisor listens on `127.0.0.1:7091` by default. Docker Compose reaches it internally as `supervisor:7091`; there is intentionally no host port `7092`.
Model providers: set the workspace default in 本機工作區 → 設定. v1 supports xAI (`XAI_API_KEY`), OpenCode Go (`OPENCODE_GO_API_KEY` or a key saved in settings), and a self-hosted OpenAI-compatible endpoint (base URL + optional key). Keys saved in the UI are stored on the local workspace and take precedence over env vars.
</div>

133
README.zh-TW.md Normal file
View File

@ -0,0 +1,133 @@
<div align="center">
<img src="./apps/web/public/lazyboy-icon.png" width="96" height="96" alt="LazyBoy" />
[English](./README.md) · **繁體中文**
# LazyBoy
**給 AI 一台電腦,讓它替你動手做事。**
自架的 AI Agent 工作空間。用文字或語音交代任務,即時觀看桌面操作,隨時接回控制權。
[快速開始](#快速開始) · [功能](#功能) · [架構與流程圖](./docs/architecture.md) · [部署指南](./docs/operations.md) · [開發指南](./docs/development.md)
</div>
![LazyBoy 工作空間Agent 清單、對話與即時桌面](./docs/readme-hero.png)
LazyBoy 讓 Agent 在 Docker 裡使用自己的 Linux 桌面,操作瀏覽器、終端與檔案。你可以建立不同的 Agent、在群組中協作把示範整理成技能再安排定時執行。
目前為早期版本 `0.1.0`,提供桌面與手機瀏覽器介面;模型金鑰由你自行設定。
## 功能
- **持續的工作空間**:每個 Agent 有自己的對話、工作紀錄與可設定的長期記憶。
- **真的能操作電腦**:開網頁、使用終端、整理檔案、操作圖形介面,過程可即時觀看。
- **隨時人工接管**:在同一個桌面完成登入、驗證或手動調整,再交回 Agent。
- **多 Agent 與群組**:支援 Team 共用電腦與 Private 獨立電腦模式。
- **示範教學與排程**:把操作示範整理成技能,使用 cron 安排重複工作。
- **自選模型與工具**:支援 xAI、OpenCode Go、OpenAI 相容端點,以及 MCP 與檔案技能。
- **語音通話**:設定語音服務後,可以透過通話與 Agent 互動。
- **手機操作**:可收合聊天側欄,遠端桌面提供鍵盤、觸控板、右鍵、拖曳與捲動控制。
- **英文與繁體中文**:可在介面切換語言。
## 快速開始
需要 Docker 與 Compose、Git、Make、Python 3以及一組支援的模型 API 金鑰。macOS 可使用 Docker Desktop 或 OrbStackLinux 可使用 Docker Engine。
取得本專案原始碼後,在專案根目錄執行:
```bash
make env
```
編輯產生的 `.env`,填入 `XAI_API_KEY`、`OPENCODE_GO_API_KEY` 或 `OPENAI_API_KEY` 其中之一。初始化工具會產生登入與服務所需的金鑰,重新執行時保留既有設定。
```bash
make up
make health
```
開啟 **[http://127.0.0.1:3101](http://127.0.0.1:3101)**,用 `.env` 裡的 `LAZYBOY_APP_TOKEN` 登入,建立 Agent 並選擇模型。
第一次會從原始碼建置 API 與 Linux 桌面映像,所需時間較長。容器內已包含建置環境,完整 Docker 部署不需要在主機安裝 Rust 或 Node.js。
可以先試一個具體任務:
> 開啟我指定的網站,整理頁面重點,將結果存成工作區裡的 Markdown 檔案。
服務狀態與日誌:
```bash
make ps
make logs
make down # 停止服務,保留 PostgreSQL 資料
```
資源限制、環境變數、HTTPS 與容器內 sudo 設定,請見 [部署與操作](./docs/operations.md)。
## 在手機上使用
手機與桌面使用同一個 Web 介面。將服務部署在可連線的主機後,用手機瀏覽器開啟該服務的網址即可;`127.0.0.1` 只代表手機本身,不能拿來連另一台電腦。
聊天側欄可點外側空白處收合。操作遠端桌面時,可切換直接點選與觸控板模式,使用工具列叫出鍵盤、按右鍵或拖曳。對外提供服務時請設定 HTTPS詳見 [部署指南](./docs/operations.md#安全模型)。
## 技術組成
- **前端**React 19、TypeScript、Vite、noVNC
- **後端**Rust 2024、Axum、Tokio
- **資料**PostgreSQL、pgvector、SQLx
- **桌面**Docker、Debian、XFCE、Chromium、Xvfb
- **電腦控制**CDP、AT-SPI、X11
- **擴充**MCP、檔案技能、示範 playbook
流程圖、控制權交接、元件職責與電腦生命週期狀態機,集中在 **[架構與流程](./docs/architecture.md)**。原有的 **[互動流程圖](./docs/workflow.html)** 也保留;下載後用瀏覽器開啟即可操作。
## 本機開發
除了 Docker需安裝支援 Rust 2024 edition 的 Rust 工具鏈,以及 Node.jsnpm。
```bash
make dev
make dev-supervisor # 終端 1
make dev-api # 終端 2
```
前端熱更新使用另一個終端:
```bash
cd apps/web
npm install
npm run dev
```
開啟 [http://127.0.0.1:5173](http://127.0.0.1:5173)。測試指令與專案目錄說明請見 [開發指南](./docs/development.md)。
## 文件
| 文件 | 內容 |
| --- | --- |
| [架構與流程](./docs/architecture.md) | 任務流程圖、系統架構、電腦生命週期狀態機 |
| [互動流程圖](./docs/workflow.html) | 可縮放、搜尋的 HTML 圖表;下載後開啟 |
| [部署與操作](./docs/operations.md) | 資源、環境變數、安全設定、網站驗證、sudo |
| [開發指南](./docs/development.md) | 本機開發、檢查與測試、目錄結構 |
| [設定範例](./.env.example) | 環境變數與預設值 |
## 資料
對話、記憶、瀏覽器設定檔與加密憑證保存在自架主機。使用外部模型時,任務所需的提示詞、工具結果與截圖仍可能傳送給該模型供應商。
---
<div align="center">
**danielwang** <img src="https://flagcdn.com/w20/tw.png" width="20" alt="Taiwan" />
[igs170911@gmail.com](mailto:igs170911@gmail.com)
<a href="https://www.buymeacoffee.com/daniel.wang.1993"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="請我喝杯咖啡" height="50" /></a>
[Apache License 2.0](./LICENSE)
</div>

51
a.txt
View File

@ -1,51 +0,0 @@
我這邊體感滑,是因為多數工作走 ShellAPI瀏覽器 DOM很少靠截圖猜 XY。你 repo 裡 瀏覽器這條其實已經對了;卡的是 桌面原生 App。
你已經有的(別拆)
能力 在哪
Docker 真實 Linux 桌面 + Screenleasetakeover supervisor / sandbox / control
shell 工具 crates/api/src/tools.rs
browserCDP DOM snapshotclick by idselector control/cdp.rs + cdp.py工具描述已寫「網頁優先用 browser」
編號元素、apply_element_targets contracts/action.rs、control/actions.rs
畫面沒變就不重傳圖frame signature control/observe.rs
原生動作執行xdotool / wmctrl control/x11.rs、image/computer/Dockerfile
這已經比純「截圖 computer-use」產品完整一截。
現在實際主路徑(為什麼不順)
computer_observe → 截圖 +瀏覽器時CDP 元素
→(否則)只有 wmctrl 視窗級元素
computer_act → element id 先換成中心點 XY → xdotool 點像素
關鍵缺口:
P0沒有 AT-SPI無障礙樹
Dockerfile 有 xdotool、wmctrl沒有 at-spi2python3-pyatspi 之類。
原生「elements」在 window_list_command 只是 整窗標題,不是按鈕/輸入框。模型對 Thunar、對話框、面板仍要看圖或瞎點。
P0element 最終還是 XY
apply_element_targets 填的是中心座標,執行仍走 ComputerAction::Pointer → xdotool。
視窗一動、動畫、縮放就歪。agent-desktopCua 是 對 element 做 AXAT-SPI action不是點中心像素。
P1ComputerAction 契約只有像素/鍵
action.rs 沒有 ActByRef { id }、SetValue、A11yPress 這類語意動作。擴 CDPa11y 都會卡在契約層。
P1控的是容器桌面不是你 Mac
對「操控我電腦」若指本機 Finder系統設定現在這條 stack 接不到 Mac AX。那是另一條 provideragent-desktopCua不是再調 xdotool。
P2模型路由依賴 prompt
tools 已寫「Chromium 用 browser」但若模型仍狂 computer_observe座標體感就跟我差一截。缺硬性策略例如有 CDP page 時禁止對 browser 窗做 computer_act click
跟「我為什麼滑」的對照
路徑 我Grok Bot LazyBoy feat/test
Shell檔案 預設 有
瀏覽器 DOMelement ref 有 CDP方向對
桌面原生 另有 a11y委派截圖是備援 截圖 + xdotool 仍是主幹
本機 Mac 受限/核准 Shell 尚未做 Host provider
所以:你換好幾套「控制」若不順,多半都還停在 視覺/座標層;你 repo 差的是把 AT-SPI容器內 接到 observeact並讓 browsershell 硬優先。
建議下一步(只做這 3 件)
容器加 AT-SPIimage 裝 at-spiobserve 回傳 buttontextfield 樹(精簡 YAMLJSON + stable idact 對 id 做 DoActionset text失敗才 fallback XY。
擴 ComputerActionUiElementkind: a11y|dom|windowref 語意動作computer_act 有 selectora11y ref 時不要先轉座標。
路由硬規則Chromium 前台 → 強制 browser能 shell 完成 → 禁止截圖迴圈;並加「連續同點無畫面變化 → 強制換策略」(你已有一點 stale-click 提示,可再硬一點)。
本機 Mac 若也要滑:另開 HostComputerProvideragent-desktopCua別硬塞進現在的 X11 xdotool 路徑。
要的話我可以直接幫你開一張 P0 AT-SPI observe/act 的實作規格(改哪些檔、契約長怎樣),或直接在 feat/test 上開工。

View File

@ -1 +1 @@
<!doctype html><html lang="zh-Hant"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#050506"/><title>LazyBoy</title></head><body><div id="root"></div><script type="module" src="/src/main.tsx"></script></body></html>
<!doctype html><html lang="zh-Hant"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#050506"/><title>LazyBoy</title><link rel="icon" href="/favicon.png?v=white-1" type="image/png" sizes="64x64"/><link rel="apple-touch-icon" href="/apple-touch-icon.png?v=white-1"/></head><body><div id="root"></div><script type="module" src="/src/main.tsx"></script></body></html>

File diff suppressed because it is too large Load Diff

View File

@ -11,10 +11,14 @@
"dependencies": {
"@blobatar/react": "^2.7.0",
"@fontsource/huninn": "^5.3.0",
"@novnc/novnc": "1.7.0",
"blobatar": "^2.7.0",
"react": "^19.0.0",
"react-dom": "^19.0.0",
"react-useanimations": "^2.10.0"
"react-markdown": "^10.1.0",
"react-useanimations": "^2.10.0",
"remark-breaks": "^4.0.0",
"remark-gfm": "^4.0.1"
},
"devDependencies": {
"@types/react": "^19.0.0",

Binary file not shown.

After

Width:  |  Height:  |  Size: 36 KiB

View File

@ -0,0 +1,14 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
<rect width="100" height="100" rx="22" fill="#050506"/>
<g transform="translate(4 4) scale(.92)">
<path fill="#f1f1ef" d="M84.5 48.88C84.5 66.49 68.81 81.73 50.68 81.73C32.55 81.73 16.87 66.49 16.87 48.88C16.87 31.27 32.55 16.04 50.68 16.04C68.81 16.04 84.5 31.27 84.5 48.88Z"/>
<g fill="#1b1b22">
<ellipse cx="43.16" cy="49.68" rx="4.2" ry="11.2"/>
<ellipse cx="61.4" cy="48.81" rx="5.0" ry="14.1"/>
</g>
<g fill="#f3f3f6">
<circle cx="44.6" cy="43.2" r="1.7"/>
<circle cx="63.1" cy="41.4" r="2"/>
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 629 B

BIN
apps/web/public/favicon.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.3 KiB

View File

@ -0,0 +1,13 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
<title>LazyBoy</title>
<!-- Round blobatar (traits.shape 0.11) with enlarged eyes for tab size. -->
<path fill="#f1f1ef" d="M84.5 48.88C84.5 66.49 68.81 81.73 50.68 81.73C32.55 81.73 16.87 66.49 16.87 48.88C16.87 31.27 32.55 16.04 50.68 16.04C68.81 16.04 84.5 31.27 84.5 48.88Z"/>
<g fill="#1b1b22">
<ellipse cx="43.16" cy="49.68" rx="4.2" ry="11.2"/>
<ellipse cx="61.4" cy="48.81" rx="5.0" ry="14.1"/>
</g>
<g fill="#f3f3f6">
<circle cx="44.6" cy="43.2" r="1.7"/>
<circle cx="63.1" cy="41.4" r="2"/>
</g>
</svg>

After

Width:  |  Height:  |  Size: 606 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 259 KiB

View File

@ -1,8 +1,7 @@
import { FormEvent, KeyboardEvent as ReactKeyboardEvent, useCallback, useEffect, useMemo, useRef, useState } from "react";
import { BotIcon, Brain, ChevronDown, ChevronsRight, CircleHelp, ClipboardPaste, Computer, Download, Ellipsis, Info, LogOut, Megaphone, Paperclip, Pencil, Pin, Plug, Plus, RefreshCw, Settings, Smartphone, Sparkle, Square, Upload, Users, X } from "./animated-icons";
import UseAnimations from "react-useanimations";
import UseAnimations from "./use-animations";
import loading from "react-useanimations/lib/loading";
import loading2 from "react-useanimations/lib/loading2";
import arrowUp from "react-useanimations/lib/arrowUp";
import bookmark from "react-useanimations/lib/bookmark";
import copy from "react-useanimations/lib/copy";
@ -17,25 +16,69 @@ import visibility2 from "react-useanimations/lib/visibility2";
import searchToX from "react-useanimations/lib/searchToX";
import { api, ApiError } from "./api";
import { Avatar, AvatarLookProvider, AvatarStack, BLOBATAR_BACKGROUNDS, BLOBATAR_EXPRESSIONS, BLOBATAR_SHAPES, DEFAULT_LOOK, persistBlobatarShape, readAvatarLooks, resolveBlobatarShape, writeAvatarLook, type AvatarBackground, type AvatarExpression, type AvatarLook } from "./avatar";
import { t, type MessageKey } from "./i18n";
import type { AvatarShape, Bot, ComputerMode, ComputerStatus, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, MessageFile, ModelProviderId, Playbook, PlaybookInput, PlaybookStep, Room, RoomMember, Session, TaughtSkill, WorkspaceSettings } from "./types";
import { dateLocale, getLocale, listJoin, setLocale, t, useLocale, type MessageKey } from "./i18n";
import type { AvatarShape, Bot, ComputerMode, ComputerStatus, FileSkill, McpCatalogEntry, McpServer, McpTransport, MemoryItem, Message, MessageFile, ModelProviderId, Playbook, PlaybookInput, PlaybookStep, Room, RoomMember, Session, TaughtSkill, VoiceSettings, WorkspaceSettings } from "./types";
import { ChatMarkdown, CopyMessageButton } from "./markdown";
import { ScheduleEditor, ScheduleList, cronFromPreset, defaultCronPreset, presetFromCron, scheduleWhen, type CronPreset, type ScheduleItem } from "./schedule";
import { CallOverlay, PhoneIcon } from "./call";
import { VoiceSettingsDialog } from "./voice-settings";
const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,busyStep:null,waitingRunId:null,waitingSessionId:null,queuedRuns:0,display:null,profileMode:"per-bot",screenAvailable:false};
const blankComputer:ComputerStatus={botId:"",mode:"team",state:"stopped",controlHolder:"none",takeoverRequested:false,busyBotName:null,busySessionId:null,busyRunId:null,busyStep:null,usingComputer:false,waitingRunId:null,waitingSessionId:null,queuedRuns:0,display:null,profileMode:"per-bot",screenAvailable:false};
const SESSION_STORE="lazyboy.sessionByBot";
const PANE_STORE="lazyboy.rightPane";
const WORKSPACE_STORE="lazyboy.workspace";
type RightPart="computer"|"memory"|"settings"|"plugins";
type AccountDialog="phone"|"settings"|"model"|"about"|"help"|"feedback"|null;
type RightPart="computer"|"memory"|"settings"|"plugins"|"accounts";
type AccountDialog="phone"|"settings"|"model"|"voice"|"about"|"help"|"feedback"|null;
function readSessionStore():Record<string,string>{try{const raw=localStorage.getItem(SESSION_STORE);return raw?JSON.parse(raw) as Record<string,string>:{}}catch{return {}}}
function writeSessionStore(botId:string,sessionId:string){const store=readSessionStore();store[botId]=sessionId;localStorage.setItem(SESSION_STORE,JSON.stringify(store))}
function readPaneStore():{collapsed:boolean;part:RightPart}{try{const raw=localStorage.getItem(PANE_STORE);if(!raw)return{collapsed:false,part:"computer"};const value=JSON.parse(raw) as {collapsed?:boolean;part?:string};return{collapsed:Boolean(value.collapsed),part:value.part==="memory"||value.part==="settings"||value.part==="plugins"?value.part:"computer"}}catch{return{collapsed:false,part:"computer"}}}
function readPaneStore():{collapsed:boolean;part:RightPart}{try{const raw=localStorage.getItem(PANE_STORE);if(!raw)return{collapsed:false,part:"computer"};const value=JSON.parse(raw) as {collapsed?:boolean;part?:string};return{collapsed:Boolean(value.collapsed),part:value.part==="memory"||value.part==="settings"||value.part==="plugins"||value.part==="accounts"?value.part:"computer"}}catch{return{collapsed:false,part:"computer"}}}
type WorkspacePrefs={name:string;showHidden:boolean};
function readWorkspace():WorkspacePrefs{try{const raw=localStorage.getItem(WORKSPACE_STORE);if(!raw)return{name:t("localWorkspace"),showHidden:false};const value=JSON.parse(raw) as {name?:string;showHidden?:boolean};const name=value.name?.trim();return{name:name&&name!=="Local workspace"?name:t("localWorkspace"),showHidden:Boolean(value.showHidden)}}catch{return{name:t("localWorkspace"),showHidden:false}}}
function isDefaultWorkspaceName(name?:string|null){return !name||name==="Local workspace"||name==="本機工作區"}
function readWorkspace():WorkspacePrefs{try{const raw=localStorage.getItem(WORKSPACE_STORE);if(!raw)return{name:t("localWorkspace"),showHidden:false};const value=JSON.parse(raw) as {name?:string;showHidden?:boolean};const name=value.name?.trim();return{name:isDefaultWorkspaceName(name)?t("localWorkspace"):name||t("localWorkspace"),showHidden:Boolean(value.showHidden)}}catch{return{name:t("localWorkspace"),showHidden:false}}}
function WorkspaceAvatar({name}:{name:string}){const parts=name.trim().split(/\s+/).filter(Boolean);const initials=(parts.length>1?parts.map(part=>part[0]).join(""):parts[0]?.slice(0,2)||"LB").slice(0,2).toUpperCase();return <span className="workspace-avatar" aria-hidden="true">{initials}</span>}
function modeLabel(mode:ComputerMode){return mode==="team"?t("sharedComputer"):t("privateComputer")}
function stateLabel(state:ComputerStatus["state"]){return ({stopped:t("stopped"),booting:t("booting"),running:t("running"),suspended:t("suspended"),error:t("error")})[state]}
function inboxTime(value:string|null){if(!value)return "";const date=new Date(value),now=new Date();if(date.toDateString()===now.toDateString())return new Intl.DateTimeFormat("zh-TW",{hour:"2-digit",minute:"2-digit",hour12:false}).format(date);const days=Math.floor((new Date(now.getFullYear(),now.getMonth(),now.getDate()).getTime()-new Date(date.getFullYear(),date.getMonth(),date.getDate()).getTime())/86400000);if(days<7)return new Intl.DateTimeFormat("zh-TW",{weekday:"long"}).format(date);return new Intl.DateTimeFormat("zh-TW",{month:"numeric",day:"numeric"}).format(date)}
function isTransitionStep(step?:string|null){return /電腦啟動中|喚醒中|換手中/.test(step||"")}
function localizeStepDetail(detail:string){
if(detail==="電腦啟動中"||detail==="電腦啟動中…")return t("hudBooting");
if(detail==="喚醒中"||detail==="喚醒中…")return t("hudWaking");
if(detail==="換手中"||detail==="換手中…")return t("hudHandoff");
if(detail==="思考中")return t("stepThinking");
if(detail==="看畫面")return t("stepWatchingScreen");
if(detail==="重新確認畫面")return t("stepRecheckingScreen");
if(detail==="填入已存帳號")return t("stepFillingLogin");
if(detail==="列出已存帳號")return t("stepListingAccounts");
if(detail==="列出排程")return t("stepListingSchedules");
if(detail==="取消排程")return t("stepCancelingSchedule");
if(detail.startsWith("讀取技能 "))return t("stepUsingSkill",{name:detail.slice("讀取技能 ".length)});
if(detail.startsWith("排程 "))return t("stepScheduling",{name:detail.slice("排程 ".length)});
return detail;
}
function localizeStep(step?:string|null){
if(!step)return "";
const cut=step.indexOf(": ");
if(cut>=0){
const head=step.slice(0,cut);
const tail=localizeStepDetail(step.slice(cut+2));
return `${head}: ${tail}`;
}
return localizeStepDetail(step);
}
function localizeError(message:string){
if(message==="示範進行中:先按「完成示範」或「取消」,再送訊息。")return t("teachInProgress");
if(message==="AI 回應逾時120 秒)")return t("aiTimeout");
return message;
}
function hudLabel(computer:ComputerStatus,connecting:boolean,handingOff:boolean){
const step=computer.busyStep||"";
if(computer.state==="booting"||step==="電腦啟動中")return t("hudBooting");
if(computer.state==="suspended"||step==="喚醒中")return t("hudWaking");
if(handingOff||step==="換手中")return t("hudHandoff");
if(connecting)return t("hudConnecting");
return null;
}
function inboxTime(value:string|null){if(!value)return "";const date=new Date(value),now=new Date();const loc=dateLocale();if(date.toDateString()===now.toDateString())return new Intl.DateTimeFormat(loc,{hour:"2-digit",minute:"2-digit",hour12:false}).format(date);const days=Math.floor((new Date(now.getFullYear(),now.getMonth(),now.getDate()).getTime()-new Date(date.getFullYear(),date.getMonth(),date.getDate()).getTime())/86400000);if(days<7)return new Intl.DateTimeFormat(loc,{weekday:"long"}).format(date);return new Intl.DateTimeFormat(loc,{month:"numeric",day:"numeric"}).format(date)}
const ATTACH_MAX=4;
const ATTACH_MAX_BYTES=10*1024*1024;
const ATTACH_ACCEPT=".png,.jpg,.jpeg,.gif,.webp,.pdf,.txt,.md,.csv,.json,.html,.htm,.xml,.docx,.xlsx,.pptx,image/*,text/*,application/pdf";
@ -45,7 +88,9 @@ function readAsBase64(file:File){return new Promise<string>((resolve,reject)=>{c
function formatBytes(size:number){if(size<1024)return `${size} B`;if(size<1024*1024)return `${Math.round(size/102.4)/10} KB`;return `${Math.round(size/104857.6)/10} MB`}
function fileExt(name:string){const dot=name.lastIndexOf(".");const ext=dot>=0?name.slice(dot+1).replace(/[^a-z0-9]/gi,""):"";return (ext||"FILE").slice(0,4).toUpperCase()}
function messageFiles(blocks:unknown):MessageFile[]{if(!Array.isArray(blocks))return [];return blocks.flatMap(block=>{if(!block||typeof block!=="object")return [];const value=block as {kind?:string;name?:string;mimeType?:string;size?:number};if(value.kind!=="file"&&value.kind!=="image")return [];return [{kind:value.kind,name:value.name||"file",mimeType:value.mimeType,size:value.size}]})}
function isAutoAttachCaption(body:string,files:MessageFile[]){const text=body.trim();if(!files.length)return false;if(!text)return true;return files.some(file=>text===file.name||text===`附件 ${file.name}`||text===t("attachedFile",{name:file.name}))}
function chipBlocks(blocks:unknown){if(!Array.isArray(blocks))return [] as {kind:string;site?:string;why?:string;name?:string;human?:string;cron?:string}[];return blocks.flatMap(block=>{if(!block||typeof block!=="object")return [];const value=block as {kind?:string;site?:string;why?:string;name?:string;human?:string;cron?:string};if(value.kind==="login"||value.kind==="schedule"||value.kind==="scheduleRun")return [value];return []})}
function isAutoAttachCaption(body:string,files:MessageFile[]){const text=body.trim();if(!files.length)return false;if(!text)return true;return files.some(file=>text===file.name||text===`附件 ${file.name}`||text===`Attached ${file.name}`||text===t("attachedFile",{name:file.name}))}
function isAutoScheduleCaption(body:string,chips:{kind?:string}[]){if(!chips.some(chip=>chip.kind==="schedule"||chip.kind==="scheduleRun"))return false;const text=body.trim();return /^\[排程(試跑)?\]/.test(text)||/^\[Schedule( test)?\]/i.test(text)}
function FileCard({file,preview,onRemove}:{file:{name:string;size?:number};preview?:string|null;onRemove?:()=>void}){const ext=fileExt(file.name);return <div className={`file-card ${onRemove?"is-pending":""}`}>{preview?<img className="file-card-thumb" src={preview} alt=""/>:<div className="file-card-badge" aria-hidden="true">{ext}</div>}<div className="file-card-meta"><strong>{file.name}</strong><small>{typeof file.size==="number"?formatBytes(file.size):ext}</small></div>{onRemove&&<button type="button" className="file-card-remove" title={t("attachRemove",{name:file.name})} onClick={onRemove}><X/></button>}</div>}
function clientNonce(){
const webCrypto=globalThis.crypto;
@ -60,6 +105,7 @@ function clientNonce(){
}
export function App(){
const locale=useLocale();
const [bots,setBots]=useState<Bot[]>([]); const [rooms,setRooms]=useState<Room[]>([]); const [activeId,setActiveId]=useState<string|null>(null); const [activeRoomId,setActiveRoomId]=useState<string|null>(null); const [busyMembers,setBusyMembers]=useState<RoomMember[]>([]);
const [sessions,setSessions]=useState<Session[]>([]); const [activeSessionId,setActiveSessionId]=useState<string|null>(null);
const [messages,setMessages]=useState<Message[]>([]); const [computer,setComputer]=useState<ComputerStatus>(blankComputer);
@ -69,6 +115,12 @@ export function App(){
const [rightCollapsed,setRightCollapsed]=useState(paneStart.collapsed); const [rightPart,setRightPart]=useState<RightPart>(paneStart.part);
const [sessionMenuOpen,setSessionMenuOpen]=useState(false); const [clearOpen,setClearOpen]=useState(false); const [sessionToDelete,setSessionToDelete]=useState<string|null>(null); const [remembered,setRemembered]=useState<Record<string,boolean>>({});
const [mobileNav,setMobileNav]=useState(false); const [error,setError]=useState<string|null>(null); const [busy,setBusy]=useState(false);
useEffect(()=>{
if(!mobileNav)return;
const dismiss=(event:KeyboardEvent)=>{if(event.key==="Escape")setMobileNav(false)};
window.addEventListener("keydown",dismiss);
return()=>window.removeEventListener("keydown",dismiss);
},[mobileNav]);
const [desktopClipboard,setDesktopClipboard]=useState(""); const active=bots.find(b=>b.id===activeId)||null;
const activeRoom=rooms.find(room=>room.id===activeRoomId)||null;
const [clipboardOpen,setClipboardOpen]=useState(false);
@ -78,10 +130,16 @@ export function App(){
const [roomContext,setRoomContext]=useState<{room:Room;x:number;y:number}|null>(null);
const [roomToDelete,setRoomToDelete]=useState<Room|null>(null); const [mcpServers,setMcpServers]=useState<McpServer[]>([]);
const [accountOpen,setAccountOpen]=useState(false); const [accountDialog,setAccountDialog]=useState<AccountDialog>(null);
const [skills,setSkills]=useState<TaughtSkill[]>([]); const [plusOpen,setPlusOpen]=useState(false); const [skillQuery,setSkillQuery]=useState(""); const [teachOpen,setTeachOpen]=useState(false); const [editingSkillId,setEditingSkillId]=useState<string|null>(null);
const [voiceSettings,setVoiceSettings]=useState<VoiceSettings|null>(null); const [callOpen,setCallOpen]=useState(false);
const [skills,setSkills]=useState<TaughtSkill[]>([]); const [fileSkills,setFileSkills]=useState<FileSkill[]>([]); const [plusOpen,setPlusOpen]=useState(false); const [skillQuery,setSkillQuery]=useState(""); const [teachOpen,setTeachOpen]=useState(false); const [editingSkillId,setEditingSkillId]=useState<string|null>(null);
const [schedules,setSchedules]=useState<ScheduleItem[]>([]); const [scheduleDraft,setScheduleDraft]=useState<{name:string;instructions:string;enabled:boolean;preset:CronPreset;id?:string;timezone?:string;threadId?:string|null}|null>(null);
const [scheduleError,setScheduleError]=useState<string|null>(null); const [scheduleSaving,setScheduleSaving]=useState(false); const [runningScheduleId,setRunningScheduleId]=useState<string|null>(null);
const [workspaceName,setWorkspaceName]=useState(workspaceStart.name);
const [looks,setLooks]=useState(readAvatarLooks);
const sendingRef=useRef(false); const refreshSeqRef=useRef(0); const importRef=useRef<HTMLInputElement>(null); const attachRef=useRef<HTMLInputElement>(null);
const desktopFrameRef=useRef<HTMLIFrameElement>(null);
const holderRef=useRef(computer.controlHolder); const paneBotRef=useRef<string|null>(null); const skipHandoffRef=useRef(true);
const [desktopReady,setDesktopReady]=useState(false); const [handingOff,setHandingOff]=useState(false);
const [pendingFiles,setPendingFiles]=useState<PendingFile[]>([]);
const messageEndRef=useRef<HTMLDivElement|null>(null);
const sentHistoryRef=useRef<string[]>([]); const historyIndexRef=useRef<number|null>(null); const historyDraftRef=useRef("");
@ -92,7 +150,10 @@ export function App(){
const sections=useMemo(()=>{const map=new Map<string,Bot[]>();for(const bot of filtered){const key=bot.pinned?t("pinned"):bot.groupName||t("agentGroup");map.set(key,[...(map.get(key)||[]),bot])}return [...map.entries()]},[filtered]);
const paneBotId=busyMembers[0]?.id||activeRoom?.members[0]?.id||activeId;
const paneBot=bots.find(bot=>bot.id===paneBotId)||active;
const workingMembers=activeRoom?busyMembers:active&&computer.busySessionId===activeSessionId?[{id:active.id,name:active.name,avatarColor:active.avatarColor,avatarShape:active.avatarShape}]:[];
const currentPaneRef=useRef(paneBotId);currentPaneRef.current=paneBotId;
const pasteQueueRef=useRef<Promise<unknown>>(Promise.resolve());
const [clipboardStatus,setClipboardStatus]=useState("");
const workingMembers=activeRoom?busyMembers:active&&activeSessionId&&computer.busySessionId===activeSessionId?[{id:active.id,name:active.name,avatarColor:active.avatarColor,avatarShape:active.avatarShape}]:[];
const lastMessageId=messages[messages.length-1]?.id||"";
// The bot is parked in waiting_takeover: nothing moves (including queued
// messages) until the human releases the screen, so say so loudly.
@ -105,6 +166,13 @@ export function App(){
const savedSkills=skills.filter(skill=>skill.status==="saved");
const skillNeedle=skillQuery.trim().toLowerCase();
const listedSkills=skillNeedle?savedSkills.filter(skill=>skill.name.toLowerCase().includes(skillNeedle)||(skill.playbook.whenToUse||"").toLowerCase().includes(skillNeedle)||skill.goal.toLowerCase().includes(skillNeedle)):savedSkills;
const [slashIndex,setSlashIndex]=useState(0);
const [slashDismissed,setSlashDismissed]=useState(false);
useEffect(()=>{setSlashIndex(0);setSlashDismissed(false)},[draft]);
const slashToken=draft.trimStart().split(/\s/,1)[0].slice(1).toLowerCase();
const slashSuggestions=!slashDismissed&&/^\/[^\s]*$/.test(draft.trimStart())
? [{name:"goal",description:t("goalCommandHint"),kind:t("slashMode")},...fileSkills.map(skill=>({...skill,kind:t("slashFileSkill")}))].filter(skill=>!slashToken||skill.name.startsWith(slashToken)).slice(0,8)
: [];
const loadMcp=useCallback(async()=>{setMcpServers(await api<McpServer[]>("/api/mcp-servers").catch(()=>[] as McpServer[]))},[]);
const loadBots=useCallback(async()=>{const [next,nextRooms]=await Promise.all([api<Bot[]>("/api/bots"),api<Room[]>("/api/rooms").catch(()=>[] as Room[])]);setBots(next);setRooms(nextRooms);setActiveRoomId(id=>id&&nextRooms.some(room=>room.id===id)?id:null);setActiveId(id=>id&&next.some(b=>b.id===id)?id:null);await loadMcp()},[loadMcp]);
@ -124,27 +192,57 @@ export function App(){
if(refreshSeq!==refreshSeqRef.current)return;
setBusyMembers(nextBusy);setComputer(status);setMessages(nextMessages);setSkills(nextSkills);setScreenUrl(status.botId===computerBot?screen.url:null)
},[activeId,activeRoomId,activeSessionId]);
useEffect(()=>{loadBots().catch(e=>{if(e instanceof ApiError&&e.status===401)setAuthRequired(true);else setError(e.message)})},[loadBots]);
useEffect(()=>{loadBots().catch(e=>{if(e instanceof ApiError&&e.status===401)setAuthRequired(true);else setError(localizeError(e.message))})},[loadBots]);
useEffect(()=>{if(!voiceSettings?.enabled)setCallOpen(false)},[voiceSettings?.enabled]);
useEffect(()=>{api<VoiceSettings>("/api/voice/settings").then(setVoiceSettings).catch(()=>setVoiceSettings(null))},[]);
useEffect(()=>{if(authRequired)return;api<FileSkill[]>("/api/file-skills").then(setFileSkills).catch(()=>setFileSkills([]))},[authRequired]);
useEffect(()=>{if(activeId||activeRoomId||bots.length===0)return;setActiveId(bots[0].id)},[bots,activeId,activeRoomId]);
useEffect(()=>{setMessages([]);loadSessions().catch(e=>setError(e.message))},[loadSessions]);
useEffect(()=>{setMessages([]);loadSessions().catch(e=>setError(localizeError(e.message)))},[loadSessions]);
useEffect(()=>{historyIndexRef.current=null;historyDraftRef.current="";setPendingFiles(current=>{current.forEach(file=>file.preview&&URL.revokeObjectURL(file.preview));return []})},[activeSessionId]);
useEffect(()=>{if(!plusOpen)setSkillQuery("")},[plusOpen]);
useEffect(()=>{messageEndRef.current?.scrollIntoView({block:"end",behavior:"smooth"})},[activeSessionId,lastMessageId,workingMembers.length,pausedForUser]);
useEffect(()=>{if(!activeSessionId||(!activeId&&!activeRoomId)){refreshSeqRef.current+=1;setMessages([]);setScreenUrl(null);if(!activeId&&!activeRoomId)setComputer(blankComputer);return}setScreenUrl(null);refresh().catch(e=>setError(e.message));const timer=setInterval(()=>{refresh().catch(()=>{});const beat=roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||activeId;if(beat)api(`/api/computer/${beat}/heartbeat`,{method:"POST",body:"{}"}).catch(()=>{})},2000);return()=>{clearInterval(timer);refreshSeqRef.current+=1}},[activeId,activeRoomId,activeSessionId,refresh]);
useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard.writeText(text).catch(()=>{})}};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)});
useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.data?.type!=="lazyboy-request-control"||!paneBotId)return;void action(()=>api(`/api/computer/${paneBotId}/takeover`,{method:"POST",body:"{}"}))};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)},[paneBotId]);
useEffect(()=>{if(!activeSessionId||(!activeId&&!activeRoomId)){refreshSeqRef.current+=1;setMessages([]);setScreenUrl(null);if(!activeId&&!activeRoomId)setComputer(blankComputer);return}setScreenUrl(null);refresh().catch(e=>setError(localizeError(e.message)));const timer=setInterval(()=>{refresh().catch(()=>{});const beat=roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||activeId;if(beat)api(`/api/computer/${beat}/heartbeat`,{method:"POST",body:"{}"}).catch(()=>{})},2000);return()=>{clearInterval(timer);refreshSeqRef.current+=1}},[activeId,activeRoomId,activeSessionId,refresh]);
useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard?.writeText(text).then(()=>setClipboardStatus(t("clipboardSynced"))).catch(()=>setClipboardStatus(t("clipboardSyncBlocked")))}if(event.data.type==="lazyboy-copy-request"&&computer.controlHolder==="user")void copySelection();if(event.data.type==="lazyboy-paste-text"&&typeof event.data.text==="string")pasteText(event.data.text);if(event.data.type==="lazyboy-mobile-key"&&typeof event.data.key==="string"&&["Return","BackSpace","Tab","Escape","Left","Right","Up","Down"].includes(event.data.key))queueDesktopInput({kind:"key",key:event.data.key});if(event.data.type==="lazyboy-paste-request"&&computer.controlHolder==="user")setClipboardOpen(true);if(event.data.type==="lazyboy-desktop-ready")setDesktopReady(true);if(event.data.type==="lazyboy-desktop-lost")setDesktopReady(false)};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)});
useEffect(()=>{setDesktopReady(false)},[screenUrl,paneBotId]);
useEffect(()=>{if(skipHandoffRef.current){skipHandoffRef.current=false;holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId;return}if((holderRef.current!==computer.controlHolder||paneBotRef.current!==paneBotId)&&computer.state==="running")setHandingOff(true);holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId},[computer.controlHolder,paneBotId,computer.state]);
useEffect(()=>{if(!handingOff)return;const timer=setTimeout(()=>setHandingOff(false),1600);return()=>clearTimeout(timer)},[handingOff]);
useEffect(()=>{const frame=desktopFrameRef.current;if(!frame?.contentWindow||!screenUrl)return;frame.contentWindow.postMessage({type:"lazyboy-view-only",viewOnly:computer.controlHolder!=="user"},location.origin)},[computer.controlHolder,screenUrl,desktopReady]);
useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||event.data?.type!=="lazyboy-request-control"||!paneBotId)return;void action(()=>api(`/api/computer/${paneBotId}/takeover`,{method:"POST",body:"{}"}))};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)},[paneBotId]);
useEffect(()=>{const close=(event:MouseEvent)=>{const target=event.target;if(target instanceof Element&&target.closest(".create-menu-wrap,.account-wrap,.session-picker,.context-menu,.plus-menu-wrap"))return;setContext(null);setRoomContext(null);setSessionMenuOpen(false);setAccountOpen(false);setCreateMenuOpen(false);setPlusOpen(false)};window.addEventListener("click",close);return()=>window.removeEventListener("click",close)},[]);
useEffect(()=>{const onKey=(event:KeyboardEvent)=>{if(event.key!=="Escape")return;setAccountOpen(false);setAccountDialog(null);setCreateMenuOpen(false);setPlusOpen(false);setTeachOpen(false);setEditingSkillId(null);setSessionMenuOpen(false);setContext(null);setRoomContext(null)};window.addEventListener("keydown",onKey);return()=>window.removeEventListener("keydown",onKey)},[]);
useEffect(()=>{setWorkspaceName(name=>isDefaultWorkspaceName(name)?t("localWorkspace"):name)},[locale]);
useEffect(()=>{localStorage.setItem(WORKSPACE_STORE,JSON.stringify({name:workspaceName,showHidden}))},[workspaceName,showHidden]);
useEffect(()=>{if(!sessionStoreKey||!activeSessionId)return;if(!sessions.some(session=>session.id===activeSessionId))return;if(!activeRoomId&&!sessions.some(session=>session.id===activeSessionId&&session.botId===activeId))return;writeSessionStore(sessionStoreKey,activeSessionId)},[sessionStoreKey,activeId,activeRoomId,activeSessionId,sessions]);
useEffect(()=>{localStorage.setItem(PANE_STORE,JSON.stringify({collapsed:rightCollapsed,part:rightPart}))},[rightCollapsed,rightPart]);
function openPane(part:RightPart){setRightPart(part);setRightCollapsed(false)}
useEffect(()=>{if(!paneBotId){setSchedules([]);return}api<ScheduleItem[]>(`/api/bots/${paneBotId}/schedules`).then(setSchedules).catch(()=>setSchedules([]))},[paneBotId]);
useEffect(()=>{if(computer.takeoverRequested){setRightPart("computer");setRightCollapsed(false)}},[computer.takeoverRequested]);
function openPane(part:RightPart){setMobileNav(false);setRightPart(part);setRightCollapsed(false)}
async function openLoginScreen(){
const id=paneBot?.id||active?.id;if(!id)return;
setRightPart("computer");setRightCollapsed(false);setComputerOpen(true);
await action(async()=>{
try{await api(`/api/computer/${id}/boot`,{method:"POST",body:"{}"})}catch{/* already up */}
await api(`/api/computer/${id}/takeover`,{method:"POST",body:"{}"}).catch(()=>{});
});
}
async function reloadSchedules(){if(!paneBotId)return;setSchedules(await api<ScheduleItem[]>(`/api/bots/${paneBotId}/schedules`))}
async function saveScheduleDraft(){
if(!paneBot||!scheduleDraft)return;
setScheduleSaving(true);setScheduleError(null);
try{
const body={name:scheduleDraft.name.trim(),cron:cronFromPreset(scheduleDraft.preset),instructions:scheduleDraft.instructions.trim(),timezone:scheduleDraft.timezone||Intl.DateTimeFormat().resolvedOptions().timeZone||"Asia/Taipei",enabled:scheduleDraft.enabled,threadId:scheduleDraft.id?scheduleDraft.threadId:activeRoomId?undefined:activeSessionId};
if(scheduleDraft.id)await api(`/api/schedules/${scheduleDraft.id}`,{method:"PATCH",body:JSON.stringify(body)});
else await api(`/api/bots/${paneBot.id}/schedules`,{method:"POST",body:JSON.stringify(body)});
setScheduleDraft(null);await reloadSchedules();
}catch(e){setScheduleError(e instanceof Error?localizeError(e.message):t("operationFailed"))}
finally{setScheduleSaving(false)}
}
const sessionBusy=workingMembers.length>0;
const otherSessionBusy=Boolean(computer.busyBotName&&!sessionBusy);
const chatName=activeRoom?.name||active?.name||"";
async function action(work:()=>Promise<unknown>){setBusy(true);setError(null);try{await work();await refresh()}catch(e){setError(e instanceof Error?e.message:t("operationFailed"))}finally{setBusy(false)}}
async function action(work:()=>Promise<unknown>){setBusy(true);setError(null);try{await work();await refresh()}catch(e){setError(e instanceof Error?localizeError(e.message):t("operationFailed"))}finally{setBusy(false)}}
async function stopChat(){if(activeSessionId)await api(`/api/sessions/${activeSessionId}/stop`,{method:"POST",body:"{}"});else if(active)await api(`/api/bots/${active.id}/stop`,{method:"POST",body:"{}"});}
async function startTeaching(goal:string){if(!active)return;setTeachOpen(false);await action(async()=>{await api(`/api/bots/${active.id}/skills/start`,{method:"POST",body:JSON.stringify({goal})});setComputerOpen(true)})}
async function stopTeaching(){if(!active)return;await action(()=>api(`/api/bots/${active.id}/skills/stop`,{method:"POST",body:"{}"}));setComputerOpen(false)}
@ -155,42 +253,76 @@ export function App(){
async function updateSkill(skill:TaughtSkill,name:string,playbook:Playbook){await action(()=>api(`/api/skills/${skill.id}`,{method:"PATCH",body:JSON.stringify({name,playbook,save:true})}));setEditingSkillId(null)}
async function deleteSkill(skill:TaughtSkill){if(!window.confirm(t("deleteSkillConfirm",{name:skill.name||skill.goal})))return;await action(()=>api(`/api/skills/${skill.id}`,{method:"DELETE"}));setEditingSkillId(null)}
const editingSkill=editingSkillId?skills.find(skill=>skill.id===editingSkillId)||null:null;
function runSkill(skill:TaughtSkill){setPlusOpen(false);setDraft(current=>`${current.trim()?current.trimEnd()+"\n":""}執行「${skill.name}`);document.querySelector<HTMLTextAreaElement>(".composer textarea")?.focus()}
function runSkill(skill:TaughtSkill){setPlusOpen(false);setDraft(current=>`${current.trim()?current.trimEnd()+"\n":""}${t("runSkillNamed",{name:skill.name})}`);document.querySelector<HTMLTextAreaElement>(".composer textarea")?.focus()}
async function importSkillFile(file:File){if(!active)return;await action(async()=>{let payload:unknown;try{payload=JSON.parse(await file.text())}catch{throw new Error(t("skillImportInvalid"))}const skill=await api<TaughtSkill>(`/api/bots/${active.id}/skills/import`,{method:"POST",body:JSON.stringify(payload)});setEditingSkillId(skill.id)})}
function addPendingFiles(list:FileList|File[]){const incoming=[...list];if(!incoming.length)return;setError(null);setPendingFiles(current=>{const next=[...current];for(const file of incoming){if(next.length>=ATTACH_MAX){setError(t("attachTooMany"));break}if(file.size>ATTACH_MAX_BYTES){setError(t("attachTooLarge"));continue}if(!attachAllowed(file)){setError(t("attachType"));continue}next.push({id:clientNonce(),file,preview:file.type.startsWith("image/")?URL.createObjectURL(file):null})}return next})}
function removePendingFile(id:string){setPendingFiles(current=>current.filter(item=>{if(item.id===id&&item.preview)URL.revokeObjectURL(item.preview);return item.id!==id}))}
async function send(event:FormEvent){event.preventDefault();if(sendingRef.current||busy)return;const text=draft.trim();const files=pendingFiles;if((!active&&!activeRoom)||!activeSessionId||(!text&&files.length===0))return;sendingRef.current=true;setDraft("");setPendingFiles([]);historyIndexRef.current=null;historyDraftRef.current="";try{await action(async()=>{try{const attachments=await Promise.all(files.map(async item=>({name:item.file.name,mimeType:item.file.type,content:await readAsBase64(item.file)})));await api(`/api/sessions/${activeSessionId}/messages`,{method:"POST",body:JSON.stringify({text,clientNonce:clientNonce(),attachments})});sentHistoryRef.current.push(text||files[0]?.file.name||"");if(sentHistoryRef.current.length>100)sentHistoryRef.current.shift();files.forEach(item=>item.preview&&URL.revokeObjectURL(item.preview));await loadSessions()}catch(error){setPendingFiles(files);throw error}})}finally{sendingRef.current=false}}
function composerKeyDown(event:ReactKeyboardEvent<HTMLTextAreaElement>){if(event.nativeEvent.isComposing||event.key==="Process")return;const history=sentHistoryRef.current;if(event.key==="ArrowUp"&&history.length>0&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionStart===0)){event.preventDefault();if(historyIndexRef.current===null){historyDraftRef.current=draft;historyIndexRef.current=history.length-1}else historyIndexRef.current=Math.max(0,historyIndexRef.current-1);setDraft(history[historyIndexRef.current]);return}if(event.key==="ArrowDown"&&historyIndexRef.current!==null&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionEnd===event.currentTarget.value.length)){event.preventDefault();if(historyIndexRef.current<history.length-1){historyIndexRef.current+=1;setDraft(history[historyIndexRef.current])}else{historyIndexRef.current=null;setDraft(historyDraftRef.current)}return}if(event.key==="Enter"&&!event.shiftKey){event.preventDefault();if(sendingRef.current||busy)return;event.currentTarget.form?.requestSubmit()}}
function composerKeyDown(event:ReactKeyboardEvent<HTMLTextAreaElement>){if(event.nativeEvent.isComposing||event.key==="Process")return;if(slashSuggestions.length){if(event.key==="Escape"){event.preventDefault();setSlashDismissed(true);return}if(event.key==="ArrowDown"||event.key==="ArrowUp"){event.preventDefault();setSlashIndex(index=>(index+(event.key==="ArrowDown"?1:slashSuggestions.length-1))%slashSuggestions.length);return}if(event.key==="Tab"||(event.key==="Enter"&&!event.shiftKey)){event.preventDefault();setDraft(`/${slashSuggestions[slashIndex%slashSuggestions.length].name} `);return}}const history=sentHistoryRef.current;if(event.key==="ArrowUp"&&history.length>0&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionStart===0)){event.preventDefault();if(historyIndexRef.current===null){historyDraftRef.current=draft;historyIndexRef.current=history.length-1}else historyIndexRef.current=Math.max(0,historyIndexRef.current-1);setDraft(history[historyIndexRef.current]);return}if(event.key==="ArrowDown"&&historyIndexRef.current!==null&&(!event.currentTarget.value.includes("\n")||event.currentTarget.selectionEnd===event.currentTarget.value.length)){event.preventDefault();if(historyIndexRef.current<history.length-1){historyIndexRef.current+=1;setDraft(history[historyIndexRef.current])}else{historyIndexRef.current=null;setDraft(historyDraftRef.current)}return}if(event.key==="Enter"&&!event.shiftKey){event.preventDefault();if(sendingRef.current||busy)return;event.currentTarget.form?.requestSubmit()}}
function selectSession(id:string){setActiveSessionId(id);setSessionMenuOpen(false);if(sessionStoreKey)writeSessionStore(sessionStoreKey,id)}
async function createSession(){if(!sessionsPath||!sessionStoreKey)return;setSessionMenuOpen(false);setBusy(true);setError(null);try{const session=await api<Session>(sessionsPath,{method:"POST",body:JSON.stringify({title:t("newConversation")})});writeSessionStore(sessionStoreKey,session.id);const next=await api<Session[]>(sessionsPath);setSessions(next);setActiveSessionId(session.id);setMessages([])}catch(e){setError(e instanceof Error?e.message:t("operationFailed"))}finally{setBusy(false)}}
async function createSession(){if(!sessionsPath||!sessionStoreKey)return;setSessionMenuOpen(false);setBusy(true);setError(null);try{const session=await api<Session>(sessionsPath,{method:"POST",body:JSON.stringify({title:t("newConversation")})});writeSessionStore(sessionStoreKey,session.id);const next=await api<Session[]>(sessionsPath);setSessions(next);setActiveSessionId(session.id);setMessages([])}catch(e){setError(e instanceof Error?localizeError(e.message):t("operationFailed"))}finally{setBusy(false)}}
async function clearSession(){if(!activeSessionId)return;setClearOpen(false);setSessionMenuOpen(false);await action(async()=>{await api(`/api/sessions/${activeSessionId}/messages`,{method:"DELETE"});setMessages([]);await loadSessions()})}
async function deleteSession(id:string){if(!sessionsPath||!sessionStoreKey)return;setSessionMenuOpen(false);setBusy(true);setError(null);try{await api(`/api/sessions/${id}`,{method:"DELETE"});const next=await api<Session[]>(sessionsPath);setSessions(next);const pick=id===activeSessionId||!next.some(session=>session.id===activeSessionId)?next[0]?.id||null:activeSessionId;setActiveSessionId(pick);if(pick)writeSessionStore(sessionStoreKey,pick)}catch(e){setError(e instanceof Error?e.message:t("operationFailed"))}finally{setBusy(false)}}
async function rememberMessage(message:Message){const botId=message.speakerBotId||active?.id||activeRoom?.members[0]?.id;if(!botId||!message.body.trim())return;try{await api(`/api/bots/${botId}/memories`,{method:"POST",body:JSON.stringify({content:message.body,sessionId:activeSessionId})});setRemembered(current=>({...current,[message.id]:true}))}catch(e){setError(e instanceof Error?e.message:t("rememberFailed"))}}
async function pasteClipboard(){try{const text=await navigator.clipboard.readText();document.querySelectorAll<HTMLIFrameElement>(".desktop-frame").forEach(frame=>frame.contentWindow?.postMessage({type:"lazyboy-host-clipboard",text},location.origin))}catch{setClipboardOpen(true)}}
async function deleteSession(id:string){if(!sessionsPath||!sessionStoreKey)return;setSessionMenuOpen(false);setBusy(true);setError(null);try{await api(`/api/sessions/${id}`,{method:"DELETE"});const next=await api<Session[]>(sessionsPath);setSessions(next);const pick=id===activeSessionId||!next.some(session=>session.id===activeSessionId)?next[0]?.id||null:activeSessionId;setActiveSessionId(pick);if(pick)writeSessionStore(sessionStoreKey,pick)}catch(e){setError(e instanceof Error?localizeError(e.message):t("operationFailed"))}finally{setBusy(false)}}
async function rememberMessage(message:Message){const botId=message.speakerBotId||active?.id||activeRoom?.members[0]?.id;if(!botId||!message.body.trim())return;try{await api(`/api/bots/${botId}/memories`,{method:"POST",body:JSON.stringify({content:message.body,sessionId:activeSessionId})});setRemembered(current=>({...current,[message.id]:true}))}catch(e){setError(e instanceof Error?localizeError(e.message):t("rememberFailed"))}}
function pasteText(text:string){queueDesktopInput({kind:"clipboard",text})}
function queueDesktopInput(input:{kind:"clipboard";text:string}|{kind:"key";key:string}){
const botId=paneBotId;
if(!botId||computer.controlHolder!=="user")return;
pasteQueueRef.current=pasteQueueRef.current.catch(()=>{}).then(async()=>{
if(currentPaneRef.current!==botId)return;
await api(`/api/computer/${botId}/input`,{method:"POST",body:JSON.stringify(input)});
if(input.kind==="clipboard"&&currentPaneRef.current===botId)setClipboardStatus(t("clipboardPasted"));
}).catch(()=>setClipboardStatus(t("clipboardPasteFailed")));
}
async function copySelection(){
const botId=paneBotId;if(!botId)return;
try{const result=await api<{text:string}>(`/api/computer/${botId}/input`,{method:"POST",body:JSON.stringify({kind:"copy"})});
if(currentPaneRef.current!==botId)return;
setDesktopClipboard(result.text);await navigator.clipboard.writeText(result.text);setClipboardStatus(t("clipboardCopiedSelection"));
}catch{setClipboardStatus(t("clipboardCopyFailed"));}
}
async function pasteClipboard(){try{pasteText(await navigator.clipboard.readText())}catch{setClipboardOpen(true)}}
async function copyClipboard(){try{await navigator.clipboard.writeText(desktopClipboard)}catch{setError(t("clipboardWriteBlocked"))}}
async function inbox(bot:Bot,actionName:string,groupName?:string|null){await api(`/api/bots/${bot.id}/inbox`,{method:"POST",body:JSON.stringify({action:actionName,groupName})});await loadBots()}
function openBot(bot:Bot){setMobileNav(false);setSessionMenuOpen(false);if(bot.unreadCount>0)void inbox(bot,"read");if(bot.id===activeId&&!activeRoomId){if(!activeSessionId){const stored=readSessionStore()[bot.id];if(stored)setActiveSessionId(stored);else void loadSessions()}return}setMessages([]);setActiveSessionId(null);setActiveRoomId(null);setBusyMembers([]);setActiveId(bot.id)}
function openRoom(room:Room){setMobileNav(false);setSessionMenuOpen(false);if(rightPart==="settings")setRightPart("computer");if(room.id===activeRoomId){if(!activeSessionId){const stored=readSessionStore()[`room:${room.id}`];if(stored)setActiveSessionId(stored);else void loadSessions()}return}setMessages([]);setActiveSessionId(null);setActiveId(null);setBusyMembers([]);setActiveRoomId(room.id)}
function openBot(bot:Bot){setCallOpen(false);setMobileNav(false);setSessionMenuOpen(false);if(bot.unreadCount>0)void inbox(bot,"read");if(bot.id===activeId&&!activeRoomId){if(!activeSessionId){const stored=readSessionStore()[bot.id];if(stored)setActiveSessionId(stored);else void loadSessions()}return}setMessages([]);setActiveSessionId(null);setActiveRoomId(null);setBusyMembers([]);setActiveId(bot.id)}
function openRoom(room:Room){setCallOpen(false);setMobileNav(false);setSessionMenuOpen(false);if(rightPart==="settings")setRightPart("computer");if(room.id===activeRoomId){if(!activeSessionId){const stored=readSessionStore()[`room:${room.id}`];if(stored)setActiveSessionId(stored);else void loadSessions()}return}setMessages([]);setActiveSessionId(null);setActiveId(null);setBusyMembers([]);setActiveRoomId(room.id)}
async function deleteRoom(room:Room){setRoomToDelete(null);await action(async()=>{await api(`/api/rooms/${room.id}`,{method:"DELETE"});if(activeRoomId===room.id){setActiveRoomId(null);setActiveSessionId(null);setMessages([]);setBusyMembers([])}await loadBots()})}
function openAccount(dialog:AccountDialog){setAccountOpen(false);setAccountDialog(dialog)}
async function logout(){setAccountOpen(false);await api("/api/session",{method:"DELETE",body:"{}"}).catch(()=>{});setBots([]);setRooms([]);setMcpServers([]);setActiveId(null);setActiveRoomId(null);setAuthRequired(true)}
// Re-mount noVNC after every status transition so a freshly booted Docker
// display cannot remain stuck on the previous disconnected iframe.
const startBoot=async()=>{setScreenUrl(null);setComputer(current=>({...current,state:"booting"}));await api(`/api/computer/${paneBot?.id||active?.id}/boot`,{method:"POST",body:"{}"})};
const restartComputer=async()=>{setScreenUrl(null);setComputer(current=>({...current,state:"booting"}));await api(`/api/computer/${paneBot?.id||active?.id}/restart`,{method:"POST",body:"{}"})};
const frame=screenUrl?<iframe key={`${screenUrl}:${computer.state}:${computer.botId}:${computer.controlHolder}`} className="desktop-frame" src={screenUrl} title={t("agentComputer")} allow="fullscreen; clipboard-read; clipboard-write"/>:<EmptyComputer state={computer.state}/>;
async function changeComputer(operation:"boot"|"restart"|"stop"){
const botId=paneBotId;if(!botId)return;
const previous=computer;setScreenUrl(null);setDesktopReady(false);
setComputer(current=>({...current,state:operation==="stop"?current.state:operation==="boot"&&current.state==="suspended"?"suspended":"booting"}));
try{const status=await api<ComputerStatus>(`/api/computer/${botId}/${operation}`,{method:"POST",body:"{}",signal:AbortSignal.timeout(120_000)});if(currentPaneRef.current===botId)setComputer(status)}
catch(error){
const status=await api<ComputerStatus>(`/api/computer/${botId}/status`,{signal:AbortSignal.timeout(5_000)}).catch(()=>previous);
if(currentPaneRef.current===botId)setComputer(status);
throw error;
}
}
const startBoot=()=>changeComputer("boot");
const stopComputer=()=>changeComputer("stop");
const restartComputer=()=>changeComputer("restart");
const connecting=computer.state==="running"&&Boolean(screenUrl)&&!desktopReady;
const overlayLabel=hudLabel(computer,connecting,handingOff);
const frame=screenUrl?<iframe ref={desktopFrameRef} key={screenUrl} className="desktop-frame" src={screenUrl} title={t("agentComputer")} allow="fullscreen; clipboard-read; clipboard-write"/>:<EmptyComputer state={computer.state}/>;
const hud=paneBot&&overlayLabel?<ComputerHud bot={paneBot} label={overlayLabel}/>:null;
const statusMembers=workingMembers;
const topTools=<nav className="top-tools" aria-label={t("workTools")}>
{active&&!activeRoomId?<span className="call-entry" tabIndex={!voiceSettings?.enabled?0:undefined} title={!voiceSettings?.enabled?t("voiceDisabledHint"):undefined} aria-label={!voiceSettings?.enabled?t("voiceDisabledHint"):undefined}><button type="button" className={`top-tool-button ${callOpen?"call-active":""}`} title={!voiceSettings?.enabled?t("voiceDisabledHint"):voiceSettings.ready?t("call"):t("setUpVoiceToCall")} aria-label={t("call")} disabled={!activeSessionId||!voiceSettings?.enabled} onClick={()=>{if(!voiceSettings?.enabled)return;if(!voiceSettings.ready){setAccountDialog("voice");return}setCallOpen(true)}}><PhoneIcon/></button></span>:null}
<button type="button" className={`top-tool-button ${!rightCollapsed&&rightPart==="computer"?"active":""}`} title={t("computer")} aria-label={t("computer")} onClick={()=>openPane("computer")}><Computer/></button>
<button type="button" className={`top-tool-button ${!rightCollapsed&&rightPart==="accounts"?"active":""}`} title={t("accounts")} aria-label={t("accounts")} onClick={()=>openPane("accounts")} disabled={!paneBot}><svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.8"><rect x="5" y="11" width="14" height="10" rx="2"/><path d="M8 11V8a4 4 0 0 1 8 0v3"/></svg></button>
<button type="button" className={`top-tool-button ${!rightCollapsed&&rightPart==="memory"?"active":""}`} title={t("memory")} aria-label={t("memory")} onClick={()=>openPane("memory")} disabled={!paneBot}><Brain/></button>
<button type="button" className={`top-tool-button ${!rightCollapsed&&rightPart==="plugins"?"active":""}`} title={t("plugins")} aria-label={t("plugins")} onClick={()=>openPane("plugins")}><Plug/></button>
<button type="button" className={`top-tool-button ${!rightCollapsed&&rightPart==="settings"?"active":""}`} title={active?t("botSettings"):t("settings")} aria-label={active?t("botSettings"):t("settings")} onClick={()=>active?openPane("settings"):setAccountDialog("settings")}><Settings/></button>
</nav>;
if(authRequired)return <LoginScreen authenticated={async()=>{setAuthRequired(false);setError(null);try{await loadBots()}catch(e){if(e instanceof ApiError&&e.status===401)setAuthRequired(true);else setError(e instanceof Error?e.message:t("loginFailed"))}}}/>;
if(authRequired)return <LoginScreen authenticated={async()=>{setAuthRequired(false);setError(null);try{await loadBots()}catch(e){if(e instanceof ApiError&&e.status===401)setAuthRequired(true);else setError(e instanceof Error?localizeError(e.message):t("loginFailed"))}}}/>;
return <AvatarLookProvider value={looks}><div className={`app-shell ${rightCollapsed?"right-collapsed":"right-open"}`}>
<aside className={`sidebar ${mobileNav?"open":""}`}>
<div className="brand"><span>LazyBoy</span><div className="create-menu-wrap" onClick={event=>event.stopPropagation()}><button type="button" className="icon-button" onClick={event=>{event.stopPropagation();setCreateMenuOpen(v=>!v)}} aria-label={t("add")} aria-haspopup="menu" aria-expanded={createMenuOpen}><UseAnimations animation={plusToX} size={18} strokeColor="#dfdfe2" wrapperStyle={{pointerEvents:"none"}}/></button>{createMenuOpen&&<div className="create-menu" role="menu"><button type="button" role="menuitem" onClick={()=>{setCreateMenuOpen(false);setCreateOpen(true)}}><BotIcon/>{t("addBot")}</button><button type="button" role="menuitem" onClick={()=>{setCreateMenuOpen(false);setGroupOpen(true)}} disabled={bots.length===0}><Users/>{t("addGroup")}</button></div>}</div></div>
{mobileNav&&<button type="button" className="mobile-nav-backdrop" aria-label={t("close")} onClick={()=>setMobileNav(false)}/>}
<aside id="conversation-sidebar" className={`sidebar ${mobileNav?"open":""}`}>
<div className="brand"><img className="brand-icon" src="/lazyboy-icon.png?v=white-1" alt=""/><span>LazyBoy</span><button type="button" className="icon-button mobile-nav-close" aria-label={t("close")} onClick={()=>setMobileNav(false)}>×</button><div className="create-menu-wrap" onClick={event=>event.stopPropagation()}><button type="button" className="icon-button" onClick={event=>{event.stopPropagation();setCreateMenuOpen(v=>!v)}} aria-label={t("add")} aria-haspopup="menu" aria-expanded={createMenuOpen}><UseAnimations animation={plusToX} size={18} strokeColor="var(--ink)" wrapperStyle={{pointerEvents:"none"}}/></button>{createMenuOpen&&<div className="create-menu" role="menu"><button type="button" role="menuitem" onClick={()=>{setCreateMenuOpen(false);setCreateOpen(true)}}><BotIcon/>{t("addBot")}</button><button type="button" role="menuitem" onClick={()=>{setCreateMenuOpen(false);setGroupOpen(true)}} disabled={bots.length===0}><Users/>{t("addGroup")}</button></div>}</div></div>
<label className="search"><UseAnimations animation={searchToX} size={16} strokeColor="var(--muted)"/><input value={query} onChange={e=>setQuery(e.target.value)} placeholder={t("search")}/></label>
<div className="bot-list">
{filteredRooms.length>0&&<section className="bot-group"><div className="group-label">{t("groups")}</div>{filteredRooms.map(room=><button className={`bot-row room-row ${room.id===activeRoomId?"selected":""}`} key={room.id} onClick={()=>openRoom(room)} onContextMenu={e=>{e.preventDefault();setRoomContext({room,x:e.clientX,y:e.clientY})}}><span className="avatar-wrap"><AvatarStack members={room.members} online thinkingIds={room.id===activeRoomId?busyMembers.map(member=>member.id):[]}/>{room.unreadCount>0&&<i className="unread-dot" title={t("unreadMessages",{count:room.unreadCount})}/>}</span><span className="bot-copy"><strong>{room.name}</strong><small>{room.lastPreview||t("members",{count:room.members.length})}</small></span>{room.lastMessageAt&&<time className="row-time">{inboxTime(room.lastMessageAt)}</time>}</button>)}</section>}
@ -207,6 +339,7 @@ export function App(){
<button type="button" role="menuitem" onClick={()=>openAccount("phone")}><Smartphone/>{t("openOnPhone")}</button>
<button type="button" role="menuitem" onClick={()=>openAccount("settings")}><Settings/>{t("settings")}</button>
<button type="button" role="menuitem" onClick={()=>openAccount("model")}><BotIcon/>{t("modelSettings")}</button>
<button type="button" role="menuitem" onClick={()=>openAccount("voice")}><PhoneIcon/>{t("voiceSettings")}</button>
<button type="button" role="menuitem" disabled={!paneBot} onClick={()=>{setAccountOpen(false);openPane("memory")}}><Brain/>{t("memory")}</button>
<button type="button" role="menuitem" onClick={()=>openAccount("about")}><Info/>{t("about")}</button>
<button type="button" role="menuitem" onClick={()=>openAccount("help")}><CircleHelp/>{t("helpCenter")}</button>
@ -222,26 +355,31 @@ export function App(){
</aside>
<main className="chat-panel">
<header className="topbar"><button className="icon-button mobile-menu" onClick={()=>setMobileNav(v=>!v)}><UseAnimations animation={menu} size={18} strokeColor="#dfdfe2"/></button>{activeRoom?<><AvatarStack members={activeRoom.members} online thinkingIds={busyMembers.map(member=>member.id)}/><strong>{activeRoom.name}</strong><SessionMenu sessions={sessions} activeSessionId={activeSessionId} open={sessionMenuOpen} setOpen={setSessionMenuOpen} busy={busy} onSelect={selectSession} onCreate={createSession} onDelete={id=>{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/><span className="grow"/>{topTools}</>:active?<><Avatar lookId={active.id} name={active.name} color={active.avatarColor} shape={active.avatarShape} active online/><strong>{active.name}</strong><SessionMenu sessions={sessions} activeSessionId={activeSessionId} open={sessionMenuOpen} setOpen={setSessionMenuOpen} busy={busy} onSelect={selectSession} onCreate={createSession} onDelete={id=>{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/><span className="grow"/>{topTools}</>:<><strong>{t("chooseBot")}</strong><span className="grow"/>{topTools}</>}</header>
<div className="messages">{(activeRoom||active)&&messages.length===0?<div className="welcome">{activeRoom?<AvatarStack members={activeRoom.members} size={56} online/>:<Avatar lookId={active!.id} name={active!.name} color={active!.avatarColor} shape={active!.avatarShape} active online size={64}/>}<h1>{activeRoom?t("startRoomDiscussion",{name:activeRoom.name}):t("startBotWork",{name:active!.name})}</h1><p>{activeRoom?t("roomWillReply",{names:activeRoom.members.map(member=>member.name).join("、")}):active!.description||t("botWelcome")}</p></div>:messages.map(message=>{const spoken=message.role!=="user"&&Boolean(activeRoom);const speakerName=message.speakerName||(spoken?paneBot?.name:undefined);const speakerShape=(message.speakerShape||paneBot?.avatarShape||"blob") as AvatarShape;const files=messageFiles(message.blocks);const hideBody=isAutoAttachCaption(message.body,files);return <div key={message.id} className={`message ${message.role} ${spoken?"spoken":""} ${files.length?"with-files":""}`}>{spoken&&<span className="msg-avatar"><Avatar lookId={message.speakerBotId||paneBot?.id||undefined} name={speakerName||"agent"} color={message.speakerColor||undefined} shape={speakerShape} size={22}/></span>}{spoken&&<b className="speaker" style={{color:message.speakerColor||undefined}}>{speakerName}</b>}{files.length>0&&<div className="msg-attachments">{files.map(file=><FileCard key={file.name} file={file}/>)}</div>}{!hideBody&&<span className="message-body">{message.body}</span>}{!hideBody&&message.body.trim()&&<button type="button" className={`remember-msg ${remembered[message.id]?"saved":""}`} title={remembered[message.id]?t("remembered"):t("remember")} disabled={!!remembered[message.id]} onClick={()=>void rememberMessage(message)}><UseAnimations animation={bookmark} size={14} strokeColor="var(--muted)"/></button>}</div>})}{workingMembers.map(member=><div className="thinking-row" key={member.id}><Avatar lookId={member.id} name={member.name} color={member.avatarColor} shape={member.avatarShape} thinking online/><span className="working-label">{computer.busyStep&&member.id===computer.botId?t("workingStep",{name:member.name,step:computer.busyStep}):t("working",{name:member.name})}</span></div>)}{pausedForUser&&paneBot&&<div className="pause-banner" role="status"><span>{computer.controlHolder==="user"?t("pausedUserControl",{name:paneBot.name}):t("pausedNeedsUser",{name:paneBot.name})}{(computer.queuedRuns||0)>0&&<> {t("queuedMessages",{count:computer.queuedRuns||0})}</>}</span>{computer.controlHolder==="user"?<button type="button" className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${paneBot.id}/release`,{method:"POST",body:"{}"}))}>{t("releaseAndContinue")}</button>:<button type="button" className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${paneBot.id}/takeover`,{method:"POST",body:"{}"}))}>{t("takeOverNow")}</button>}</div>}{teaching&&active&&<div className="teach-banner recording" role="status"><span><i className="record-dot live"/>{t("teachingLive",{goal:teaching.goal})}<small>{t("teachingHint")}{teaching.eventCount>0&&<> · {t("teachingCaptured",{count:teaching.eventCount})}</>}</small></span><button type="button" className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button type="button" className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>}{drafting&&<div className="teach-banner" role="status"><UseAnimations animation={loading} size={18} wrapperStyle={{display:"inline-block",verticalAlign:"middle"}}/><span>{t("distilling",{goal:drafting.goal})}</span></div>}{skillDraft&&active&&<SkillDraftCard key={skillDraft.id} skill={skillDraft} busy={busy} onSave={(name,playbook)=>void saveSkill(skillDraft,name,playbook)} onTest={(name,playbook)=>void testSkill(skillDraft,name,playbook)} onDiscard={()=>void discardSkill(skillDraft)} onEdit={()=>setEditingSkillId(skillDraft.id)} onExport={(name,playbook)=>downloadSkill(name,skillDraft.goal,playbook)}/>}<div ref={messageEndRef} aria-hidden="true"/></div>
<header className="topbar"><button type="button" className="icon-button mobile-menu" aria-label={t("chooseBot")} aria-expanded={mobileNav} aria-controls="conversation-sidebar" onClick={()=>setMobileNav(v=>!v)}><UseAnimations animation={menu} size={18} strokeColor="var(--ink)"/></button>{activeRoom?<><AvatarStack members={activeRoom.members} size={32} online thinkingIds={busyMembers.map(member=>member.id)}/><strong>{activeRoom.name}</strong><SessionMenu sessions={sessions} activeSessionId={activeSessionId} open={sessionMenuOpen} setOpen={setSessionMenuOpen} busy={busy} onSelect={selectSession} onCreate={createSession} onDelete={id=>{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/><span className="grow"/>{topTools}</>:active?<><Avatar lookId={active.id} name={active.name} color={active.avatarColor} shape={active.avatarShape} active online/><strong>{active.name}</strong><SessionMenu sessions={sessions} activeSessionId={activeSessionId} open={sessionMenuOpen} setOpen={setSessionMenuOpen} busy={busy} onSelect={selectSession} onCreate={createSession} onDelete={id=>{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/><span className="grow"/>{topTools}</>:<><strong>{t("chooseBot")}</strong><span className="grow"/>{topTools}</>}</header>
{callOpen&&voiceSettings?.enabled&&active&&activeSessionId&&!activeRoomId?<CallOverlay bot={active} sessionId={activeSessionId} takeover={computer.takeoverRequested||computer.controlHolder==="user"} onHangUp={()=>setCallOpen(false)} onTakeOver={()=>{setRightPart("computer");setRightCollapsed(false);if(active)void action(()=>api(`/api/computer/${active.id}/takeover`,{method:"POST",body:"{}"}))}}/>:null}
<div className="messages">{(activeRoom||active)&&messages.length===0?<div className="welcome">{activeRoom?<AvatarStack members={activeRoom.members} size={56} online/>:<Avatar lookId={active!.id} name={active!.name} color={active!.avatarColor} shape={active!.avatarShape} active online size={64}/>}<h1>{activeRoom?t("startRoomDiscussion",{name:activeRoom.name}):t("startBotWork",{name:active!.name})}</h1><p>{activeRoom?t("roomWillReply",{names:listJoin(activeRoom.members.map(member=>member.name))}):active!.description||t("botWelcome")}</p></div>:messages.map(message=>{const spoken=message.role!=="user"&&Boolean(activeRoom);const speakerName=message.speakerName||(spoken?paneBot?.name:undefined);const speakerShape=(message.speakerShape||paneBot?.avatarShape||"blob") as AvatarShape;const files=messageFiles(message.blocks);const chips=chipBlocks(message.blocks);const hideBody=isAutoAttachCaption(message.body,files)||isAutoScheduleCaption(message.body,chips);return <div key={message.id} className={`message ${message.role} ${spoken?"spoken":""} ${files.length?"with-files":""}`}>{spoken&&<span className="msg-avatar"><Avatar lookId={message.speakerBotId||paneBot?.id||undefined} name={speakerName||"agent"} color={message.speakerColor||undefined} shape={speakerShape} size={22}/></span>}{spoken&&<b className="speaker" style={{color:message.speakerColor||undefined}}>{speakerName}</b>}{files.length>0&&<div className="msg-attachments">{files.map(file=><FileCard key={file.name} file={file}/>)}</div>}{chips.map((chip,index)=>chip.kind==="login"?<div className="login-chip" key={`${message.id}-login-${index}`}><div className="login-label">{t("loginNeedsYou")}</div><div className="login-site">{chip.site||message.body}</div>{chip.why?<div className="login-why">{t("loginWhy",{why:chip.why})}</div>:null}<button type="button" className="primary" onClick={()=>void openLoginScreen()}>{t("loginOpenScreen")}</button></div>:chip.kind==="schedule"?<div className="sched-chip" key={`${message.id}-sched-${index}`}><div className="sched-label">{t("scheduleChip")}</div><strong>{chip.name}</strong><small>{scheduleWhen(chip.cron,chip.human)}</small></div>:<div className="sched-chip" key={`${message.id}-run-${index}`}><div className="sched-label">{t("scheduleRunChip")}</div><strong>{chip.name}</strong><small>{scheduleWhen(chip.cron,chip.human)}</small></div>)}{!hideBody&&(message.role==="assistant"?<div className="message-stack"><div className="message-body md"><ChatMarkdown>{message.body}</ChatMarkdown></div>{message.body.trim()?<CopyMessageButton text={message.body}/>:null}</div>:<span className="message-body">{message.body}</span>)}{!hideBody&&message.body.trim()&&<button type="button" className={`remember-msg ${remembered[message.id]?"saved":""}`} title={remembered[message.id]?t("remembered"):t("remember")} disabled={!!remembered[message.id]} onClick={()=>void rememberMessage(message)}><UseAnimations animation={bookmark} size={14} strokeColor="var(--muted)"/></button>}</div>})}{pausedForUser&&paneBot&&<div className="pause-banner" role="status"><span>{computer.controlHolder==="user"?t("pausedUserControl",{name:paneBot.name}):t("pausedNeedsUser",{name:paneBot.name})}{(computer.queuedRuns||0)>0&&<> {t("queuedMessages",{count:computer.queuedRuns||0})}</>}</span>{computer.controlHolder==="user"?<button type="button" className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${paneBot.id}/release`,{method:"POST",body:"{}"}))}>{t("releaseAndContinue")}</button>:<button type="button" className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${paneBot.id}/takeover`,{method:"POST",body:"{}"}))}>{t("takeOverNow")}</button>}</div>}{teaching&&active&&<div className="teach-banner recording" role="status"><span><i className="record-dot live"/>{t("teachingLive",{goal:teaching.goal})}<small>{t("teachingHint")}{teaching.eventCount>0&&<> · {t("teachingCaptured",{count:teaching.eventCount})}</>}</small></span><button type="button" className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button type="button" className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>}{drafting&&<div className="teach-banner" role="status"><UseAnimations animation={loading} size={18} wrapperStyle={{display:"inline-block",verticalAlign:"middle"}}/><span>{t("distilling",{goal:drafting.goal})}</span></div>}{skillDraft&&active&&<SkillDraftCard key={skillDraft.id} skill={skillDraft} busy={busy} onSave={(name,playbook)=>void saveSkill(skillDraft,name,playbook)} onTest={(name,playbook)=>void testSkill(skillDraft,name,playbook)} onDiscard={()=>void discardSkill(skillDraft)} onEdit={()=>setEditingSkillId(skillDraft.id)} onExport={(name,playbook)=>downloadSkill(name,skillDraft.goal,playbook)}/>}<div ref={messageEndRef} aria-hidden="true"/></div>
{error&&<div className="error-banner"><span>{error}</span><button onClick={()=>setError(null)}><X/></button></div>}
{otherSessionBusy&&<div className="queue-hint">{t("anotherConversationQueued")}</div>}
<form className={`composer ${pendingFiles.length?"has-files":""}`} onSubmit={send} onDragOver={event=>{event.preventDefault()}} onDrop={event=>{event.preventDefault();if(event.dataTransfer.files.length)addPendingFiles(event.dataTransfer.files)}}><div className="plus-menu-wrap" onClick={event=>event.stopPropagation()}><button type="button" className={`composer-plus ${plusOpen?"open":""}`} disabled={!activeSessionId} title={t("moreActions")} aria-label={t("moreActions")} aria-haspopup="menu" aria-expanded={plusOpen} onClick={()=>setPlusOpen(v=>!v)}><Plus/></button>{plusOpen&&<div className="plus-menu" role="menu"><button type="button" role="menuitem" disabled={!activeSessionId||Boolean(teaching)} title={t("attachFileHint")} onClick={()=>{setPlusOpen(false);attachRef.current?.click()}}><Paperclip/>{t("attachFile")}</button><button type="button" role="menuitem" disabled={!active||Boolean(teaching)||Boolean(drafting)} title={active?t("teachTaskHint"):t("teachNeedsBot")} onClick={()=>{setPlusOpen(false);setTeachOpen(true)}}><i className="record-dot"/>{t("teachTask")}</button><button type="button" role="menuitem" disabled={!active} title={t("importSkillHint")} onClick={()=>{setPlusOpen(false);importRef.current?.click()}}><Upload/>{t("importSkill")}</button>{savedSkills.length>0&&<><hr/><div className="plus-menu-skills"><small className="plus-menu-label">{t("taughtSkills")}{savedSkills.length>5?` · ${savedSkills.length}`:""}</small>{savedSkills.length>=6&&<input className="plus-menu-search" value={skillQuery} onChange={e=>setSkillQuery(e.target.value)} placeholder={t("searchSkills")} aria-label={t("searchSkills")} onClick={e=>e.stopPropagation()}/>}<div className="plus-menu-skill-list">{listedSkills.map(skill=><div className="plus-menu-skill" key={skill.id}><button type="button" role="menuitem" title={t("runSkillNamed",{name:skill.name})+(skill.playbook.whenToUse?`\n${skill.playbook.whenToUse}`:"")} onClick={()=>runSkill(skill)}><Sparkle/>{skill.name}</button><button type="button" className="skill-edit" title={t("exportSkillHint")} aria-label={t("exportSkill")} onClick={()=>downloadSkill(skill.name,skill.goal,skill.playbook)}><Download/></button><button type="button" className="skill-edit" title={t("editSkill")} aria-label={t("editSkill")} onClick={()=>{setPlusOpen(false);setEditingSkillId(skill.id)}}><Pencil/></button></div>)}{listedSkills.length===0&&<small className="plus-menu-empty">{t("noMatchingSkills")}</small>}</div></div></>}</div>}</div><input ref={importRef} className="skill-import-input" type="file" accept="application/json,.json" tabIndex={-1} aria-hidden="true" onChange={event=>{const file=event.target.files?.[0];event.currentTarget.value="";if(file)void importSkillFile(file)}}/><input ref={attachRef} className="skill-import-input attach-input" type="file" multiple accept={ATTACH_ACCEPT} tabIndex={-1} aria-hidden="true" onChange={event=>{const files=[...event.target.files||[]];event.currentTarget.value="";if(files.length)addPendingFiles(files)}}/>{pendingFiles.length>0&&<div className="composer-files">{pendingFiles.map(item=><FileCard key={item.id} file={{name:item.file.name,size:item.file.size}} preview={item.preview} onRemove={()=>removePendingFile(item.id)}/>)}</div>}<textarea rows={1} value={draft} onChange={e=>setDraft(e.target.value)} onKeyDown={composerKeyDown} onPaste={event=>{const files=event.clipboardData?.files;if(files&&files.length){event.preventDefault();addPendingFiles(files)}}} placeholder={teaching?t("teachingComposerHint"):activeSessionId&&chatName?t("messageTo",{name:chatName}):t("chooseConversationFirst")} disabled={!activeSessionId||Boolean(teaching)}/>{sessionBusy?<button type="button" className="send stop-send" title={t("stopConversation")} onClick={()=>void action(stopChat)}><Square/></button>:<button className="send" disabled={!activeSessionId||(!draft.trim()&&pendingFiles.length===0)||busy}><UseAnimations animation={arrowUp} size={20} strokeColor="#1b1b1c"/></button>}</form>
<div className={`composer-dock ${statusMembers.length?"has-status":""}`}>
{statusMembers.map(member=>{const step=computer.busyStep&&member.id===computer.botId?computer.busyStep:null;const transition=isTransitionStep(step);const label=t("working",{name:member.name});return <div className="thinking-row" key={member.id}><Avatar lookId={member.id} name={member.name} color={member.avatarColor} shape={member.avatarShape} thinking online/><span className="working-copy"><span className="working-label">{label}</span>{!transition&&step?<span className="working-step">{localizeStep(step)}</span>:null}</span></div>})}
<form className={`composer ${pendingFiles.length?"has-files":""}`} onSubmit={send} onDragOver={event=>{event.preventDefault()}} onDrop={event=>{event.preventDefault();if(event.dataTransfer.files.length)addPendingFiles(event.dataTransfer.files)}}><div className="plus-menu-wrap" onClick={event=>event.stopPropagation()}><button type="button" className={`composer-plus ${plusOpen?"open":""}`} disabled={!activeSessionId} title={t("moreActions")} aria-label={t("moreActions")} aria-haspopup="menu" aria-expanded={plusOpen} onClick={()=>setPlusOpen(v=>!v)}><Plus/></button>{plusOpen&&<div className="plus-menu" role="menu"><button type="button" role="menuitem" disabled={!activeSessionId||Boolean(teaching)} title={t("attachFileHint")} onClick={()=>{setPlusOpen(false);attachRef.current?.click()}}><Paperclip/>{t("attachFile")}</button><button type="button" role="menuitem" disabled={!active||Boolean(teaching)||Boolean(drafting)} title={active?t("teachTaskHint"):t("teachNeedsBot")} onClick={()=>{setPlusOpen(false);setTeachOpen(true)}}><i className="record-dot"/>{t("teachTask")}</button><button type="button" role="menuitem" disabled={!active} title={t("importSkillHint")} onClick={()=>{setPlusOpen(false);importRef.current?.click()}}><Upload/>{t("importSkill")}</button>{savedSkills.length>0&&<><hr/><div className="plus-menu-skills"><small className="plus-menu-label">{t("taughtSkills")}{savedSkills.length>5?` · ${savedSkills.length}`:""}</small>{savedSkills.length>=6&&<input className="plus-menu-search" value={skillQuery} onChange={e=>setSkillQuery(e.target.value)} placeholder={t("searchSkills")} aria-label={t("searchSkills")} onClick={e=>e.stopPropagation()}/>}<div className="plus-menu-skill-list">{listedSkills.map(skill=><div className="plus-menu-skill" key={skill.id}><button type="button" role="menuitem" title={t("runSkillNamed",{name:skill.name})+(skill.playbook.whenToUse?`\n${skill.playbook.whenToUse}`:"")} onClick={()=>runSkill(skill)}><Sparkle/>{skill.name}</button><button type="button" className="skill-edit" title={t("exportSkillHint")} aria-label={t("exportSkill")} onClick={()=>downloadSkill(skill.name,skill.goal,skill.playbook)}><Download/></button><button type="button" className="skill-edit" title={t("editSkill")} aria-label={t("editSkill")} onClick={()=>{setPlusOpen(false);setEditingSkillId(skill.id)}}><Pencil/></button></div>)}{listedSkills.length===0&&<small className="plus-menu-empty">{t("noMatchingSkills")}</small>}</div></div></>}</div>}</div><input ref={importRef} className="skill-import-input" type="file" accept="application/json,.json" tabIndex={-1} aria-hidden="true" onChange={event=>{const file=event.target.files?.[0];event.currentTarget.value="";if(file)void importSkillFile(file)}}/><input ref={attachRef} className="skill-import-input attach-input" type="file" multiple accept={ATTACH_ACCEPT} tabIndex={-1} aria-hidden="true" onChange={event=>{const files=[...event.target.files||[]];event.currentTarget.value="";if(files.length)addPendingFiles(files)}}/>{pendingFiles.length>0&&<div className="composer-files">{pendingFiles.map(item=><FileCard key={item.id} file={{name:item.file.name,size:item.file.size}} preview={item.preview} onRemove={()=>removePendingFile(item.id)}/>)}</div>}{slashSuggestions.length>0&&<div className="slash-suggestions" role="listbox" aria-label={t("slashList")}>{slashSuggestions.map((skill,index)=><button type="button" role="option" aria-selected={index===slashIndex} key={skill.name} onClick={()=>{setDraft(`/${skill.name} `);requestAnimationFrame(()=>document.querySelector<HTMLTextAreaElement>(".composer textarea")?.focus())}}><strong>/{skill.name}</strong><small>{skill.kind}</small><span>{skill.description}</span></button>)}</div>}<textarea rows={1} value={draft} onChange={e=>setDraft(e.target.value)} onKeyDown={composerKeyDown} onPaste={event=>{const files=event.clipboardData?.files;if(files&&files.length){event.preventDefault();addPendingFiles(files)}}} placeholder={teaching?t("teachingComposerHint"):activeSessionId&&chatName?t("messageTo",{name:chatName}):t("chooseConversationFirst")} disabled={!activeSessionId||Boolean(teaching)}/>{sessionBusy?<button type="button" className="send stop-send" title={t("stopConversation")} onClick={()=>void action(stopChat)}><Square/></button>:<button className="send" disabled={!activeSessionId||(!draft.trim()&&pendingFiles.length===0)||busy}><UseAnimations animation={arrowUp} size={20} strokeColor="var(--on-send)"/></button>}</form>
</div>
</main>
{!rightCollapsed&&<div className="side-card-backdrop" onClick={()=>setRightCollapsed(true)}/>}
{!rightCollapsed&&<aside className="side-card">
<>
<header className="side-card-head">
<span className="side-card-title">{rightPart==="computer"?t("computer"):rightPart==="memory"?t("memory"):rightPart==="plugins"?t("plugins"):t("settings")}</span>
<span className="side-card-title">{rightPart==="computer"?t("computer"):rightPart==="memory"?t("memory"):rightPart==="plugins"?t("plugins"):rightPart==="accounts"?t("accounts"):t("settings")}</span>
<button type="button" className="icon-button" title={t("collapseSidebar")} onClick={()=>setRightCollapsed(true)}><ChevronsRight/></button>
</header>
<div className="side-card-body">
<div className={`side-part computer-part ${rightPart==="computer"?"":"hidden-part"}`}>
<div className="computer-status-row">{paneBot?<span>{t("botComputer",{name:paneBot.name})}</span>:<span>{t("computer")}</span>}{computer.state==="booting"?<UseAnimations animation={loading} size={17} wrapperStyle={{display:"inline-block",verticalAlign:"middle"}}/>:<i className={`state-dot ${computer.state}`}/>}<small>{stateLabel(computer.state)}</small></div>
<div className="preview">{computerOpen?<EmptyComputer state={computer.state}/>:frame}</div>
{paneBot&&<><div className="computer-caption"><span>{t("dedicatedScreen")}</span><button className="outline" onClick={()=>setComputerOpen(true)}>{t("enlarge")}</button></div>{teaching?<div className="control-bar"><div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div></div>:<ControlBar active={paneBot} computer={computer} busy={busy} action={action} paste={pasteClipboard} copy={copyClipboard} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer}/>}</>}
<div className="preview">{computerOpen?<EmptyComputer state={computer.state}/>:frame}{!computerOpen&&hud}</div>
{paneBot&&<><div className="computer-caption"><span>{t("dedicatedScreen")}</span><button className="outline" onClick={()=>setComputerOpen(true)}>{t("enlarge")}</button></div>{teaching?<div className="control-bar"><div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div></div>:<ControlBar active={paneBot} computer={computer} busy={busy} action={action} paste={pasteClipboard} copy={copyClipboard} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer} onStop={stopComputer}/>}<p className="computer-login-hint">{t("computerLoginHint")}</p><p className="clipboard-status" role="status">{clipboardStatus}</p>{scheduleDraft?<ScheduleEditor draft={scheduleDraft} timezone={scheduleDraft.timezone||Intl.DateTimeFormat().resolvedOptions().timeZone||"Asia/Taipei"} saving={scheduleSaving} error={scheduleError} onChange={next=>setScheduleDraft(current=>current?{...current,...next}:next)} onBack={()=>setScheduleDraft(null)} onSave={()=>void saveScheduleDraft()} onDelete={scheduleDraft.id?()=>void action(async()=>{await api(`/api/schedules/${scheduleDraft.id}`,{method:"DELETE"});setScheduleDraft(null);await reloadSchedules()}):undefined}/>:<ScheduleList items={schedules} runningId={runningScheduleId} onCreate={()=>setScheduleDraft({name:"",instructions:"",enabled:true,preset:defaultCronPreset()})} onOpen={item=>setScheduleDraft({id:item.id,timezone:item.timezone,threadId:item.threadId,name:item.name,instructions:item.instructions,enabled:item.enabled,preset:presetFromCron(item.cron)})} onRun={item=>void action(async()=>{setRunningScheduleId(item.id);try{await api(`/api/schedules/${item.id}/run`,{method:"POST",body:"{}"});await loadSessions()}finally{setRunningScheduleId(null)}})}/>}</>}
</div>
{rightPart==="accounts"&&paneBot&&<VaultPane bot={paneBot}/>}
{rightPart==="memory"&&paneBot&&<MemoryPane bot={paneBot} changed={loadBots}/>}
{rightPart==="plugins"&&<McpPane servers={mcpServers} reload={loadMcp}/>}
{rightPart==="settings"&&active&&<BotSettingsPane bot={active} look={looks[active.id]||DEFAULT_LOOK} onLook={look=>{writeAvatarLook(active.id,look);setLooks(readAvatarLooks())}} saved={loadBots} onDelete={()=>setDeleteOpen(true)}/>}
@ -249,10 +387,10 @@ export function App(){
</>
</aside>}
{computerOpen&&paneBot&&<div className="computer-overlay"><header><div><Avatar lookId={paneBot.id} name={paneBot.name} color={paneBot.avatarColor} shape={paneBot.avatarShape} active online/><strong>{modeLabel(computer.mode)}</strong><span className={`control-badge ${teaching?"teaching":""}`}>{teaching?t("teachingBadge"):computer.controlHolder==="user"?t("userControlling"):computer.busyBotName?t("aiReadOnly"):t("readOnly")}</span></div><div>{teaching?<div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>:<ControlButtons computer={computer} busy={busy} action={action} active={paneBot} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer}/>}<button className="icon-button" onClick={pasteClipboard} disabled={computer.controlHolder!=="user"} title={t("pasteClipboard")}><ClipboardPaste/></button><button className="icon-button" onClick={copyClipboard} disabled={computer.controlHolder!=="user"||!desktopClipboard} title={t("copyDesktopClipboard")}><UseAnimations animation={copy} size={18} strokeColor="#dfdfe2"/></button><button className="icon-button" title={t("moreActions")}><Ellipsis/></button><button className="icon-button" onClick={()=>setComputerOpen(false)}><X/></button></div></header><div className="overlay-screen">{frame}</div>{error&&<div className="overlay-error">{error}</div>}</div>}
{computerOpen&&paneBot&&<div className="computer-overlay"><header><div><Avatar lookId={paneBot.id} name={paneBot.name} color={paneBot.avatarColor} shape={paneBot.avatarShape} active online/><strong>{modeLabel(computer.mode)}</strong><span className={`control-badge ${teaching?"teaching":""}`}>{teaching?t("teachingBadge"):computer.controlHolder==="user"?t("userControlling"):computer.usingComputer?t("aiReadOnly"):t("readOnly")}</span></div><div>{teaching?<div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>:<ControlButtons computer={computer} busy={busy} action={action} active={paneBot} sessionId={activeSessionId} onBoot={startBoot} onRestart={restartComputer} onStop={stopComputer}/>}<button className="icon-button" onClick={pasteClipboard} disabled={computer.controlHolder!=="user"} title={t("pasteClipboard")}><ClipboardPaste/></button><button className="icon-button" onClick={copyClipboard} disabled={computer.controlHolder!=="user"||!desktopClipboard} title={t("copyDesktopClipboard")}><UseAnimations animation={copy} size={18} strokeColor="var(--ink)"/></button><button className="icon-button" onClick={()=>setComputerOpen(false)}><X/></button></div></header><div className="overlay-screen"><div className="overlay-desktop">{frame}{hud}</div></div>{error&&<div className="overlay-error">{error}</div>}</div>}
{teachOpen&&active&&<TeachDialog bot={active} busy={busy} close={()=>setTeachOpen(false)} start={goal=>void startTeaching(goal)}/>}
{editingSkill&&<SkillEditDialog key={editingSkill.id} skill={editingSkill} busy={busy} close={()=>setEditingSkillId(null)} save={(name,playbook)=>void updateSkill(editingSkill,name,playbook)} test={(name,playbook)=>void testSkill(editingSkill,name,playbook)} remove={()=>void deleteSkill(editingSkill)} exportFile={(name,playbook)=>downloadSkill(name,editingSkill.goal,playbook)}/>}
{clipboardOpen&&<ClipboardDialog close={()=>setClipboardOpen(false)} paste={text=>{document.querySelectorAll<HTMLIFrameElement>(".desktop-frame").forEach(frame=>frame.contentWindow?.postMessage({type:"lazyboy-host-clipboard",text},location.origin));setClipboardOpen(false)}}/>}
{clipboardOpen&&<ClipboardDialog close={()=>setClipboardOpen(false)} paste={text=>{pasteText(text);setClipboardOpen(false)}}/>}
{createOpen&&<CreateDialog close={()=>setCreateOpen(false)} created={async bot=>{setCreateOpen(false);await loadBots();setActiveId(bot.id)}}/>}
{groupOpen&&<CreateGroupDialog bots={bots} close={()=>setGroupOpen(false)} created={async room=>{setGroupOpen(false);setRooms(current=>[room,...current.filter(item=>item.id!==room.id)]);setActiveId(null);setActiveSessionId(null);setBusyMembers([]);setMessages([]);setRightPart("computer");setActiveRoomId(room.id);await loadBots()}}/>}
@ -265,13 +403,14 @@ export function App(){
{accountDialog==="phone"&&<PhoneAccessDialog close={()=>setAccountDialog(null)}/>}
{accountDialog==="settings"&&<WorkspaceSettingsDialog name={workspaceName} setName={setWorkspaceName} showHidden={showHidden} setShowHidden={setShowHidden} rightCollapsed={rightCollapsed} setRightCollapsed={setRightCollapsed} close={()=>setAccountDialog(null)}/>}
{accountDialog==="model"&&<ModelSettingsDialog close={()=>setAccountDialog(null)}/>}
{accountDialog==="voice"&&<VoiceSettingsDialog close={()=>{setAccountDialog(null);api<VoiceSettings>("/api/voice/settings").then(setVoiceSettings).catch(()=>setVoiceSettings(null))}}/>}
{accountDialog==="about"&&<AboutDialog close={()=>setAccountDialog(null)}/>}
{accountDialog==="help"&&<HelpDialog close={()=>setAccountDialog(null)}/>}
{accountDialog==="feedback"&&<FeedbackDialog close={()=>setAccountDialog(null)}/>}
</div></AvatarLookProvider>
}
function BotContextMenu({context,close,run}:{context:{bot:Bot;x:number;y:number};close:()=>void;run:(action:string,group?:string|null)=>void}){const bot=context.bot;return <div className="context-menu" style={{left:Math.min(context.x,window.innerWidth-220),top:Math.min(context.y,window.innerHeight-280)}} onClick={e=>e.stopPropagation()}><button onClick={()=>run(bot.pinned?"unpin":"pin")}><Pin/>{bot.pinned?t("unpin"):t("pin")}</button><button onClick={()=>run("unread")}><UseAnimations animation={mail} size={15} strokeColor="#dfdfe2"/>{t("markUnread")}</button><button onClick={()=>{const name=window.prompt(t("enterGroupName"),bot.groupName||"");if(name!==null)run("group",name)}}><UseAnimations animation={folder} size={15} strokeColor="#dfdfe2"/>{bot.groupName?t("changeGroup"):t("createOrMoveGroup")}</button>{bot.groupName&&<button onClick={()=>run("group",null)}><X/>{t("removeFromGroup")}</button>}<button onClick={()=>run(bot.hidden?"show":"hide")}>{bot.hidden?<UseAnimations animation={visibility} size={15} strokeColor="#dfdfe2"/>:<UseAnimations animation={visibility2} size={15} strokeColor="#dfdfe2"/>}{bot.hidden?t("unhide"):t("hide")}</button><hr/><button className="danger-item" onClick={()=>run("delete")}><UseAnimations animation={trash2} size={15} strokeColor="#ff7777"/>{t("delete")}</button><button className="context-close" onClick={close}><X/></button></div>}
function BotContextMenu({context,close,run}:{context:{bot:Bot;x:number;y:number};close:()=>void;run:(action:string,group?:string|null)=>void}){const bot=context.bot;return <div className="context-menu" style={{left:Math.min(context.x,window.innerWidth-220),top:Math.min(context.y,window.innerHeight-280)}} onClick={e=>e.stopPropagation()}><button onClick={()=>run(bot.pinned?"unpin":"pin")}><Pin/>{bot.pinned?t("unpin"):t("pin")}</button><button onClick={()=>run("unread")}><UseAnimations animation={mail} size={15} strokeColor="var(--ink)"/>{t("markUnread")}</button><button onClick={()=>{const name=window.prompt(t("enterGroupName"),bot.groupName||"");if(name!==null)run("group",name)}}><UseAnimations animation={folder} size={15} strokeColor="var(--ink)"/>{bot.groupName?t("changeGroup"):t("createOrMoveGroup")}</button>{bot.groupName&&<button onClick={()=>run("group",null)}><X/>{t("removeFromGroup")}</button>}<button onClick={()=>run(bot.hidden?"show":"hide")}>{bot.hidden?<UseAnimations animation={visibility} size={15} strokeColor="var(--ink)"/>:<UseAnimations animation={visibility2} size={15} strokeColor="var(--ink)"/>}{bot.hidden?t("unhide"):t("hide")}</button><hr/><button className="danger-item" onClick={()=>run("delete")}><UseAnimations animation={trash2} size={15} strokeColor="var(--danger-soft)"/>{t("delete")}</button><button className="context-close" onClick={close}><X/></button></div>}
function SessionMenu({sessions,activeSessionId,open,setOpen,busy,onSelect,onCreate,onDelete,onClear}:{sessions:Session[];activeSessionId:string|null;open:boolean;setOpen:(fn:(value:boolean)=>boolean)=>void;busy:boolean;onSelect:(id:string)=>void;onCreate:()=>void;onDelete:(id:string)=>void;onClear:()=>void}){
const active=sessions.find(session=>session.id===activeSessionId);
@ -288,7 +427,7 @@ function SessionMenu({sessions,activeSessionId,open,setOpen,busy,onSelect,onCrea
<strong>{session.title}</strong>
<time>{inboxTime(session.updatedAt)}</time>
</button>
<button type="button" className="icon-button" title={t("deleteConversation")} disabled={busy} onClick={()=>onDelete(session.id)}><UseAnimations animation={trash2} size={18} strokeColor="#dfdfe2"/></button>
<button type="button" className="icon-button" title={t("deleteConversation")} disabled={busy} onClick={()=>onDelete(session.id)}><UseAnimations animation={trash2} size={18} strokeColor="var(--ink)"/></button>
</div>)}</div>
<hr/>
<button type="button" className="danger-item" disabled={busy||!activeSessionId} onClick={onClear}>{t("clearConversation")}</button>
@ -308,8 +447,8 @@ const EXPR_I18N = {idle:"exprIdle",happy:"exprHappy",sad:"exprSad",mad:"exprMad"
const BG_I18N = {none:"bgNone",circle:"bgCircle",squircle:"bgSquircle",square:"bgSquare"} as const satisfies Record<AvatarBackground, MessageKey>;
function BotSettingsPane({bot,look,onLook,saved,onDelete}:{bot:Bot;look:AvatarLook;onLook:(look:AvatarLook)=>void;saved:()=>Promise<void>|void;onDelete:()=>void}){
const[name,setName]=useState(bot.name);const[title,setTitle]=useState(bot.title);const[description,setDescription]=useState(bot.description);const[color,setColor]=useState(bot.avatarColor||"#8B5CF6");const[shape,setShape]=useState<AvatarShape>(resolveBlobatarShape(bot.avatarShape));const[expression,setExpression]=useState<AvatarExpression>(look.expression);const[background,setBackground]=useState<AvatarBackground>(look.background);const[tagText,setTagText]=useState((bot.tags||[]).join("、"));const[busy,setBusy]=useState(false);
useEffect(()=>{setName(bot.name);setTitle(bot.title);setDescription(bot.description);setColor(bot.avatarColor||"#8B5CF6");setShape(resolveBlobatarShape(bot.avatarShape));setExpression(look.expression);setBackground(look.background);setTagText((bot.tags||[]).join("、"))},[bot.id,bot.name,bot.title,bot.description,bot.avatarColor,bot.avatarShape,bot.tags,look.expression,look.background]);
const[name,setName]=useState(bot.name);const[title,setTitle]=useState(bot.title);const[description,setDescription]=useState(bot.description);const[color,setColor]=useState(bot.avatarColor||"#8B5CF6");const[shape,setShape]=useState<AvatarShape>(resolveBlobatarShape(bot.avatarShape));const[expression,setExpression]=useState<AvatarExpression>(look.expression);const[background,setBackground]=useState<AvatarBackground>(look.background);const[tagText,setTagText]=useState(listJoin(bot.tags||[]));const[busy,setBusy]=useState(false);
useEffect(()=>{setName(bot.name);setTitle(bot.title);setDescription(bot.description);setColor(bot.avatarColor||"#8B5CF6");setShape(resolveBlobatarShape(bot.avatarShape));setExpression(look.expression);setBackground(look.background);setTagText(listJoin(bot.tags||[]))},[bot.id,bot.name,bot.title,bot.description,bot.avatarColor,bot.avatarShape,bot.tags,look.expression,look.background]);
const colors=["#08A99D","#F1F2F2","#956A43","#DD263B","#F36C05","#F39A00","#00B873","#1985E6","#7140D9","#DC2781","#A7A7A7"];
const face={name:name||bot.name,color,shape,expression,background};
return <form className="pane-form settings-pane" onSubmit={async e=>{e.preventDefault();if(!name.trim())return;setBusy(true);try{await api(`/api/bots/${bot.id}`,{method:"PATCH",body:JSON.stringify({name:name.trim(),title,description,avatarColor:color,avatarShape:persistBlobatarShape(shape),tags:tagText.split(/[、,]/).map(v=>v.trim()).filter(Boolean)})});onLook({expression,background});await saved()}finally{setBusy(false)}}}>
@ -326,16 +465,40 @@ function BotSettingsPane({bot,look,onLook,saved,onDelete}:{bot:Bot;look:AvatarLo
</form>
}
function VaultPane({bot}:{bot:Bot}){
type Account={id:string;site:string;host:string;username:string;notes:string};
const[items,setItems]=useState<Account[]>([]);const[busy,setBusy]=useState(false);const[error,setError]=useState("");
const[form,setForm]=useState({site:"",host:"",username:"",password:"",notes:""});
const load=useCallback(()=>api<Account[]>(`/api/bots/${bot.id}/accounts`).then(setItems),[bot.id]);
useEffect(()=>{load().catch(e=>setError(e instanceof Error?localizeError(e.message):t("loadFailed")))},[load]);
async function run(work:()=>Promise<unknown>){setBusy(true);setError("");try{await work();await load()}catch(e){setError(e instanceof Error?localizeError(e.message):t("operationFailed"))}finally{setBusy(false)}}
function setField<K extends keyof typeof form>(key:K,value:string){setForm(current=>({...current,[key]:value}))}
return <div className="vault-pane">
<p className="memory-help">{t("accountsHelp")}</p>
<form className="vault-form" onSubmit={e=>{e.preventDefault();if(!form.site.trim()||!form.username.trim()||!form.password)return;void run(async()=>{await api(`/api/bots/${bot.id}/accounts`,{method:"POST",body:JSON.stringify(form)});setForm({site:"",host:"",username:"",password:"",notes:""})})}}>
<div className="vault-fields">
<label>{t("accountSite")}<input value={form.site} onChange={e=>setField("site",e.target.value)} placeholder={t("accountSitePlaceholder")} autoComplete="off"/></label>
<label>{t("accountHost")}<input value={form.host} onChange={e=>setField("host",e.target.value)} placeholder={t("accountHostPlaceholder")} autoComplete="off"/></label>
<label>{t("accountUsername")}<input value={form.username} onChange={e=>setField("username",e.target.value)} autoComplete="username"/></label>
<label>{t("accountPassword")}<input type="password" value={form.password} onChange={e=>setField("password",e.target.value)} autoComplete="new-password"/></label>
</div>
<label>{t("accountNotes")}<input value={form.notes} onChange={e=>setField("notes",e.target.value)} placeholder={t("accountNotesPlaceholder")} autoComplete="off"/></label>
<button className="primary" disabled={busy||!form.site.trim()||!form.username.trim()||!form.password}>{t("addAccount")}</button>
</form>
<div className="account-list">{items.length===0?<p className="vault-empty">{t("noAccounts")}</p>:items.map(item=><div className="account-row" key={item.id}><div className="account-row-main"><strong>{item.site}</strong><small>{item.username}{item.host?` · ${item.host}`:""}</small>{item.notes?<small className="account-notes">{item.notes}</small>:null}</div><button type="button" className="icon-button" title={t("delete")} aria-label={t("delete")} disabled={busy} onClick={()=>void run(()=>api(`/api/bots/${bot.id}/accounts/${item.id}`,{method:"DELETE"}))}><UseAnimations animation={trash2} size={18} strokeColor="var(--ink)"/></button></div>)}</div>
{error&&<div className="pane-error">{error}</div>}
</div>
}
function MemoryPane({bot,changed}:{bot:Bot;changed:()=>Promise<void>}){
const[items,setItems]=useState<MemoryItem[]>([]);const[draft,setDraft]=useState("");const[query,setQuery]=useState("");
const[enabled,setEnabled]=useState(bot.memoryEnabled);const[busy,setBusy]=useState(false);const[error,setError]=useState("");
const[editingId,setEditingId]=useState<string|null>(null);const[editingText,setEditingText]=useState("");const[confirmClear,setConfirmClear]=useState(false);
const load=useCallback(()=>api<MemoryItem[]>(`/api/bots/${bot.id}/memories`).then(setItems),[bot.id]);
useEffect(()=>{load().catch(e=>setError(e instanceof Error?e.message:t("loadFailed")))},[load]);
useEffect(()=>{load().catch(e=>setError(e instanceof Error?localizeError(e.message):t("loadFailed")))},[load]);
useEffect(()=>setEnabled(bot.memoryEnabled),[bot.id,bot.memoryEnabled]);
const visible=items.filter(item=>!query.trim()||item.content.toLowerCase().includes(query.trim().toLowerCase()));
async function run(work:()=>Promise<unknown>){setBusy(true);setError("");try{await work();await load()}catch(e){setError(e instanceof Error?e.message:t("operationFailed"))}finally{setBusy(false)}}
async function toggle(value:boolean){setEnabled(value);try{await api(`/api/bots/${bot.id}`,{method:"PATCH",body:JSON.stringify({name:bot.name,title:bot.title,description:bot.description,avatarColor:bot.avatarColor,avatarShape:bot.avatarShape,tags:bot.tags,memoryEnabled:value})});await changed()}catch(e){setEnabled(!value);setError(e instanceof Error?e.message:t("settingsFailed"))}}
async function run(work:()=>Promise<unknown>){setBusy(true);setError("");try{await work();await load()}catch(e){setError(e instanceof Error?localizeError(e.message):t("operationFailed"))}finally{setBusy(false)}}
async function toggle(value:boolean){setEnabled(value);try{await api(`/api/bots/${bot.id}`,{method:"PATCH",body:JSON.stringify({name:bot.name,title:bot.title,description:bot.description,avatarColor:bot.avatarColor,avatarShape:bot.avatarShape,tags:bot.tags,memoryEnabled:value})});await changed()}catch(e){setEnabled(!value);setError(e instanceof Error?localizeError(e.message):t("settingsFailed"))}}
return <div className="memory-pane">
<p className="memory-help">{t("memoryHelp")}</p>
<label className="memory-toggle"><input type="checkbox" checked={enabled} onChange={e=>void toggle(e.target.checked)}/> {t("enableLongTermMemory")}</label>
@ -365,7 +528,7 @@ function MemoryPane({bot,changed}:{bot:Bot;changed:()=>Promise<void>}){
function parsePairs(text:string){const out:Record<string,string>={};for(const line of text.split(/\n+/)){const trimmed=line.trim();if(!trimmed)continue;const cut=trimmed.indexOf("=");if(cut<=0)continue;out[trimmed.slice(0,cut).trim()]=trimmed.slice(cut+1)}return out}
function McpPane({servers,reload}:{servers:McpServer[];reload:()=>Promise<void>}){
const[picker,setPicker]=useState(false);const[busy,setBusy]=useState(false);const[error,setError]=useState("");const[openId,setOpenId]=useState<string|null>(null);
async function run(work:()=>Promise<unknown>){setBusy(true);setError("");try{await work();await reload()}catch(e){setError(e instanceof Error?e.message:t("operationFailed"))}finally{setBusy(false)}}
async function run(work:()=>Promise<unknown>){setBusy(true);setError("");try{await work();await reload()}catch(e){setError(e instanceof Error?localizeError(e.message):t("operationFailed"))}finally{setBusy(false)}}
return <div className="mcp-pane">
<p className="memory-help">{t("mcpHelp")}</p>
<button type="button" className="primary mcp-choose" disabled={busy} onClick={()=>setPicker(true)}><Plug/>{t("chooseMcp")}</button>
@ -396,7 +559,7 @@ function McpPickerDialog({added,close,connected}:{added:string[];close:()=>void;
const[secrets,setSecrets]=useState<Record<string,string>>({});
const[busy,setBusy]=useState(false);const[error,setError]=useState("");
const addedSet=useMemo(()=>new Set(added.map(name=>name.toLowerCase())),[added]);
useEffect(()=>{const timer=setTimeout(()=>{setLoading(true);api<{servers:McpCatalogEntry[]}>(`/api/mcp-catalog?q=${encodeURIComponent(query.trim())}`).then(result=>setItems(result.servers)).catch(e=>setError(e instanceof Error?e.message:t("loadFailed"))).finally(()=>setLoading(false))},query?280:0);return()=>clearTimeout(timer)},[query]);
useEffect(()=>{const timer=setTimeout(()=>{setLoading(true);api<{servers:McpCatalogEntry[]}>(`/api/mcp-catalog?q=${encodeURIComponent(query.trim())}`).then(result=>setItems(result.servers)).catch(e=>setError(e instanceof Error?localizeError(e.message):t("loadFailed"))).finally(()=>setLoading(false))},query?280:0);return()=>clearTimeout(timer)},[query]);
function needsSecrets(entry:McpCatalogEntry){return entry.envKeys.some(field=>field.required)||entry.headerKeys.some(field=>field.required)}
async function connect(entry:McpCatalogEntry,values:Record<string,string>){
const env:Record<string,string>={};const headers:Record<string,string>={};
@ -407,7 +570,7 @@ function McpPickerDialog({added,close,connected}:{added:string[];close:()=>void;
const server=await api<McpServer>("/api/mcp-servers",{method:"POST",body:JSON.stringify({name:entry.title.slice(0,80),transport:entry.transport,command:entry.command,args:entry.args,url:entry.url,env,headers,enabled:true})});
if(server.enabled&&server.status!=="connected"){setError(server.error||t("mcpConnectFailed"));return}
await connected();
}catch(e){setError(e instanceof Error?e.message:t("operationFailed"))}
}catch(e){setError(e instanceof Error?localizeError(e.message):t("operationFailed"))}
finally{setBusy(false)}
}
function choose(entry:McpCatalogEntry){
@ -424,7 +587,7 @@ function McpPickerDialog({added,close,connected}:{added:string[];close:()=>void;
{[...picked.envKeys,...picked.headerKeys].map(field=><label key={field.name}>{field.name}{field.required?" *":""}<input type={field.secret?"password":"text"} autoComplete="off" value={secrets[field.name]||""} onChange={e=>setSecrets(current=>({...current,[field.name]:e.target.value}))} placeholder={field.hint||field.name}/></label>)}
{error&&<div className="pane-error">{error}</div>}
<div className="dialog-actions"><button type="button" className="outline" onClick={()=>setPicked(null)}>{t("cancel")}</button><button type="button" className="primary" disabled={busy} onClick={()=>void connect(picked,secrets)}>{busy?t("connecting"):t("connectMcp")}</button></div>
</div>:custom?<McpCustomForm busy={busy} error={error} onBack={()=>setCustom(false)} onSubmit={async payload=>{setBusy(true);setError("");try{const server=await api<McpServer>("/api/mcp-servers",{method:"POST",body:JSON.stringify(payload)});if(server.enabled&&server.status!=="connected"){setError(server.error||t("mcpConnectFailed"));setBusy(false);return}await connected()}catch(e){setError(e instanceof Error?e.message:t("operationFailed"));setBusy(false)}}}/>:<>
</div>:custom?<McpCustomForm busy={busy} error={error} onBack={()=>setCustom(false)} onSubmit={async payload=>{setBusy(true);setError("");try{const server=await api<McpServer>("/api/mcp-servers",{method:"POST",body:JSON.stringify(payload)});if(server.enabled&&server.status!=="connected"){setError(server.error||t("mcpConnectFailed"));setBusy(false);return}await connected()}catch(e){setError(e instanceof Error?localizeError(e.message):t("operationFailed"));setBusy(false)}}}/>:<>
<p className="dialog-lead">{t("mcpPickerLead")}</p>
<label className="mcp-picker-search"><UseAnimations animation={searchToX} size={16} strokeColor="var(--muted)"/><input autoFocus value={query} onChange={e=>setQuery(e.target.value)} placeholder={t("searchMcp")}/></label>
<div className="mcp-picker-grid">{loading&&items.length===0?<div className="mcp-picker-empty">{t("connecting")}</div>:items.length===0?<div className="mcp-picker-empty">{t("mcpNoResults")}</div>:items.map(entry=>{const already=addedSet.has(entry.title.toLowerCase())||addedSet.has(entry.id.toLowerCase());return <button type="button" className={`mcp-pick-card ${already?"added":""}`} key={entry.id} disabled={already||busy} onClick={()=>choose(entry)}>
@ -460,9 +623,42 @@ function McpCustomForm({busy,error,onBack,onSubmit}:{busy:boolean;error:string;o
</form>
}
function EmptyComputer({state}:{state:ComputerStatus["state"]}){const loading=state==="booting";return <div className="empty-computer">{loading?<UseAnimations animation={loading2} size={38} wrapperStyle={{display:"block"}}/>:<Computer/>}<strong>{stateLabel(state)}</strong><span>{loading?t("preparingDesktop"):t("computerPreviewHint")}</span></div>}
function ControlButtons({computer,busy,action,active,sessionId,onBoot,onRestart}:{computer:ComputerStatus;busy:boolean;action:(w:()=>Promise<unknown>)=>Promise<void>;active:Bot;sessionId?:string|null;onBoot?:()=>Promise<void>;onRestart?:()=>Promise<void>}){const working=Boolean(computer.busyBotName);const restart=<button className="outline restart-computer" disabled={busy} title={t("restartDocker")} onClick={()=>void action(onRestart||(()=>api(`/api/computer/${active.id}/restart`,{method:"POST",body:"{}"})))}><RefreshCw/> {t("restartDocker")}</button>;if(computer.state!=="running")return <div className="computer-actions"><button className="primary" disabled={busy||computer.state==="booting"} onClick={()=>void action(onBoot||(()=>api(`/api/computer/${active.id}/boot`,{method:"POST",body:"{}"})))}>{(busy||computer.state==="booting")&&<UseAnimations animation={loading} size={17} wrapperStyle={{display:"inline-block",verticalAlign:"middle",marginRight:7}}/>}{computer.state==="booting"?t("bootingProgress"):t("openComputer")}</button>{(computer.state==="booting"||computer.state==="error")&&restart}</div>;if(working)return <div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>action(()=>api(`/api/computer/${active.id}/takeover`,{method:"POST",body:"{}"}))}>{t("takeOverNow")}</button><button className="outline" disabled={busy} onClick={()=>action(async()=>{if(sessionId)await api(`/api/sessions/${sessionId}/stop`,{method:"POST",body:"{}"});else await api(`/api/bots/${active.id}/stop`,{method:"POST",body:"{}"});})}><Square/>{t("stopTask")}</button>{restart}</div>;if(computer.controlHolder==="user")return <div className="computer-actions"><button className="outline" disabled={busy} onClick={()=>action(()=>api(`/api/computer/${active.id}/release`,{method:"POST",body:"{}"}))}>{t("releaseControl")}</button>{restart}</div>;return <div className="computer-actions"><button className="primary" disabled={busy} onClick={()=>action(()=>api(`/api/computer/${active.id}/takeover`,{method:"POST",body:"{}"}))}>{t("takeControl")}</button>{restart}</div>}
function ControlBar(props:{active:Bot;computer:ComputerStatus;busy:boolean;action:(w:()=>Promise<unknown>)=>Promise<void>;paste:()=>void;copy:()=>void;sessionId?:string|null;onBoot?:()=>Promise<void>;onRestart?:()=>Promise<void>}){const interactive=props.computer.controlHolder==="user";return <div className="control-bar"><ControlButtons {...props}/><button className="icon-button" disabled={!interactive} onClick={props.paste}><ClipboardPaste/></button><button className="icon-button" disabled={!interactive} onClick={props.copy}><UseAnimations animation={copy} size={18} strokeColor="#dfdfe2"/></button></div>}
function ComputerHud({bot,label}:{bot:{id:string;name:string;avatarColor?:string;avatarShape?:AvatarShape};label:string}){
const sep=getLocale()==="en"?": ":"";
return <div className="computer-hud" role="status" aria-label={`${bot.name}${sep}${label}`}><span className="computer-signal" aria-hidden="true"><span className="computer-signal-face"><i/><i/></span></span><span className="computer-hud-label">{label}</span></div>
}
function EmptyComputer({state}:{state:ComputerStatus["state"]}){if(state==="booting"||state==="suspended")return <div className="empty-computer is-waiting" aria-hidden="true"/>;return <div className="empty-computer"><Computer/><strong>{stateLabel(state)}</strong><span>{t("computerPreviewHint")}</span></div>}
function ControlButtons({computer,busy,action,active,sessionId,onBoot,onRestart,onStop}:{computer:ComputerStatus;busy:boolean;action:(w:()=>Promise<unknown>)=>Promise<void>;active:Bot;sessionId?:string|null;onBoot?:()=>Promise<void>;onRestart?:()=>Promise<void>;onStop?:()=>Promise<void>}){
const [menuOpen,setMenuOpen]=useState(false);
const menuRef=useRef<HTMLDivElement>(null);
useEffect(()=>{
if(!menuOpen)return;
const outside=(event:PointerEvent)=>{if(!menuRef.current?.contains(event.target as Node))setMenuOpen(false)};
const escape=(event:KeyboardEvent)=>{if(event.key==="Escape"){event.stopPropagation();setMenuOpen(false);menuRef.current?.querySelector<HTMLButtonElement>("button")?.focus()}};
document.addEventListener("pointerdown",outside);document.addEventListener("keydown",escape);
return()=>{document.removeEventListener("pointerdown",outside);document.removeEventListener("keydown",escape)};
},[menuOpen]);
useEffect(()=>setMenuOpen(false),[active.id,computer.state]);
const working=Boolean(computer.usingComputer);
const running=computer.state==="running";
const booting=computer.state==="booting";
const run=(operation:"boot"|"restart"|"stop",handler?:()=>Promise<void>)=>{setMenuOpen(false);void action(handler||(()=>api(`/api/computer/${active.id}/${operation}`,{method:"POST",body:"{}"})))};
return <div className="computer-actions">
{!running?<button className="primary" disabled={busy||booting} onClick={()=>run("boot",onBoot)}>{booting?t("bootingProgress"):t("openComputer")}</button>
:computer.controlHolder==="user"?<button className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${active.id}/release`,{method:"POST",body:"{}"}))}>{t("releaseControl")}</button>
:<button className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${active.id}/takeover`,{method:"POST",body:"{}"}))}>{working?t("takeOverNow"):t("takeControl")}</button>}
{running&&working&&<button className="outline" disabled={busy} onClick={()=>void action(()=>api(sessionId?`/api/sessions/${sessionId}/stop`:`/api/bots/${active.id}/stop`,{method:"POST",body:"{}"}))}><Square/>{t("stopTask")}</button>}
{computer.state!=="stopped"&&<div className="computer-power" ref={menuRef}>
<button type="button" className="outline computer-power-trigger" disabled={busy||booting} aria-expanded={menuOpen} aria-label={t("computerPower")} title={t("computerPower")} onClick={()=>setMenuOpen(open=>!open)}><Ellipsis/></button>
{menuOpen&&<div className="computer-power-menu">
<button type="button" disabled={busy} onClick={()=>run("restart",onRestart)}><RefreshCw/>{t("restartComputer")}</button>
<button type="button" className="computer-power-stop" disabled={busy} onClick={()=>run("stop",onStop)}><Square/>{t("shutDownComputer")}</button>
{computer.mode==="team"&&<small>{t("sharedPowerHint")}</small>}
</div>}
</div>}
</div>;
}
function ControlBar(props:{active:Bot;computer:ComputerStatus;busy:boolean;action:(w:()=>Promise<unknown>)=>Promise<void>;paste:()=>void;copy:()=>void;sessionId?:string|null;onBoot?:()=>Promise<void>;onRestart?:()=>Promise<void>;onStop?:()=>Promise<void>}){const interactive=props.computer.controlHolder==="user";return <div className="control-bar"><ControlButtons {...props}/><button className="icon-button" disabled={!interactive} title={t("pasteClipboard")} onClick={props.paste}><ClipboardPaste/></button><button className="icon-button" disabled={!interactive} title={t("copyDesktopClipboard")} onClick={props.copy}><UseAnimations animation={copy} size={18} strokeColor="var(--ink)"/></button></div>}
function TeachDialog({bot,busy,close,start}:{bot:Bot;busy:boolean;close:()=>void;start:(goal:string)=>void}){
const [goal,setGoal]=useState("");
return <div className="modal-backdrop" onClick={close}><form className="dialog teach-dialog" onClick={e=>e.stopPropagation()} onSubmit={e=>{e.preventDefault();if(goal.trim())start(goal.trim())}}>
@ -497,8 +693,12 @@ function SkillDraftCard({skill,busy,onSave,onTest,onDiscard,onEdit,onExport}:{sk
const lines=(text:string)=>text.split("\n").map(line=>line.replace(/^\s*(?:\d+[.、)]|[-•*])\s*/,"").trim()).filter(Boolean);
function stepsToText(steps:(PlaybookStep|string)[]|undefined){return (steps||[]).map(step=>{if(typeof step==="string")return step;return step.expect?`${step.do}${step.expect}`:step.do}).join("\n")}
function textToSteps(text:string,previous:(PlaybookStep|string)[]|undefined):PlaybookStep[]{const old=(previous||[]).filter((step):step is PlaybookStep=>typeof step==="object"&&step!==null);return lines(text).map(line=>{const [doPart,...rest]=line.split(/\s*(?:→|->|=>)\s*/);const expect=rest.join(" → ").trim();const match=old.find(step=>step.do===doPart.trim());return {do:doPart.trim(),expect:expect||(match&&!rest.length?match.expect:undefined)||"",note:match?.note||""}})}
function inputsToText(inputs:PlaybookInput[]|undefined){return (inputs||[]).map(input=>`${input.name}${input.description?`${input.description}`:""}${input.example?`|例:${input.example}`:""}`).join("\n")}
function textToInputs(text:string):PlaybookInput[]{return lines(text).map(line=>{const [head,...exampleParts]=line.split(/\s*[|]\s*/);const example=exampleParts.join("").replace(/^例[:]\s*/,"").trim();const [name,...descParts]=head.split(/[:]/);return {name:name.trim(),description:descParts.join("").trim()||undefined,example:example||undefined}}).filter(input=>input.name)}
function inputsToText(inputs:PlaybookInput[]|undefined){
const colon=getLocale()==="en"?": ":"";
const exampleMark=getLocale()==="en"?" | e.g. ":"|例:";
return (inputs||[]).map(input=>`${input.name}${input.description?`${colon}${input.description}`:""}${input.example?`${exampleMark}${input.example}`:""}`).join("\n");
}
function textToInputs(text:string):PlaybookInput[]{return lines(text).map(line=>{const [head,...exampleParts]=line.split(/\s*[|]\s*/);const example=exampleParts.join("").replace(/^(?:例[:]|e\.g\.?:?)\s*/i,"").trim();const [name,...descParts]=head.split(/[:]/);return {name:name.trim(),description:descParts.join(getLocale()==="en"?": ":"").trim()||undefined,example:example||undefined}}).filter(input=>input.name)}
function SkillEditDialog({skill,busy,close,save,test,remove,exportFile}:{skill:TaughtSkill;busy:boolean;close:()=>void;save:(name:string,playbook:Playbook)=>void;test:(name:string,playbook:Playbook)=>void;remove:()=>void;exportFile:(name:string,playbook:Playbook)=>void}){
const p=skill.playbook;
const [name,setName]=useState(skill.name||p.name||"");
@ -531,7 +731,7 @@ function SkillEditDialog({skill,busy,close,save,test,remove,exportFile}:{skill:T
function ClipboardDialog({close,paste}:{close:()=>void;paste:(text:string)=>void}){const[text,setText]=useState("");return <div className="modal-backdrop"><div className="dialog compact"><div className="dialog-title"><h2>{t("pasteToRemoteComputer")}</h2><button onClick={close}><X/></button></div><p>{t("pasteRemoteHelp")}</p><textarea className="clipboard-text" autoFocus value={text} onChange={e=>setText(e.target.value)} placeholder={t("pasteTextPlaceholder")}/><div className="dialog-actions"><button className="outline" onClick={close}>{t("cancel")}</button><button className="primary" disabled={!text} onClick={()=>paste(text)}>{t("pasteIntoVnc")}</button></div></div></div>}
function CreateDialog({close,created}:{close:()=>void;created:(bot:Bot)=>void}){const[name,setName]=useState("");const[mode,setMode]=useState<ComputerMode>("team");const[busy,setBusy]=useState(false);return <div className="modal-backdrop"><form className="dialog" onSubmit={async e=>{e.preventDefault();if(!name.trim())return;setBusy(true);try{created(await api<Bot>("/api/bots",{method:"POST",body:JSON.stringify({name:name.trim(),computerMode:mode})}))}finally{setBusy(false)}}}><div className="dialog-title"><h2>{t("addBot")}</h2><button type="button" onClick={close}><X/></button></div><label>{t("name")}<input autoFocus value={name} onChange={e=>setName(e.target.value)} placeholder={t("botNamePlaceholder")}/></label><div className="mode-grid"><button type="button" className={mode==="team"?"picked":""} onClick={()=>setMode("team")}><BotIcon/><strong>{t("sharedComputer")}</strong><small>{t("sharedComputerHint")}</small></button><button type="button" className={mode==="dedicated"?"picked":""} onClick={()=>setMode("dedicated")}><Computer/><strong>{t("privateComputer")}</strong><small>{t("privateComputerHint")}</small></button></div><div className="dialog-actions"><button type="button" className="outline" onClick={close}>{t("cancel")}</button><button className="primary" disabled={busy||!name.trim()}>{t("create")}</button></div></form></div>}
function CreateGroupDialog({bots,close,created}:{bots:Bot[];close:()=>void;created:(room:Room)=>void}){const[name,setName]=useState("");const[selected,setSelected]=useState<string[]>([]);const[busy,setBusy]=useState(false);const visible=bots.filter(bot=>!bot.hidden);return <div className="modal-backdrop"><form className="dialog" onSubmit={async e=>{e.preventDefault();const groupName=name.trim();if(!groupName||selected.length<2)return;setBusy(true);try{created(await api<Room>("/api/rooms",{method:"POST",body:JSON.stringify({name:groupName,memberIds:selected})}))}finally{setBusy(false)}}}><div className="dialog-title"><h2>{t("addGroup")}</h2><button type="button" onClick={close}><X/></button></div><p className="dialog-lead">{t("groupDescription")}</p><label>{t("groupName")}<input autoFocus value={name} maxLength={30} onChange={e=>setName(e.target.value)} placeholder={t("groupNamePlaceholder")}/></label><fieldset className="group-picker"><legend>{t("chooseBots")}</legend>{visible.map(bot=><label key={bot.id}><input type="checkbox" checked={selected.includes(bot.id)} onChange={()=>setSelected(ids=>ids.includes(bot.id)?ids.filter(id=>id!==bot.id):[...ids,bot.id])}/><Avatar lookId={bot.id} name={bot.name} color={bot.avatarColor} shape={bot.avatarShape} online/><span>{bot.name}</span></label>)}</fieldset><div className="dialog-actions"><button type="button" className="outline" onClick={close}>{t("cancel")}</button><button className="primary" disabled={busy||!name.trim()||selected.length<2}>{busy?t("creating"):t("createGroup")}</button></div></form></div>}
function RoomContextMenu({context,close,onDelete}:{context:{room:Room;x:number;y:number};close:()=>void;onDelete:()=>void}){return <div className="context-menu" style={{left:Math.min(context.x,window.innerWidth-220),top:Math.min(context.y,window.innerHeight-160)}} onClick={e=>e.stopPropagation()}><button className="danger-item" onClick={onDelete}><UseAnimations animation={trash2} size={15} strokeColor="#ff7777"/>{t("deleteGroup")}</button><button className="context-close" onClick={close}><X/></button></div>}
function RoomContextMenu({context,close,onDelete}:{context:{room:Room;x:number;y:number};close:()=>void;onDelete:()=>void}){return <div className="context-menu" style={{left:Math.min(context.x,window.innerWidth-220),top:Math.min(context.y,window.innerHeight-160)}} onClick={e=>e.stopPropagation()}><button className="danger-item" onClick={onDelete}><UseAnimations animation={trash2} size={15} strokeColor="var(--danger-soft)"/>{t("deleteGroup")}</button><button className="context-close" onClick={close}><X/></button></div>}
function ConfirmRoomDelete({room,close,confirm}:{room:Room;close:()=>void;confirm:()=>void}){return <div className="modal-backdrop"><div className="dialog compact"><h2>{t("deleteNamed",{name:room.name})}</h2><p>{t("deleteGroupDescription")}</p><div className="dialog-actions"><button className="outline" onClick={close}>{t("cancel")}</button><button className="danger" onClick={confirm}>{t("deleteGroup")}</button></div></div></div>}
function ConfirmDelete({bot,close,confirm}:{bot:Bot;close:()=>void;confirm:()=>void}){return <div className="modal-backdrop"><div className="dialog compact"><h2>{t("deleteNamed",{name:bot.name})}</h2><p>{bot.computerMode==="dedicated"?t("deleteDedicatedBotDescription"):t("deleteSharedBotDescription")}</p><div className="dialog-actions"><button className="outline" onClick={close}>{t("cancel")}</button><button className="danger" onClick={confirm}>{t("delete")}</button></div></div></div>}
@ -553,6 +753,13 @@ function WorkspaceSettingsDialog({name,setName,showHidden,setShowHidden,rightCol
<label>{t("workspaceName")}<input autoFocus value={draft} maxLength={40} onChange={e=>setDraft(e.target.value)}/></label>
<label className="memory-toggle"><input type="checkbox" checked={showHidden} onChange={e=>setShowHidden(e.target.checked)}/> {t("showHiddenBots")}</label>
<label className="memory-toggle"><input type="checkbox" checked={rightCollapsed} onChange={e=>setRightCollapsed(e.target.checked)}/> {t("collapseRightSidebar")}</label>
<label>{t("language")}
<select value={getLocale()} onChange={e=>setLocale(e.target.value==="en"?"en":"zh-TW")}>
<option value="zh-TW">{t("languageZh")}</option>
<option value="en">{t("languageEn")}</option>
</select>
</label>
<p className="dialog-lead">{t("languageHint")}</p>
<p className="dialog-lead">{t("workspaceSettingsHint")}</p>
<div className="dialog-actions"><button type="button" className="outline" onClick={close}>{t("cancel")}</button><button className="primary">{t("save")}</button></div>
</form></div>
@ -568,7 +775,7 @@ function ModelSettingsDialog({close}:{close:()=>void}){
const[busy,setBusy]=useState(false);
const[error,setError]=useState("");
const current=settings?.providers.find(item=>item.id===provider);
useEffect(()=>{api<WorkspaceSettings>("/api/workspace/settings").then(value=>{setSettings(value);setProvider(value.provider);setModelId(value.modelId);setBaseUrl(value.baseUrl);setModels(value.models)}).catch(e=>setError(e instanceof Error?e.message:t("loadFailed")))},[]);
useEffect(()=>{api<WorkspaceSettings>("/api/workspace/settings").then(value=>{setSettings(value);setProvider(value.provider);setModelId(value.modelId);setBaseUrl(value.baseUrl);setModels(value.models)}).catch(e=>setError(e instanceof Error?localizeError(e.message):t("loadFailed")))},[]);
async function loadModels(nextProvider:ModelProviderId,nextBaseUrl:string){
if(nextProvider==="openai-compatible"&&!nextBaseUrl.trim()){setModels([]);return}
try{
@ -592,7 +799,7 @@ function ModelSettingsDialog({close}:{close:()=>void}){
if(info?.defaultModel)setModelId(info.defaultModel);
void loadModels(id,info?.defaultBaseUrl||"");
}
return <div className="modal-backdrop"><form className="dialog settings-dialog" onSubmit={async e=>{e.preventDefault();setBusy(true);setError("");try{await api("/api/workspace/settings",{method:"PATCH",body:JSON.stringify({provider,modelId:modelId.trim(),baseUrl:baseUrl.trim()||null,apiKey:clearKey?"":apiKey.trim()||null,clearApiKey:clearKey})});close()}catch(e){setError(e instanceof Error?e.message:t("settingsFailed"))}finally{setBusy(false)}}}>
return <div className="modal-backdrop"><form className="dialog settings-dialog" onSubmit={async e=>{e.preventDefault();setBusy(true);setError("");try{await api("/api/workspace/settings",{method:"PATCH",body:JSON.stringify({provider,modelId:modelId.trim(),baseUrl:baseUrl.trim()||null,apiKey:clearKey?"":apiKey.trim()||null,clearApiKey:clearKey})});close()}catch(e){setError(e instanceof Error?localizeError(e.message):t("settingsFailed"))}finally{setBusy(false)}}}>
<div className="dialog-title"><h2>{t("modelSettings")}</h2><button type="button" onClick={close}><X/></button></div>
<p className="dialog-lead">{t("modelSettingsHint")}</p>
<fieldset className="provider-fieldset"><legend>{t("modelProvider")}</legend>
@ -619,7 +826,7 @@ function AboutDialog({close}:{close:()=>void}){
const[health,setHealth]=useState<"ok"|"bad"|"…">("…");
useEffect(()=>{fetch("/api/health").then(r=>r.ok?setHealth("ok"):setHealth("bad")).catch(()=>setHealth("bad"))},[]);
return <div className="modal-backdrop"><div className="dialog compact"><div className="dialog-title"><h2>{t("about")}</h2><button type="button" onClick={close}><X/></button></div>
<div className="about-brand"><Avatar name="L" size={52} online/><strong>LazyBoy</strong><small>0.1.0</small></div>
<div className="about-brand"><img src="/lazyboy-icon.png?v=white-1" width="52" height="52" alt="" style={{borderRadius:14}}/><strong>LazyBoy</strong><small>0.1.0</small></div>
<p>{t("aboutDescription")}</p>
<p className="dialog-lead">{t("apiStatus",{status:health==="ok"?t("statusNormal"):health==="bad"?t("statusUnavailable"):t("statusChecking")})}</p>
<div className="dialog-actions"><button className="primary" onClick={close}>{t("close")}</button></div>
@ -635,6 +842,8 @@ function HelpDialog({close}:{close:()=>void}){
<section><h3>{t("helpMcpTitle")}</h3><p>{t("helpMcp")}</p></section>
<section><h3>{t("helpSkillsTitle")}</h3><p>{t("helpSkills")}</p></section>
<section><h3>{t("helpAttachTitle")}</h3><p>{t("helpAttach")}</p></section>
<section><h3>{t("helpVoiceTitle")}</h3><p>{t("helpVoice")}</p></section>
<section><h3>{t("helpLanguageTitle")}</h3><p>{t("helpLanguage")}</p></section>
<section><h3>{t("helpShortcutsTitle")}</h3><p>{t("helpShortcuts")}</p></section>
</div>
<div className="dialog-actions"><button className="primary" onClick={close}>{t("close")}</button></div>
@ -649,4 +858,15 @@ function FeedbackDialog({close}:{close:()=>void}){
</div></div>
}
function LoginScreen({authenticated}:{authenticated:()=>void}){const[token,setToken]=useState("");const[busy,setBusy]=useState(false);const[error,setError]=useState("");return <main className="login-screen"><form className="dialog compact login-dialog" onSubmit={async e=>{e.preventDefault();if(!token)return;setBusy(true);setError("");try{await api("/api/session",{method:"POST",body:JSON.stringify({token})});authenticated()}catch(err){setError(err instanceof Error?err.message:t("loginFailed"))}finally{setBusy(false)}}}><Avatar name="L" size={58}/><h1>{t("loginTitle")}</h1><p>{t("loginDescription")}</p><label>{t("accessToken")}<input type="password" autoFocus autoComplete="current-password" value={token} onChange={e=>setToken(e.target.value)}/></label>{error&&<div className="login-error">{error}</div>}<button className="primary" disabled={busy||!token}>{busy?t("verifying"):t("login")}</button></form></main>}
function LoginScreen({authenticated}:{authenticated:()=>void}){
useLocale();
const[token,setToken]=useState("");const[busy,setBusy]=useState(false);const[error,setError]=useState("");
return <main className="login-screen">
<div className="login-lang" role="group" aria-label={t("language")}>
<button type="button" aria-pressed={getLocale()==="zh-TW"} className={getLocale()==="zh-TW"?"picked":""} onClick={()=>setLocale("zh-TW")}>{t("languageZh")}</button>
<button type="button" aria-pressed={getLocale()==="en"} className={getLocale()==="en"?"picked":""} onClick={()=>setLocale("en")}>{t("languageEn")}</button>
</div>
<form className="dialog compact login-dialog" onSubmit={async e=>{e.preventDefault();if(!token)return;setBusy(true);setError("");try{await api("/api/session",{method:"POST",body:JSON.stringify({token})});authenticated()}catch(err){setError(err instanceof Error?localizeError(err.message):t("loginFailed"))}finally{setBusy(false)}}}>
<Avatar name="L" size={58}/><h1>{t("loginTitle")}</h1><p>{t("loginDescription")}</p><label>{t("accessToken")}<input type="password" autoFocus autoComplete="current-password" value={token} onChange={e=>setToken(e.target.value)}/></label>{error&&<div className="login-error">{error}</div>}<button className="primary" disabled={busy||!token}>{busy?t("verifying"):t("login")}</button>
</form></main>;
}

View File

@ -1,4 +1,4 @@
import UseAnimations from "react-useanimations";
import UseAnimations from "./use-animations";
import type { Animation } from "react-useanimations/utils";
import activity from "react-useanimations/lib/activity";
import archive from "react-useanimations/lib/archive";

31
apps/web/src/avatar.css Normal file
View File

@ -0,0 +1,31 @@
.avatar{flex:0 0 auto;width:36px;height:36px;display:grid;place-items:center;border-radius:50%;background:#26262a;color:#c9c9ce;font-weight:650}
.avatar.online{background:var(--accent);color:#083f34;box-shadow:inset 0 0 0 8px rgba(0,0,0,.16)}
.topbar .avatar{width:32px;height:32px}
.avatar.blobatar,.avatar.blobatar.online{position:relative;display:inline-grid;place-items:center;flex:0 0 var(--avatar-size);width:var(--avatar-size);height:var(--avatar-size);overflow:visible;border:0;border-radius:0;background:transparent;box-shadow:none;color:unset}
.avatar.blobatar>svg,.avatar.blobatar>img{display:block;width:100%;height:100%}
.avatar.blobatar.thinking:before,.avatar.blobatar.thinking:after{content:"";position:absolute;z-index:4;inset:-6px;border-radius:50%;background:conic-gradient(from 0deg,transparent 0 8%,#ff4fd8 11%,#7c5cff 18%,transparent 25% 43%,#34d9ff 48%,#58f39a 55%,transparent 62% 78%,#ffe66d 82%,#ff7a59 89%,transparent 96%);-webkit-mask:radial-gradient(farthest-side,transparent calc(100% - 2px),#000 calc(100% - 1.5px));mask:radial-gradient(farthest-side,transparent calc(100% - 2px),#000 calc(100% - 1.5px));pointer-events:none;animation:magic-orbit 1.35s linear infinite}
.avatar.blobatar.thinking:after{inset:-8px;opacity:.45;filter:blur(3px);animation-duration:2.1s;animation-direction:reverse}
.avatar-wrap{position:relative;display:grid;flex:0 0 auto}
.avatar-editor{display:grid;justify-items:center;gap:5px;padding:18px 0 2px}
.avatar-editor strong{margin-top:7px}
.avatar-editor .avatar.blobatar{margin-bottom:2px}
.avatar .presence{position:absolute;z-index:6;right:-1px;bottom:-1px;width:clamp(8px,calc(var(--avatar-size)*0.25),11px);height:clamp(8px,calc(var(--avatar-size)*0.25),11px);min-width:8px;min-height:8px;border:2px solid var(--side);border-radius:50%;background:var(--online);box-shadow:0 0 0 1px rgba(0,0,0,.4);pointer-events:none}
.bot-row .avatar-wrap::after{content:none;position:absolute;z-index:9;right:-2px;bottom:-2px;width:9px;height:9px;border:2px solid var(--side);border-radius:50%;background:var(--online);box-shadow:0 0 0 1px rgba(0,0,0,.38);pointer-events:none}
.avatar.blobatar.thinking{animation:avatar-rainbow-glow 1.8s linear infinite}
.avatar-wrap{transition:transform .16s ease}
.avatar-stack{position:relative;display:block;flex:0 0 auto}
.avatar-stack .stack-item{position:absolute;top:0}
.avatar-stack .stack-item .avatar{box-shadow:none;filter:drop-shadow(2px 0 0 var(--side)) drop-shadow(-2px 0 0 var(--side)) drop-shadow(0 2px 0 var(--side)) drop-shadow(0 -2px 0 var(--side))}
.avatar-stack .stack-item .avatar.thinking{filter:none;animation:avatar-rainbow-glow 1.8s linear infinite}
.stack-extra{position:absolute;top:0;display:grid;place-items:center;border-radius:50%;background:#2a2a2e;color:var(--muted);font-size:11px;font-weight:650;box-shadow:0 0 0 2px var(--side)}
.topbar .avatar-stack{margin-right:2px}
.avatar-stack{isolation:isolate}
.avatar-stack .stack-item{transition:transform .16s ease}
.avatar-stack .stack-item:hover{transform:translateY(-2px);z-index:8!important}
.avatar-stack .stack-item .avatar{filter:drop-shadow(2px 0 0 var(--main)) drop-shadow(-2px 0 0 var(--main)) drop-shadow(0 2px 0 var(--main)) drop-shadow(0 -2px 0 var(--main))}
.avatar-stack .stack-extra{box-shadow:0 0 0 2px var(--main)}
.avatar-stack .stack-item .avatar.thinking{filter:none}
/* Keep avatars independent of message and toolbar dimensions. */
.avatar.blobatar{min-width:var(--avatar-size);max-width:var(--avatar-size);min-height:var(--avatar-size);padding:0;line-height:0;vertical-align:middle}
.avatar.blobatar>svg{max-width:none}
.avatar-stack .stack-item{line-height:0}

212
apps/web/src/call-audio.ts Normal file
View File

@ -0,0 +1,212 @@
export const VOICE_SAMPLE_RATE = 24000;
export function floatToPcm16(input: Float32Array): Uint8Array {
const pcm = new Int16Array(input.length);
for (let i = 0; i < input.length; i += 1) {
const sample = Math.max(-1, Math.min(1, input[i] ?? 0));
pcm[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
}
return new Uint8Array(pcm.buffer);
}
export function pcm16ToFloat(bytes: ArrayBuffer): Float32Array {
const pcm = new Int16Array(bytes.byteLength % 2 === 0 ? bytes : bytes.slice(0, bytes.byteLength - 1));
const out = new Float32Array(pcm.length);
for (let i = 0; i < pcm.length; i += 1) {
out[i] = (pcm[i] ?? 0) / 32768;
}
return out;
}
export function resample(input: Float32Array, fromRate: number, toRate: number): Float32Array {
if (fromRate === toRate || input.length === 0) return input;
const ratio = fromRate / toRate;
const length = Math.max(1, Math.round(input.length / ratio));
const out = new Float32Array(length);
for (let i = 0; i < length; i += 1) {
const src = i * ratio;
const left = Math.floor(src);
const frac = src - left;
const a = input[left] ?? 0;
const b = input[Math.min(left + 1, input.length - 1)] ?? 0;
out[i] = a + (b - a) * frac;
}
return out;
}
const WORKLET = `
class CaptureProcessor extends AudioWorkletProcessor {
process(inputs) {
const channel = inputs[0] && inputs[0][0];
if (channel && channel.length) this.port.postMessage(channel);
return true;
}
}
registerProcessor("lazyboy-capture", CaptureProcessor);
`;
// One frame per 128-sample worklet callback would be ~375 websocket messages a
// second; batching to 40 ms keeps latency low without flooding the provider.
export const CAPTURE_CHUNK_SAMPLES = VOICE_SAMPLE_RATE / 25;
// Cushion so a late chunk does not have to start in the past and click.
const PLAYBACK_LEAD_SECONDS = 0.08;
export type CallAudioHandlers = {
onCapture: (pcm: Uint8Array) => void;
onError: (message: string) => void;
onPlaybackChange?: (playing: boolean) => void;
};
export class CallAudio {
private stopped = false;
private context: AudioContext | null = null;
private stream: MediaStream | null = null;
private worklet: AudioWorkletNode | null = null;
private monitor: GainNode | null = null;
private nextTime = 0;
private playing: AudioBufferSourceNode[] = [];
private playbackActive = false;
private pending: Float32Array[] = [];
private pendingLength = 0;
private carry: Uint8Array | null = null;
private handlers: CallAudioHandlers;
constructor(handlers: CallAudioHandlers) {
this.handlers = handlers;
}
private queueCapture(samples: Float32Array): void {
if (samples.length) {
this.pending.push(samples);
this.pendingLength += samples.length;
}
while (this.pendingLength >= CAPTURE_CHUNK_SAMPLES) {
const chunk = new Float32Array(CAPTURE_CHUNK_SAMPLES);
let filled = 0;
while (filled < CAPTURE_CHUNK_SAMPLES) {
const head = this.pending[0] as Float32Array;
const take = Math.min(head.length, CAPTURE_CHUNK_SAMPLES - filled);
chunk.set(head.subarray(0, take), filled);
filled += take;
if (take === head.length) this.pending.shift();
else this.pending[0] = head.subarray(take);
}
this.pendingLength -= CAPTURE_CHUNK_SAMPLES;
this.handlers.onCapture(floatToPcm16(chunk));
}
}
private notifyPlayback(): void {
const active = this.playing.length > 0;
if (active === this.playbackActive) return;
this.playbackActive = active;
this.handlers.onPlaybackChange?.(active);
}
async start(): Promise<void> {
const stream = await navigator.mediaDevices.getUserMedia({
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true },
});
if (this.stopped) {
stream.getTracks().forEach((track) => track.stop());
return;
}
this.stream = stream;
const context = new AudioContext();
this.context = context;
if (context.state === "suspended") await context.resume();
const blob = new Blob([WORKLET], { type: "application/javascript" });
const url = URL.createObjectURL(blob);
try {
await context.audioWorklet.addModule(url);
} finally {
URL.revokeObjectURL(url);
}
if (this.stopped) return;
const source = context.createMediaStreamSource(stream);
const worklet = new AudioWorkletNode(context, "lazyboy-capture");
worklet.port.onmessage = (event) => {
const samples = event.data as Float32Array;
this.queueCapture(resample(samples, context.sampleRate, VOICE_SAMPLE_RATE));
};
worklet.onprocessorerror = () => this.handlers.onError("capture stopped");
// Muted sink: the processor emits no output, and a zero gain guarantees the
// microphone can never leak back into the speakers.
const monitor = context.createGain();
monitor.gain.value = 0;
source.connect(worklet);
worklet.connect(monitor);
monitor.connect(context.destination);
this.worklet = worklet;
this.monitor = monitor;
this.nextTime = context.currentTime;
}
play(pcm: ArrayBuffer): void {
const context = this.context;
if (!context) return;
// PCM16 frames can be split mid-sample across websocket messages; dropping
// the odd tail byte would shift every later sample and turn speech to noise.
let bytes = new Uint8Array(pcm);
if (this.carry) {
const joined = new Uint8Array(this.carry.length + bytes.length);
joined.set(this.carry, 0);
joined.set(bytes, this.carry.length);
bytes = joined;
this.carry = null;
}
if (bytes.length % 2 === 1) {
this.carry = bytes.slice(bytes.length - 1);
bytes = bytes.subarray(0, bytes.length - 1);
}
if (!bytes.length) return;
const aligned = bytes.slice().buffer;
const samples = resample(pcm16ToFloat(aligned), VOICE_SAMPLE_RATE, context.sampleRate);
if (!samples.length) return;
try {
const buffer = context.createBuffer(1, samples.length, context.sampleRate);
buffer.getChannelData(0).set(samples);
const node = context.createBufferSource();
node.buffer = buffer;
node.connect(context.destination);
if (this.nextTime < context.currentTime) this.nextTime = context.currentTime + PLAYBACK_LEAD_SECONDS;
const startAt = this.nextTime;
node.start(startAt);
this.nextTime = startAt + buffer.duration;
this.playing.push(node);
node.onended = () => {
this.playing = this.playing.filter((item) => item !== node);
this.notifyPlayback();
};
this.notifyPlayback();
} catch {
this.handlers.onError("playback failed");
}
}
interrupt(): void {
for (const node of this.playing) {
node.onended = null;
try { node.stop(); } catch { /* already stopped */ }
}
this.playing = [];
this.carry = null;
if (this.context) this.nextTime = this.context.currentTime;
this.notifyPlayback();
}
stop(): void {
this.stopped = true;
this.interrupt();
this.pending = [];
this.pendingLength = 0;
this.worklet?.disconnect();
this.worklet = null;
this.monitor?.disconnect();
this.monitor = null;
this.stream?.getTracks().forEach((track) => track.stop());
this.stream = null;
void this.context?.close();
this.context = null;
}
}

21
apps/web/src/call.css Normal file
View File

@ -0,0 +1,21 @@
.call-overlay{position:absolute;inset:0;z-index:40;display:grid;place-items:center;background:rgba(5,5,6,.82);padding:24px}
.call-card{width:min(420px,100%);display:flex;flex-direction:column;align-items:center;gap:10px;padding:28px 24px 22px;border:1px solid var(--border);border-radius:28px;background:var(--elevated);box-shadow:0 30px 80px rgba(0,0,0,.55);text-align:center}
.call-kicker{font-size:12px;letter-spacing:.14em;text-transform:uppercase;color:var(--faint)}
.call-name{margin-top:4px;font-size:22px}
.call-phase{min-height:1.4em;color:var(--accent);font-size:14px}
.call-phase.working{color:#ffe08a}
.call-caption{min-height:3.2em;margin:4px 0 0;color:var(--muted);font-size:14.5px;line-height:1.5}
.call-caption.assistant{color:var(--ink)}
.call-error{margin:0;color:var(--danger);font-size:13px}
.call-time{color:var(--faint);font-variant-numeric:tabular-nums;font-size:13px}
.call-actions{display:flex;gap:10px;margin-top:8px}
.call-interrupt,.call-hangup{min-width:108px;height:40px;border:0;border-radius:999px;cursor:pointer}
.call-interrupt{background:var(--surface);color:var(--ink)}
.call-hangup{background:var(--danger);color:#fff;font-weight:600}
.call-takeover{margin-top:8px}
.call-hint{margin:8px 0 0;color:var(--faint);font-size:12px}
.top-tool-button.call-active{background:rgba(62,197,168,.16);color:var(--accent)}
.call-entry{display:inline-flex;border-radius:9px}
.call-entry:focus-visible{outline:2px solid var(--accent);outline-offset:2px}
.call-entry button:disabled{pointer-events:none}

184
apps/web/src/call.tsx Normal file
View File

@ -0,0 +1,184 @@
import { useEffect, useRef, useState } from "react";
import { Avatar } from "./avatar";
import { CallAudio } from "./call-audio";
import { t } from "./i18n";
import type { AvatarShape, Bot } from "./types";
export type CallPhase = "connecting" | "listening" | "speaking" | "working";
type ComputerEvent = { status?: string; step?: string | null; takeover?: boolean };
type ServerEvent =
| { type: "ready"; callId?: string; voice?: string }
| { type: "transcript"; role: "user" | "assistant"; text: string; final?: boolean }
| { type: "speech"; state: "started" | "stopped" }
| { type: "computer"; status?: string; step?: string | null; takeover?: boolean }
| { type: "error"; message: string };
function wsUrl(sessionId: string): string {
const protocol = location.protocol === "https:" ? "wss:" : "ws:";
return `${protocol}//${location.host}/api/sessions/${sessionId}/call`;
}
function phaseLabel(phase: CallPhase, computer: ComputerEvent | null): string {
if (computer?.takeover) return t("takeOverToContinue");
if (phase === "connecting") return t("callConnecting");
if (phase === "speaking") return t("speaking");
if (phase === "working") return t("workingOnComputer");
return t("listening");
}
function computerIsBusy(computer: ComputerEvent | null): boolean {
return computer?.status === "running" || computer?.status === "queued" || Boolean(computer?.takeover);
}
export function PhoneIcon({ size = 16 }: { size?: number }) {
return (
<svg width={size} height={size} viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.8" aria-hidden="true">
<path d="M6.5 3.8c.5-1 1.6-1.4 2.6-1l2 1c.8.4 1.2 1.3 1 2.2l-.6 2.2a2 2 0 0 0 .6 1.9l3.8 3.8c.5.5 1.3.7 1.9.6l2.2-.6c.9-.2 1.8.2 2.2 1l1 2c.4 1 .1 2.1-.9 2.6-2.3 1.2-7.2 1-11.6-3.4S4.1 7.6 5.3 5.3Z" />
</svg>
);
}
export function CallOverlay({
bot,
sessionId,
takeover,
onHangUp,
onTakeOver,
}: {
bot: Bot;
sessionId: string;
takeover: boolean;
onHangUp: () => void;
onTakeOver: () => void;
}) {
const [connected, setConnected] = useState(false);
const [speaking, setSpeaking] = useState(false);
const [caption, setCaption] = useState<{ role: "user" | "assistant"; text: string } | null>(null);
const [error, setError] = useState("");
const [elapsed, setElapsed] = useState(0);
const [computer, setComputer] = useState<ComputerEvent | null>(null);
const audioRef = useRef<CallAudio | null>(null);
const socketRef = useRef<WebSocket | null>(null);
const hanging = useRef(false);
function hangUp() {
hanging.current = true;
audioRef.current?.stop();
audioRef.current = null;
socketRef.current?.close();
socketRef.current = null;
onHangUp();
}
function interrupt() {
audioRef.current?.interrupt();
// Stopping local playback alone lets the model keep streaming the rest of
// its answer, so the provider has to be told to abandon the response too.
if (socketRef.current?.readyState === WebSocket.OPEN) {
socketRef.current.send(JSON.stringify({ type: "interrupt" }));
}
}
useEffect(() => {
hanging.current = false;
let disposed = false;
const started = Date.now();
const tick = window.setInterval(() => setElapsed(Math.floor((Date.now() - started) / 1000)), 1000);
const audio = new CallAudio({
onCapture: (pcm) => {
if (socketRef.current?.readyState === WebSocket.OPEN) socketRef.current.send(pcm);
},
onError: () => setError(t("callAudioFailed")),
onPlaybackChange: setSpeaking,
});
audioRef.current = audio;
const socket = new WebSocket(wsUrl(sessionId));
socket.binaryType = "arraybuffer";
socketRef.current = socket;
socket.onopen = () => {
void audio.start().catch((err) => {
if (disposed) return;
audio.stop();
socket.close();
setError(err instanceof Error && /NotAllowedError|PermissionDenied/.test(err.name + err.message) ? t("micDenied") : t("micFailed"));
});
setConnected(true);
};
socket.onmessage = (event) => {
if (typeof event.data !== "string") {
audio.play(event.data as ArrayBuffer);
return;
}
let payload: ServerEvent;
try { payload = JSON.parse(event.data) as ServerEvent; } catch { return; }
if (payload.type === "speech") {
if (payload.state === "started") audio.interrupt();
return;
}
if (payload.type === "transcript") {
setCaption({ role: payload.role, text: payload.text });
return;
}
if (payload.type === "computer") {
setComputer(payload);
return;
}
if (payload.type === "error") setError(payload.message);
};
socket.onerror = () => { if (!disposed && !hanging.current) setError(t("callFailed")); };
socket.onclose = () => {
audio.stop();
if (!disposed && !hanging.current) setError((previous) => previous || t("callFailed"));
};
function onKey(event: KeyboardEvent) {
if (event.key === "Escape") { event.preventDefault(); hangUp(); }
if (event.key === " " && !event.repeat && event.target === document.body) {
event.preventDefault();
interrupt();
}
}
window.addEventListener("keydown", onKey);
return () => {
disposed = true;
hanging.current = true;
window.clearInterval(tick);
window.removeEventListener("keydown", onKey);
audio.stop();
socket.close();
};
}, [sessionId, bot.id]);
const minutes = Math.floor(elapsed / 60);
const seconds = String(elapsed % 60).padStart(2, "0");
const showTakeover = takeover || Boolean(computer?.takeover);
const phase: CallPhase = !connected
? "connecting"
: speaking
? "speaking"
: computerIsBusy(computer)
? "working"
: "listening";
return (
<div className="call-overlay" role="dialog" aria-label={t("call")}>
<div className="call-card" data-testid="call-view">
<div className="call-kicker">{t("call")}</div>
<Avatar lookId={bot.id} name={bot.name} color={bot.avatarColor} shape={bot.avatarShape as AvatarShape} active online size={72} />
<strong className="call-name">{bot.name}</strong>
<div className={`call-phase ${phase}`}>{phaseLabel(phase, computer)}</div>
<p className={`call-caption ${caption ? caption.role : ""}`}>{caption?.text || t("callPrompt")}</p>
{error ? <p className="call-error">{error}</p> : null}
<div className="call-time">{minutes}:{seconds}</div>
<div className="call-actions">
<button type="button" className="call-interrupt" onClick={interrupt}>{t("interrupt")}</button>
<button type="button" className="call-hangup" onClick={hangUp}>{t("hangUp")}</button>
</div>
{showTakeover ? (
<button type="button" className="primary call-takeover" onClick={onTakeOver}>{t("takeOverNow")}</button>
) : null}
<p className="call-hint">{t("callShortcuts")}</p>
</div>
</div>
);
}

75
apps/web/src/chat.css Normal file
View File

@ -0,0 +1,75 @@
.messages{flex:1;overflow:auto;padding:34px max(34px,7vw) 150px;display:flex;flex-direction:column;gap:18px}
.message{display:flex}
.message>.message-body,.message-stack>.message-body{max-width:76%;padding:13px 17px;border-radius:22px;white-space:pre-wrap;line-height:1.5}
.message>.message-body.md,.message-stack>.message-body.md{min-width:0;white-space:normal}
.message.user{justify-content:flex-end}
.message.user>.message-body{background:var(--cream);color:var(--on-cream)}
.message.assistant>.message-body,.message-stack>.message-body{background:#19191c}
.message-stack{display:flex;flex-direction:column;align-items:flex-start;max-width:76%;min-width:0}
.message-stack>.message-body{max-width:100%}
.copy-msg{display:inline-flex;align-items:center;gap:6px;height:28px;margin-top:6px;padding:0 8px 0 6px;border:0;border-radius:8px;background:transparent;color:var(--muted);font:inherit;font-size:12px;cursor:pointer}
.copy-msg svg{flex:0 0 14px}
.copy-msg:hover{background:rgba(255,255,255,.07);color:var(--ink)}
.copy-msg.copied,.copy-msg.copied:hover{color:var(--accent);background:transparent;cursor:default}
.composer{position:absolute;left:max(34px,5vw);right:max(34px,5vw);bottom:22px;display:flex;align-items:flex-end;gap:8px;border:1px solid var(--border);background:var(--elevated);border-radius:26px;padding:8px;box-shadow:0 16px 50px rgba(0,0,0,.22)}
.composer textarea{flex:1;min-height:42px;max-height:140px;resize:none;border:0;outline:0;background:transparent;color:var(--ink);padding:11px 4px}
.composer-plus{background:transparent;color:var(--muted)}
.composer svg{width:18px}
.messages{min-width:0;padding-bottom:128px;padding-left:var(--chat-gutter);padding-right:var(--chat-gutter)}
.message{box-sizing:border-box;width:var(--chat-col);max-width:none;margin-inline:auto;overflow-wrap:anywhere}
.message>span{max-width:82%}
.thinking-row{display:flex;align-items:center;gap:10px;width:min(760px,100%);margin:0}
.thinking-dots{position:relative;display:flex;align-items:center;gap:4px;height:32px;padding:0 12px;border:1px solid var(--border);border-radius:14px;background:var(--surface);overflow:hidden}
.thinking-dots:after{content:"";position:absolute;left:12px;bottom:4px;width:24px;height:2px;border-radius:999px;background:linear-gradient(90deg,#7c5cff,#34d9ff,#58f39a,#ffe66d,#ff73dc,#7c5cff);background-size:200% 100%;animation:small-magic-line 1.25s linear infinite}
.thinking-dots i{width:5px;height:5px;border-radius:50%;background:var(--muted);animation:thinking-dot 1.15s ease-in-out infinite}
.thinking-dots i:nth-child(2){animation-delay:.16s}
.thinking-dots i:nth-child(3){animation-delay:.32s}
.composer-dock{position:absolute;left:0;right:0;bottom:0;z-index:6;display:flex;flex-direction:column;align-items:center;gap:20px;padding:28px var(--chat-gutter) 20px;background:var(--main);pointer-events:none}
.composer-dock:before{content:"";position:absolute;left:0;right:0;bottom:100%;height:32px;background:linear-gradient(180deg,transparent,var(--main));pointer-events:none}
.composer-dock>*{box-sizing:border-box;pointer-events:auto;width:var(--chat-col);max-width:760px;margin-inline:auto}
.composer{position:relative;left:auto;right:auto;bottom:auto;width:var(--chat-col);min-height:62px;align-items:center;padding:9px 10px;transform:none}
.composer.has-files{flex-wrap:wrap;align-items:flex-end;padding-top:10px}
.composer textarea{align-self:center;box-sizing:border-box;height:42px;min-height:42px;max-height:126px;padding:11px 4px;line-height:20px}
.slash-suggestions{position:absolute;left:58px;right:58px;bottom:calc(100% + 8px);z-index:8;display:grid;gap:2px;padding:6px;border:1px solid var(--border);border-radius:14px;background:var(--menu);box-shadow:0 14px 34px rgba(0,0,0,.35)}
.slash-suggestions button{display:flex;align-items:baseline;gap:10px;width:100%;padding:8px 10px;border:0;border-radius:9px;background:transparent;color:var(--ink);text-align:left;cursor:pointer}
.slash-suggestions button:hover,.slash-suggestions button:focus-visible{background:rgba(255,255,255,.08);outline:0}
.slash-suggestions strong{min-width:110px;color:var(--accent);font-weight:650}
.slash-suggestions span{overflow:hidden;color:var(--muted);font-size:12px;text-overflow:ellipsis;white-space:nowrap}
.composer-files{display:flex;flex-wrap:wrap;gap:8px;flex:1 0 100%;order:-1;padding:2px 8px 10px 46px}
.file-card{display:flex;align-items:center;gap:8px;max-width:min(240px,100%);height:48px;padding:2px 8px 2px 2px;border:1px solid var(--border);border-radius:14px;background:var(--surface);color:var(--ink)}
.file-card-thumb,.file-card-badge{flex:0 0 44px;width:44px;height:44px;border-radius:11px;object-fit:cover}
.file-card-badge{display:grid;place-items:center;background:var(--hairline);color:var(--muted);font-size:10px;font-weight:700;letter-spacing:.04em}
.file-card-meta{display:grid;min-width:0;flex:1}
.file-card-meta strong{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-size:13px;font-weight:600}
.file-card-meta small{color:var(--muted);font-size:11px}
.file-card-remove{flex:0 0 28px;width:28px;height:28px;display:grid;place-items:center;border:0;border-radius:8px;background:transparent;color:var(--muted);cursor:pointer}
.file-card-remove:hover{color:var(--ink);background:rgba(255,255,255,.08)}
.file-card-remove svg{width:14px;height:14px}
.message.with-files{align-items:flex-end;flex-direction:column;gap:8px}
.message.with-files .msg-attachments{display:flex;flex-wrap:wrap;justify-content:flex-end;gap:8px;max-width:min(420px,82%)}
.message.with-files .message-body,.message.with-files .message-stack{max-width:82%}
.composer-plus,.composer .send{box-sizing:border-box;flex:0 0 42px;width:42px;height:42px;margin:0;align-self:center}
.stop-send{background:var(--cream);color:var(--on-send)}
.stop-send svg{width:14px;height:14px;fill:currentColor}
.composer-plus:disabled{opacity:.35;cursor:not-allowed}
.message{position:relative;padding-right:28px}
.messages .remember-msg{display:none}
.working-label{font-size:13px;letter-spacing:.02em;line-height:1.35;white-space:nowrap;background:linear-gradient(90deg,#7a8088 0%,#7a8088 28%,#fff 50%,#7a8088 72%,#7a8088 100%);background-size:220% 100%;-webkit-background-clip:text;background-clip:text;color:transparent;animation:working-shimmer 1.35s linear infinite}
.message.assistant.spoken{display:grid;grid-template-columns:28px minmax(0,1fr);column-gap:10px;row-gap:3px;justify-content:start;align-items:end}
.message.assistant.spoken .msg-avatar{grid-column:1;grid-row:2;align-self:start;max-width:none;margin:4px 0 0;padding:0;border-radius:0;background:transparent;line-height:normal}
.message.assistant .msg-avatar>.avatar.blobatar{max-width:none;padding:0;background:transparent;color:inherit;line-height:normal;white-space:normal}
.message.assistant.spoken .speaker{grid-column:2;font-size:12px;font-weight:650;line-height:1.2;padding:0 6px;color:var(--accent)}
.message.assistant.spoken>span{grid-column:2;max-width:82%;border-radius:6px 18px 18px 18px}
.thinking-row{width:100%;margin:0;padding-left:2px}
.working-copy{display:flex;flex-direction:column;gap:8px;min-width:0}
.working-step{color:var(--muted);font-size:12px;letter-spacing:.02em;line-height:1.35}
/* Speaker rows: the coloured speaker label must breathe away from the text. */
.message.assistant.spoken{column-gap:11px;row-gap:6px}
.message.assistant.spoken>.msg-avatar{max-width:none;padding:0;border-radius:0;background:transparent}
.message.assistant.spoken .speaker{padding:0 2px;letter-spacing:.01em}
.message.assistant.spoken>.message-body,.message.assistant.spoken>.message-stack{grid-column:2;max-width:82%;min-width:0}
.message.assistant.spoken .message-stack>.message-body{max-width:100%;line-height:1.52;border-radius:6px 18px 18px 18px}
.composer-plus.open{color:var(--ink);background:rgba(255,255,255,.08)}
.slash-suggestions button[aria-selected="true"]{background:rgba(255,255,255,.08)}
.slash-suggestions small{flex-shrink:0;color:var(--muted);font-size:11px}

70
apps/web/src/computer.css Normal file
View File

@ -0,0 +1,70 @@
.computer-panel{display:flex;flex-direction:column;background:var(--panel);border-right:0;padding:0 20px}
.state-dot{width:7px;height:7px;border-radius:50%;background:var(--faint)}
.state-dot.running{background:var(--success)}
.state-dot.booting{background:#f5a03c;animation:pulse 1s infinite}
.state-dot.error{background:var(--danger)}
.desktop-frame{display:block;width:100%;height:100%;border:0;background:var(--inset)}
.empty-computer{height:100%;display:grid;place-content:center;justify-items:center;gap:7px;color:var(--muted);text-align:center}
.empty-computer svg{width:28px}
.empty-computer span{font-size:12px}
.computer-caption{display:flex;align-items:center;justify-content:space-between;padding:14px 0;color:var(--muted)}
.control-bar{display:flex;align-items:center;justify-content:flex-end;gap:8px;border-top:1px solid var(--hairline);padding:16px 0}
.computer-overlay{position:fixed;inset:0;z-index:50;display:flex;flex-direction:column;background:rgba(4,4,5,.98)}
.computer-overlay>header{height:64px;display:flex;align-items:center;justify-content:space-between;padding:0 18px;border-bottom:1px solid var(--line)}
.computer-overlay>header>div{display:flex;align-items:center;gap:10px}
.computer-overlay .avatar{width:30px;height:30px}
.control-badge{padding:5px 10px;border-radius:999px;background:rgba(48,162,75,.14);color:var(--success);font-size:13px}
.overlay-screen{flex:1;min-height:0;padding:20px;display:flex;justify-content:center}
.overlay-screen>.desktop-frame,.overlay-screen>.empty-computer{width:min(100%,1440px);height:100%;border:1px solid var(--border);border-radius:14px;overflow:hidden;background:var(--main)}
.overlay-error{position:absolute;top:74px;left:50%;transform:translateX(-50%);background:var(--error-bg);color:var(--error-fg);padding:9px 15px;border-radius:9px}
.computer-toggle{background:var(--surface)}
.computer-part{display:flex;flex-direction:column;min-height:0;flex:1;gap:10px;overflow:auto}
.computer-part.hidden-part{display:none}
.computer-status-row{display:flex;align-items:center;gap:8px;color:var(--muted)}
.computer-status-row>span{margin-right:auto;color:var(--ink);font-weight:600}
.computer-part .preview{flex:1;min-height:160px;aspect-ratio:auto}
/* Preserve the monitor's 16:10 shape as the right panel gets narrower. */
.computer-part .preview{position:relative;flex:0 1 auto;width:100%;height:auto;min-height:0;max-height:min(52vh,380px);aspect-ratio:16 / 10}
.computer-part .preview .desktop-frame{display:block;aspect-ratio:16 / 10}
.overlay-screen .overlay-desktop{position:relative;width:min(100%,1440px);height:100%}
.overlay-screen .overlay-desktop>.desktop-frame,.overlay-screen .overlay-desktop>.empty-computer{width:100%;height:100%;border:1px solid var(--border);border-radius:14px;overflow:hidden;background:var(--main)}
.computer-hud{position:absolute;top:10px;left:10px;z-index:3;display:grid;justify-items:center;gap:6px;padding:10px 12px 8px;border-radius:16px;background:rgba(10,10,12,.72);backdrop-filter:blur(8px);pointer-events:none;box-shadow:0 10px 30px rgba(0,0,0,.35)}
.computer-hud .avatar.blobatar.thinking{animation:computer-hud-bob 1.6s ease-in-out infinite,avatar-rainbow-glow 1.8s linear infinite}
.computer-hud-label{font-size:12px;letter-spacing:.02em;line-height:1.3;white-space:nowrap;background:linear-gradient(90deg,#7a8088 0%,#7a8088 28%,#fff 50%,#7a8088 72%,#7a8088 100%);background-size:220% 100%;-webkit-background-clip:text;background-clip:text;color:transparent;animation:working-shimmer 1.35s linear infinite}
.empty-computer.is-waiting{min-height:100%;background:var(--inset)}
.computer-actions{display:flex;align-items:center;justify-content:flex-end;gap:8px;min-width:0;flex-wrap:wrap}
.computer-actions .restart-computer{font-size:12px;padding-inline:10px;color:var(--muted)}
.computer-actions .restart-computer svg{width:14px;height:14px}
.computer-actions .primary{min-width:0}
.computer-overlay header .computer-actions{justify-content:flex-end}
.control-badge.teaching{background:rgba(239,85,85,.16);color:#ff8a8a}
.computer-login-hint{margin:0;color:var(--faint);font-size:12px;line-height:1.45}
.computer-part{scrollbar-gutter:stable;overflow-x:hidden}
.computer-part>*{flex-shrink:0;min-width:0}
.computer-part .preview{flex:none;max-height:none}
.computer-status-row>span{min-width:0;overflow-wrap:anywhere}
.computer-status-row>small,.state-dot{flex-shrink:0}
.control-bar{flex-wrap:wrap}
.control-bar>.computer-actions{flex:1 1 180px}
/* Independent monitor mascot, with mint halo and amber orbit. */
.computer-hud{inset:0;align-content:center;border-radius:inherit;gap:14px;background:radial-gradient(ellipse at center,#142722e8,#101012ed);box-shadow:none}
.computer-signal{position:relative;display:grid;place-items:center;width:64px;height:64px;border:1px solid #65dcb34d;border-radius:24px;animation:monitor-breathe 2.8s ease-in-out infinite}
.computer-signal::before{content:"";position:absolute;inset:-8px;border:1px solid #65dcb326;border-top-color:#efbe72;border-radius:50%;animation:monitor-orbit 4s linear infinite}
.computer-signal-face{display:flex;align-items:center;justify-content:center;gap:12px;width:46px;height:38px;border-radius:15px;background:#85dfbb;color:#143429;transform:rotate(-6deg)}
.computer-signal-face i{width:5px;height:11px;border-radius:5px;background:currentColor;animation:monitor-blink 4.6s ease-in-out infinite}
.computer-hud-label{color:#c9ddd5;background:none;animation:none;white-space:normal;text-align:center;max-width:90%;line-height:1.5}
.clipboard-status{margin:0;min-height:1.4em;font-size:12px;line-height:1.4;color:var(--muted)}
/* Keep control actions prominent and group less frequent power actions. */
.control-bar>.computer-actions{justify-content:flex-start}
.computer-power{position:relative;flex-shrink:0;margin-left:auto}
.computer-power-trigger{padding:8px;display:inline-flex;align-items:center;justify-content:center}
.computer-power-trigger svg{width:18px;height:18px}
.computer-power-menu{position:absolute;right:0;bottom:calc(100% + 8px);z-index:40;width:180px;padding:6px;border:1px solid var(--line);border-radius:12px;background:var(--surface);box-shadow:0 8px 28px #0005}
.computer-power-menu button{display:flex;align-items:center;gap:9px;width:100%;padding:10px;border:0;border-radius:7px;background:transparent;color:var(--ink);text-align:left;cursor:pointer}
.computer-power-menu button:hover{background:var(--surface-hover)}
.computer-power-menu svg{width:16px;height:16px}
.computer-power-menu .computer-power-stop{color:var(--danger-soft)}
.computer-power-menu small{display:block;padding:8px 10px 6px;border-top:1px solid var(--line);color:var(--muted);font-size:11px;line-height:1.6}
.computer-overlay .computer-power-menu{top:calc(100% + 8px);bottom:auto}

View File

@ -1,18 +1,94 @@
import { useSyncExternalStore, type ReactNode } from "react";
import { en } from "./locales/en";
import { zhTW } from "./locales/zh-TW";
export const supportedLocales = ["zh-TW"] as const;
export const supportedLocales = ["zh-TW", "en"] as const;
export type Locale = (typeof supportedLocales)[number];
const messages = { "zh-TW": zhTW } as const;
const messages = { "zh-TW": zhTW, en } as const;
const LOCALE_STORE = "lazyboy.locale";
const listeners = new Set<() => void>();
export const locale: Locale = "zh-TW";
export type MessageKey = keyof typeof zhTW;
export type MessageParams = Record<string, string | number>;
function isLocale(value: string): value is Locale {
return (supportedLocales as readonly string[]).includes(value);
}
function readStoredLocale(): Locale {
try {
const raw = localStorage.getItem(LOCALE_STORE);
if (raw && isLocale(raw)) return raw;
} catch {
/* private mode */
}
return "zh-TW";
}
let current: Locale = "zh-TW";
try {
current = readStoredLocale();
} catch {
current = "zh-TW";
}
export function getLocale(): Locale {
return current;
}
export function htmlLang(locale: Locale = current): string {
return locale === "en" ? "en" : "zh-Hant";
}
export function dateLocale(locale: Locale = current): string {
return locale === "en" ? "en" : "zh-TW";
}
export function listJoin(items: string[]): string {
return items.join(current === "en" ? ", " : "、");
}
function applyDocumentLang(locale: Locale) {
if (typeof document === "undefined") return;
document.documentElement.lang = htmlLang(locale);
}
applyDocumentLang(current);
export function setLocale(next: Locale) {
if (next === current) return;
current = next;
try {
localStorage.setItem(LOCALE_STORE, next);
} catch {
/* private mode */
}
applyDocumentLang(next);
listeners.forEach((fn) => fn());
}
export function subscribeLocale(listener: () => void): () => void {
listeners.add(listener);
return () => {
listeners.delete(listener);
};
}
export function t(key: MessageKey, params?: MessageParams): string {
const message: string = messages[locale][key];
const table = messages[current];
const message: string = table[key] ?? messages["zh-TW"][key];
if (!params) return message;
return message.replace(/\{(\w+)\}/g, (match, name: string) =>
Object.prototype.hasOwnProperty.call(params, name) ? String(params[name]) : match,
);
}
export function useLocale(): Locale {
return useSyncExternalStore(subscribeLocale, getLocale, getLocale);
}
export function LocaleRoot({ children }: { children: ReactNode }) {
useLocale();
return children;
}

457
apps/web/src/locales/en.ts Normal file
View File

@ -0,0 +1,457 @@
import type { zhTW } from "./zh-TW";
/** English UI copy. Keys must stay in lockstep with zh-TW. */
export const en: { [K in keyof typeof zhTW]: string } = {
search: "Search",
sharedComputer: "Shared computer",
privateComputer: "Private computer",
stopped: "Stopped",
booting: "Starting",
running: "Running",
suspended: "Sleeping",
error: "Error",
openComputer: "Start computer",
stopTask: "Stop task",
takeControl: "Take control",
takeOverNow: "Take over",
releaseControl: "Release control",
done: "Done",
skip: "Skip",
localWorkspace: "Local workspace",
pinned: "Pinned",
agentGroup: "Agent",
operationFailed: "That didnt work",
loadFailed: "Couldnt load",
settingsFailed: "Couldnt save settings",
loginFailed: "Couldnt sign in",
rememberFailed: "Couldnt remember that message",
clipboardWriteBlocked: "The browser blocked clipboard access.",
newConversation: "New chat",
workTools: "Work tools",
computer: "Computer",
memory: "Memory",
plugins: "Plugins",
pluginShort: "Plugin",
botSettings: "Bot settings",
add: "Add",
addBot: "New bot",
addGroup: "New group",
groups: "Groups",
unreadMessages: "{count} unread",
members: "{count} members",
hideHiddenItems: "Hide hidden items",
showHiddenItems: "Show hidden items",
mcpConnected: "{count} MCP connected",
connectMcpServer: "Connect an MCP server",
openOnPhone: "Open on phone",
settings: "Settings",
about: "About",
helpCenter: "Help",
sendFeedback: "Send feedback",
logout: "Sign out",
workspaceMenu: "Workspace menu",
chooseBot: "Choose a bot",
startRoomDiscussion: "Start a discussion with {name}",
startBotWork: "Start working with {name}",
roomWillReply: "{names} will all reply.",
botWelcome: "Send a message and it will get it done on its own computer.",
remembered: "Remembered",
remember: "Remember",
working: "{name} is working…",
copyCode: "Copy code",
copiedCode: "Copied",
copyMessage: "Copy",
copiedMessage: "Copied",
anotherConversationQueued: "Another chat is running, so this one is queued.",
workingStep: "{name} is working… {step}",
queuedMessages: "{count} more messages queued",
pausedUserControl: "Youre controlling the screen, so {name} is paused. Press “Release control” when youre done and it will continue from here (queued messages too).",
pausedNeedsUser: "{name} needs you on the screen (sign-in or verification). Press “Take over”, then “Release control” when youre done.",
releaseAndContinue: "Release and continue",
attachFile: "Attach file",
attachFileHint: "Images go to the model on this message. Other files land in the computer inbox and are deleted after two hours.",
attachTooMany: "Up to 4 attachments at a time",
attachTooLarge: "Each file must be 10 MB or smaller",
attachType: "That file type cant be attached",
attachRemove: "Remove {name}",
attachedFile: "Attached {name}",
attachDrop: "Drop to attach",
teachTask: "Teach a task",
teachTaskHint: "You demonstrate once; it turns that into a skill",
teachNeedsBot: "Choose a bot first",
taughtSkills: "Taught skills",
goalCommandHint: "Keep planning and executing until its verified or it needs you",
searchSkills: "Search skills",
noMatchingSkills: "No matching skills",
teachDialogLead: "Youll drive the screen while {name} watches: which controls you click, what you type, which pages you visit. It becomes a skill about intent and flow, not replayed coordinates.",
teachGoalLabel: "What are you demonstrating? (one sentence)",
teachGoalPlaceholder: "For example: open the STAR training system, finish the assigned videos, and complete the quiz",
teachTip1: "Work at a normal pace. Extra clicks are ignored.",
teachTip2: "Values that change (search terms, names, dates) can become parameters — use a real example while you demonstrate.",
teachTip3: "Password fields are not recorded. Sign in before you start if you need to.",
startDemo: "Start demo",
finishDemo: "Finish demo",
teachingBadge: "Recording",
teachingComposerHint: "Recording: finish the demo before sending a message",
teachingLive: "Learning: {goal}",
teachingHint: "The screen is yours — work on the right (or enlarge it). Press “Finish demo” when youre done.",
teachingCaptured: "{count} actions captured",
distilling: "Turning the “{goal}” demo into a skill…",
skillDraftTitle: "New skill draft",
skillDraftFrom: "From {count} actions",
skillName: "Skill name",
skillNamePlaceholder: "For example: Complete STAR training course",
skillInputs: "Inputs",
skillSteps: "Steps (intent, editable)",
moreSteps: "…and {count} more",
editSteps: "Edit steps",
skillCheck: "How to know its done",
skillDistillFailed: "The model couldnt tidy this up. This list is from the raw actions — edit before saving.",
discard: "Discard",
testRun: "Try it",
saveSkill: "Save skill",
messageTo: "Message {name}",
chooseConversationFirst: "Choose a chat first",
stopConversation: "Stop",
editSkill: "Edit skill",
editSkillFull: "Full edit",
runSkillNamed: "Run “{name}”",
skillEditLead: "Write this for the bot: intent and on-screen wording, not coordinates. One item per line.",
skillIntent: "Intent (what this skill should achieve)",
skillWhenToUse: "When to use it",
skillInputsLabel: "Inputs (one per line: name: description | e.g. example)",
skillInputsPlaceholder: "courseName: course title | e.g. Workplace Bullying Prevention (2026)",
skillPreconditions: "Preconditions",
skillStepsLabel: "Steps (one per line; use → for what you should see)",
skillStepsPlaceholder: "Click “Next” → page number increases; if Next is locked, wait until it enables",
skillHowToCheck: "How to know its done",
skillWhatToReturn: "What to report when finished",
skillCautions: "Cautions (one per line)",
deleteSkill: "Delete skill",
deleteSkillConfirm: "Delete skill “{name}”? The bot wont recognize it after that.",
skillSaved: "Saved",
exportSkill: "Export",
exportSkillHint: "Download as JSON to import on another bot",
importSkill: "Import skill",
importSkillHint: "Add a skill from a previously exported JSON file",
skillImportInvalid: "That isnt a valid skill file. Choose a JSON export from LazyBoy.",
collapseSidebar: "Collapse sidebar",
botComputer: "{name}s computer",
dedicatedScreen: "This screen",
enlarge: "Enlarge",
userControlling: "Youre in control",
aiReadOnly: "AI is working (view only)",
readOnly: "View only",
pasteClipboard: "Paste clipboard",
copyDesktopClipboard: "Copy desktop clipboard",
moreActions: "More",
unpin: "Unpin",
pin: "Pin",
markUnread: "Mark unread",
enterGroupName: "Enter a group name",
changeGroup: "Change group",
createOrMoveGroup: "Create / move to group",
removeFromGroup: "Remove from group",
unhide: "Unhide",
hide: "Hide",
delete: "Delete",
chooseConversation: "Choose a chat",
deleteConversation: "Delete chat",
clearConversation: "Clear this chat",
clearConversationTitle: "Clear this chat?",
clearConversationDescription: "Messages will be deleted. This agents long-term memory stays.",
cancel: "Cancel",
clearMessages: "Clear messages",
deleteConversationTitle: "Delete this chat?",
deleteConversationDescription: "It will leave the list. This agents long-term memory stays.",
avatarAppearance: "Avatar",
avatarAppearanceHint: "Color, shape, expression, and background. Eyes follow the cursor.",
color: "Color",
chooseColor: "Choose {value}",
customColor: "Custom color",
shape: "Shape",
expression: "Expression",
background: "Background",
bgNone: "None",
bgCircle: "Circle",
bgSquircle: "Squircle",
bgSquare: "Square",
exprIdle: "Idle",
exprHappy: "Happy",
exprSad: "Sad",
exprMad: "Mad",
exprSurprised: "Surprised",
exprWink: "Wink",
exprSleepy: "Sleepy",
exprSmug: "Smug",
exprUnsure: "Unsure",
exprScared: "Scared",
exprLove: "Love",
exprShy: "Shy",
exprSick: "Sick",
exprThinking: "Thinking",
name: "Name",
tags: "Tags",
tagsPlaceholder: "research, design, support (comma-separated)",
tagsLimit: "Up to 6 tags",
shortTitle: "Short title",
shortTitlePlaceholder: "For example: product research assistant",
description: "Description",
descriptionPlaceholder: "What this bot is for",
saving: "Saving…",
saveSettings: "Save settings",
deleteBot: "Delete bot",
memoryHelp: "Clearing a chat doesnt delete these. Store clear preferences or facts — never passwords.",
enableLongTermMemory: "Long-term memory",
addMemory: "Add memory",
memoryPlaceholder: "Clear preferences or facts only. Passwords and tokens are rejected.",
memorySearch: "Search",
filterMemory: "Filter memory",
noMemory: "No long-term memory yet. Ask the agent to remember something from chat, or add it here.",
noMatchingMemory: "No matching memories.",
save: "Save",
edit: "Edit",
clearAllMemoryConfirm: "Clear all memory?",
clearMemoryConfirm: "Clear",
clearAll: "Clear all",
mcpHelp: "Press “Choose MCP” to connect from the catalog. If a key is required, fill it in first. Local stdio servers start inside the API container.",
mcpNamePlaceholder: "For example: github",
mcpConnectFailed: "Couldnt connect. Check the key and try again.",
command: "Command",
commandPlaceholder: "npx, uvx, or a full path",
arguments: "Arguments",
environmentVariables: "Environment KEY=value",
headers: "Headers KEY=value",
connecting: "Connecting…",
connectMcp: "Connect MCP",
chooseMcp: "Choose MCP",
mcpPickerTitle: "Choose MCP",
mcpPickerLead: "Pick a featured or registry server. Every agent can use its tools.",
searchMcp: "Search MCP",
mcpRemote: "Remote",
mcpLocal: "Local",
mcpAdded: "Connected",
mcpNeedsKey: "Needs a key",
mcpNoResults: "No matching MCP. Try another search, or connect a custom server.",
mcpCustom: "Custom",
mcpBackToList: "Back to list",
mcpConnectNamed: "Connect {name}",
mcpKeyHint: "This MCP needs credentials.",
noMcp: "No MCP yet. Press “Choose MCP” above to connect one from the catalog.",
toolsCount: "{count} tools",
disabled: "Off",
disconnected: "Disconnected",
noTools: "No tools",
reconnect: "Reconnect",
disable: "Disable",
enable: "Enable",
preparingDesktop: "Preparing this agents desktop…",
computerPreviewHint: "Start the computer and the screen will show up here.",
bootingProgress: "Starting…",
restartComputer: "Restart",
shutDownComputer: "Shut down",
computerPower: "Computer menu",
sharedPowerHint: "This is a shared computer. The action affects every agent using it.",
stopAndTakeOver: "Stop and take over",
hudBooting: "Computer starting…",
hudWaking: "Waking…",
hudConnecting: "Connecting…",
hudHandoff: "Handing off…",
pasteToRemoteComputer: "Paste to remote computer",
pasteRemoteHelp: "Paste text here, then send it into VNC. This works over plain HTTP on the LAN.",
pasteTextPlaceholder: "Paste text here…",
pasteIntoVnc: "Paste into VNC",
botNamePlaceholder: "For example: research assistant",
sharedComputerHint: "Shares the environment with other bots",
privateComputerHint: "A fresh dedicated Docker desktop",
create: "Create",
groupDescription: "A group opens one chat where every selected agent speaks.",
groupName: "Group name",
groupNamePlaceholder: "For example: product research",
chooseBots: "Choose bots (at least two)",
creating: "Creating…",
createGroup: "Create group",
deleteGroup: "Delete group",
deleteNamed: "Delete {name}?",
deleteGroupDescription: "The group chat will be deleted. The bots and their own chats and memory stay.",
deleteDedicatedBotDescription: "The chat, private computer, and files on it will be permanently deleted.",
deleteSharedBotDescription: "The chat will be deleted. The shared computer and its files stay.",
phoneAccessDescription: "On the same LAN, open this URL in a phone browser and sign in with the same access token.",
localAddressWarning: "This is a loopback address — a phone cant open it. Use this computers LAN IP, for example http://192.168.x.x:3101.",
close: "Close",
copied: "Copied",
copyUrl: "Copy URL",
workspaceName: "Workspace name",
showHiddenBots: "Show hidden bots",
collapseRightSidebar: "Collapse right sidebar",
workspaceSettingsHint: "Avatar, memory, and MCP live in the right-hand tabs. This only changes how this workspace is shown.",
modelSettings: "Models",
modelSettingsHint: "This provider and model apply to bots that dont have their own.",
modelProvider: "Model provider",
providerXai: "xAI",
providerOpencodeGo: "OpenCode Go",
providerOpenaiCompatible: "OpenAI-compatible",
providerXaiHint: "Grok. Get a key from console.x.ai.",
providerOpencodeGoHint: "OpenCode Go subscription. Get a key from opencode.ai/auth.",
providerOpenaiCompatibleHint: "Self-hosted vLLM, Ollama, LiteLLM, or another OpenAI-compatible endpoint.",
apiKey: "API key",
apiKeyPlaceholder: "sk-…",
apiKeyStored: "A key is saved. Leave blank to keep it.",
usingEnvKey: "Using environment variable {name}.",
clearApiKey: "Clear saved key",
modelId: "Model",
modelIdPlaceholder: "For example: qwen2.5 or llama3.1",
reloadModels: "Reload models",
baseUrl: "Base URL",
baseUrlPlaceholder: "http://127.0.0.1:11434/v1",
baseUrlHint: "If the API runs in Docker, use host.docker.internal instead of localhost for services on this machine.",
workspaceModelHint: "Pick a provider, then the key and model. OpenAI-compatible needs your own endpoint.",
aboutDescription: "A local multi-agent workspace. Each bot has its own Linux computer. You can put them in a group chat and connect MCP tools.",
apiStatus: "API status: {status}",
statusNormal: "OK",
statusUnavailable: "Unreachable",
statusChecking: "Checking…",
helpBotsTitle: "Bots",
helpBots: "Use + at the top left to add a bot. Click the list to chat. Right-click to pin, hide, or delete.",
helpGroupsTitle: "Groups",
helpGroups: "+ → New group, pick at least two. One message, every agent in the group replies.",
helpComputerTitle: "Computer",
helpComputer: "The Computer tab on the right is this agents desktop. Start it, take over mouse and keyboard, or let it drive.",
helpMemoryTitle: "Memory",
helpMemory: "Clearing a chat doesnt wipe long-term memory. Ask the agent to remember, or add it in Memory on the right.",
helpMcpTitle: "MCP plugins",
helpMcp: "Plugins in the lower left connect MCP servers. Their tools show up on screen and the agent can use them in chat.",
helpSkillsTitle: "Skills",
helpSkills: "+ → Teach a task, demonstrate once, and it becomes a skill. Export JSON after, or import someone elses file onto another bot.",
helpAttachTitle: "Attachments",
helpAttach: "+ → Attach a file. Images go to the model on that message and arent stored in chat history. If the computer needs the original, it sits in inbox/ for two hours, then is deleted.",
helpShortcutsTitle: "Shortcuts",
helpShortcuts: "Enter sends, Shift+Enter makes a new line. While its replying, the send button becomes Stop. On a call, Space interrupts and Esc hangs up.",
helpVoiceTitle: "Voice calls",
helpVoice: "The phone in a 1:1 header calls the bot. Speak a task and it uses its computer. It answers immediately whether it can, and you can interrupt to change course.",
call: "Call",
hangUp: "Hang up",
interrupt: "Interrupt",
listening: "Listening…",
speaking: "Speaking…",
callConnecting: "Connecting…",
workingOnComputer: "Working on the computer…",
callPrompt: "Just say what you need. A short pause sends it.",
callShortcuts: "Space interrupts · Esc hangs up",
callFailed: "The call dropped.",
micDenied: "Microphone permission is required for calls.",
micFailed: "Couldnt open the microphone.",
callAudioFailed: "Call audio stopped working. Hang up and call again.",
takeOverToContinue: "You need to take over the screen.",
setUpVoiceToCall: "Set up voice before calling",
voiceEnabled: "Enable voice calls",
voiceEnabledHint: "Off by default. Turn on and save to use the call button.",
voiceDisabledHint: "Voice calls are off. Turn them on in Settings → Voice.",
voiceSettings: "Voice",
voiceSettingsHint: "Voice is separate from the text model. If you already have an xAI chat key, you usually dont need another. Billed per minute.",
voiceProvider: "Voice provider",
voiceModel: "Voice model",
voiceName: "Voice",
providerOpenai: "OpenAI",
providerXaiVoiceHint: "Grok Voice. Live calls with barge-in; computer work is handed to the text model.",
providerOpenaiVoiceHint: "OpenAI Realtime. Same call UI; you can switch later.",
voiceReusesTextKey: "Reusing the xAI key saved for the text model.",
feedbackDescription: "Write a problem, idea, or request. This is a local workspace, so the text is copied to the clipboard for you to paste into an issue or message.",
feedbackPlaceholder: "For example: in group chats I want to pick who speaks first…",
copyContent: "Copy",
loginTitle: "Sign in to LazyBoy",
loginDescription: "Enter the shared access token configured on the server.",
accessToken: "Access token",
verifying: "Checking…",
login: "Sign in",
agentComputer: "Agent computer",
url: "URL",
stdio: "stdio",
http: "HTTP",
sse: "SSE",
argumentsPlaceholder: "-y @modelcontextprotocol/server-github",
environmentVariablesPlaceholder: "GITHUB_TOKEN=…",
urlPlaceholder: "https://mcp.example.com/mcp",
headersPlaceholder: "Authorization=Bearer …",
back: "Back",
accounts: "Accounts",
accountsHelp: "Stored in this bots vault. Passwords are entered only here — never in chat, never shown to the model. Schedules fill the form at a login wall.",
accountSite: "Site",
accountSitePlaceholder: "For example: Gmail",
accountHost: "URL or domain",
accountHostPlaceholder: "mail.google.com",
accountUsername: "Username",
accountPassword: "Password",
accountPasswordKeep: "Leave blank to keep the current password",
accountNotes: "Notes",
accountNotesPlaceholder: "Optional, for example: work inbox",
addAccount: "Add account",
noAccounts: "No saved accounts yet. Add one here if a schedule should sign in by itself.",
loginNeedsYou: "Needs you to sign in on the keyboard",
loginOpenScreen: "Open its screen",
loginWhy: "After sign-in: {why}",
scheduleChip: "Scheduled",
scheduleRunChip: "Schedule run",
schedules: "Schedules",
schedule: "Schedule",
schedCreate: "New schedule",
schedEmpty: "No schedules yet. Say “every day at 9…” in chat, or press +.",
schedPaused: "Paused",
schedRunNow: "Run now",
schedRunning: "Running…",
schedActive: "On",
schedNamePlaceholder: "For example: morning digest",
schedInstruction: "What to do each time",
schedInstructionPlaceholder: "For example: open mail and write an unread summary back to chat.",
schedWhen: "When",
schedEveryHour: "Every hour",
schedEveryDay: "Every day",
schedWeekdays: "Weekdays",
schedEveryWeek: "Every Monday",
schedEveryMonth: "Monthly",
schedInterval: "Interval",
schedAdvanced: "Advanced cron",
schedMinutes: "minutes",
schedHours: "hours",
schedDays: "days",
computerLoginHint: "Sign in inside this bots own browser. The session stays on its computer — dont type passwords in chat.",
language: "Interface language",
languageZh: "繁體中文",
languageEn: "English",
languageHint: "Interface language. Chat history and agent replies are not translated.",
helpLanguageTitle: "Language",
helpLanguage: "Switch between Traditional Chinese and English in Settings, accessible from the workspace menu.",
clipboardSynced: "Clipboard synced",
clipboardSyncBlocked: "The browser blocked sync. Use the copy button.",
clipboardPasted: "Pasted",
clipboardPasteFailed: "Paste failed. Take over the computer and try again.",
clipboardCopiedSelection: "Copied selection",
clipboardCopyFailed: "Copy didnt sync. Use the copy button, or Ctrl+Shift+C on the desktop.",
slashMode: "Mode",
slashFileSkill: "Skill",
slashList: "Commands and skills",
schedIntervalHint: "Elapsed time, not the clock. A day is 24 hours and doesnt reset at month-end or DST.",
schedInvalidInterval: "Interval must be a whole number from 1 to 365",
schedHumanDaily: "Daily at {time}",
schedHumanWeekdays: "Weekdays at {time}",
schedHumanWeekly: "Mondays at {time}",
schedHumanMonthly: "Monthly at {time}",
schedHumanInterval: "Every {n} {unit}",
stepThinking: "Thinking",
stepWatchingScreen: "Looking at the screen",
stepUsingSkill: "Using skill {name}",
stepFillingLogin: "Filling a saved login",
stepListingAccounts: "Listing saved accounts",
stepScheduling: "Scheduling {name}",
stepListingSchedules: "Listing schedules",
stepCancelingSchedule: "Canceling a schedule",
stepRecheckingScreen: "Checking the screen again",
schedEveryMinute: "Every minute",
schedHumanEveryMinutes: "Every {n} minutes",
schedHumanEveryHours: "Every {n} hours",
schedHumanMonthlyDay: "Monthly on the 1st at {time}",
schedHumanElapsed: "Every {n} minutes (elapsed)",
schedCalendar: "Calendar schedule: {expr}",
teachInProgress: "A demo is in progress. Finish or cancel it before sending a message.",
aiTimeout: "The model timed out (120 seconds).",
};

View File

@ -13,6 +13,7 @@ export const zhTW = {
openOnPhone: "在手機開啟", settings: "設定", about: "關於", helpCenter: "說明中心", sendFeedback: "傳送意見回饋", logout: "登出", workspaceMenu: "工作區選單",
chooseBot: "選擇一個機器人", startRoomDiscussion: "和 {name} 開始討論", startBotWork: "和 {name} 開始工作", roomWillReply: "{names} 會一起回覆。",
botWelcome: "傳送訊息,讓它在自己的電腦上完成任務。", remembered: "已記住", remember: "記住", working: "{name} 正在工作…",
copyCode: "複製程式碼", copiedCode: "已複製", copyMessage: "複製", copiedMessage: "已複製",
anotherConversationQueued: "另一則對話正在執行,這則會排隊。",
workingStep: "{name} 正在工作… {step}", queuedMessages: "還有 {count} 則訊息排隊中",
pausedUserControl: "你正在操控畫面,{name} 已暫停。完成後按「釋放控制」,它會從目前畫面接著做(排隊中的訊息也會一起處理)。",
@ -21,6 +22,7 @@ export const zhTW = {
attachTooMany: "一次最多 4 個附件", attachTooLarge: "單檔不能超過 10 MB", attachType: "這個檔案類型不能附加",
attachRemove: "移除 {name}", attachedFile: "附件 {name}", attachDrop: "放到這裡附加",
teachTask: "教它一項任務", teachTaskHint: "你示範一次,它學成技能", teachNeedsBot: "先選擇一個機器人", taughtSkills: "已學會的技能",
goalCommandHint: "持續規劃並執行,直到驗證完成或需要你介入",
searchSkills: "搜尋技能", noMatchingSkills: "沒有符合的技能",
teachDialogLead: "接下來畫面交給你操作,{name} 會在旁邊看:記錄你點了哪些控制項、輸入了什麼、去了哪些頁面,之後整理成一個「知道目的與流程」的技能,而不是死記座標。",
teachGoalLabel: "你要示範什麼?(一句話說目標)", teachGoalPlaceholder: "例如:到 STAR 訓練系統,把指定課程的影片看完並完成測驗",
@ -62,7 +64,8 @@ export const zhTW = {
mcpNoResults: "找不到符合的 MCP。換個關鍵字或改用自訂接入。", mcpCustom: "自訂接入", mcpBackToList: "回到列表",
mcpConnectNamed: "接入 {name}", mcpKeyHint: "這個 MCP 需要憑證才能連。",
noMcp: "還沒有 MCP。點上面的「選擇 MCP」從市集接入。", toolsCount: "{count} 個工具", disabled: "已關閉", disconnected: "未連線", noTools: "沒有可用工具", reconnect: "重新連線", disable: "停用", enable: "啟用",
preparingDesktop: "正在準備 Agent 的獨立桌面…", computerPreviewHint: "開啟電腦後,畫面會顯示在這裡。", bootingProgress: "啟動中…", restartDocker: "重啟 Docker", stopAndTakeOver: "停止並接管",
preparingDesktop: "正在準備 Agent 的獨立桌面…", computerPreviewHint: "開啟電腦後,畫面會顯示在這裡。", bootingProgress: "啟動中…", restartComputer: "重啟", shutDownComputer: "關閉電腦", computerPower: "電腦選單", sharedPowerHint: "此為共用電腦,操作會影響使用它的所有 Agent。", stopAndTakeOver: "停止並接管",
hudBooting: "電腦啟動中…", hudWaking: "喚醒中…", hudConnecting: "連線中…", hudHandoff: "換手中…",
pasteToRemoteComputer: "貼到遠端電腦", pasteRemoteHelp: "把外面的文字貼在這裡,再送進 VNC。這個方式在區網 HTTP 也能使用。", pasteTextPlaceholder: "在此貼上文字…", pasteIntoVnc: "貼入 VNC",
botNamePlaceholder: "例如:研究助理", sharedComputerHint: "與其他機器人共用環境", privateComputerHint: "全新的獨立 Docker", create: "建立",
groupDescription: "拉進群組會開一個對話,選中的 Agent 都會在裡面發言。", groupName: "群組名稱", groupNamePlaceholder: "例如:產品研究", chooseBots: "選擇機器人(至少兩位)", creating: "建立中…", createGroup: "建立群組",
@ -90,10 +93,60 @@ export const zhTW = {
helpMcpTitle: "MCP 外掛", helpMcp: "左下「外掛程式」接入 MCP server。連上的工具會顯示在畫面上對話時 Agent 可以使用。",
helpSkillsTitle: "技能", helpSkills: " → 教它一項任務,示範一次就會整理成技能。示範結束後可以匯出 JSON或把別人的技能檔匯入換一個機器人也適用。",
helpAttachTitle: "附件", helpAttach: " → 附加檔案。圖片這則訊息就會給模型看,不會存進對話紀錄。若機器人電腦要打開原檔,會暫放 inbox/,兩小時後自動刪,避免把磁碟塞滿。",
helpShortcutsTitle: "快捷鍵", helpShortcuts: "Enter 送出Shift+Enter 換行。正在回覆時送出鈕會變成停止。",
helpShortcutsTitle: "快捷鍵", helpShortcuts: "Enter 送出Shift+Enter 換行。正在回覆時送出鈕會變成停止。通話中空白鍵插話Esc 掛斷。",
helpVoiceTitle: "語音通話", helpVoice: "1:1 對話標題列的電話可以打給機器人。用說話下指令,它會去操作自己的電腦;能不能做會立刻用聲音回你,中途可以插嘴改指令。",
call: "通話", hangUp: "掛斷", interrupt: "插話", listening: "正在聽你說…", speaking: "正在說話…",
callConnecting: "連線中…", workingOnComputer: "正在電腦上做…", callPrompt: "直接說要做什麼。沉默一下就會送出。",
callShortcuts: "空白鍵插話 · Esc 掛斷", callFailed: "通話中斷了。", micDenied: "需要麥克風權限才能通話。", micFailed: "麥克風打不開。",
callAudioFailed: "通話聲音出問題了,請掛斷後重打。",
takeOverToContinue: "需要你接手畫面。", setUpVoiceToCall: "先設定語音才能通話",
voiceEnabled: "啟用語音通話", voiceEnabledHint: "預設關閉。開啟並儲存後,才能使用通話按鈕。", voiceDisabledHint: "語音通話尚未啟用,請到「設定 → 語音」開啟。",
voiceSettings: "語音", voiceSettingsHint: "語音跟文字模型分開。xAI 若已有聊天金鑰,通常不用再填一次。按分鐘計費。",
voiceProvider: "語音供應商", voiceModel: "語音模型", voiceName: "聲音",
providerOpenai: "OpenAI", providerXaiVoiceHint: "Grok Voice。即時通話可插嘴並把電腦工作交給文字模型去做。",
providerOpenaiVoiceHint: "OpenAI Realtime。同一套通話介面之後可切換。",
voiceReusesTextKey: "目前會重用文字模型已存的 xAI 金鑰。",
feedbackDescription: "寫下問題、想法或想要的功能。這是本機工作區,內容會複製到剪貼簿,方便你貼到 issue 或訊息裡。", feedbackPlaceholder: "例如:群組對話希望可以指定誰先發言…", copyContent: "複製內容",
loginTitle: "登入 LazyBoy", loginDescription: "輸入伺服器設定的共享存取 token。", accessToken: "存取 token", verifying: "驗證中…", login: "登入",
agentComputer: "Agent 電腦", url: "URL", stdio: "stdio", http: "HTTP", sse: "SSE",
argumentsPlaceholder: "-y @modelcontextprotocol/server-github", environmentVariablesPlaceholder: "GITHUB_TOKEN=…",
urlPlaceholder: "https://mcp.example.com/mcp", headersPlaceholder: "Authorization=Bearer …",
back: "返回", accounts: "帳號", accountsHelp: "存在這台機器人自己的保險箱。密碼只在這裡填,不會進聊天、也不會給模型看。排程碰到登入牆時會自動填表。",
accountSite: "站名", accountSitePlaceholder: "例如Gmail", accountHost: "網址或網域", accountHostPlaceholder: "mail.google.com",
accountUsername: "帳號", accountPassword: "密碼", accountPasswordKeep: "留空表示不改密碼",
accountNotes: "備註", accountNotesPlaceholder: "選填,例如:公司信箱",
addAccount: "新增帳號", noAccounts: "還沒有已存帳號。排程若要自動登入,先在這裡加一筆。",
loginNeedsYou: "需要你在鍵盤上登入", loginOpenScreen: "打開它的畫面", loginWhy: "登入之後:{why}",
scheduleChip: "已排程", scheduleRunChip: "排程執行",
schedules: "排程", schedule: "排程", schedCreate: "新增排程", schedEmpty: "還沒有排程。對話裡說「以後每天 9 點…」或按+。",
schedPaused: "已暫停", schedRunNow: "立刻跑", schedRunning: "執行中…", schedActive: "啟用",
schedNamePlaceholder: "例如:早報摘要", schedInstruction: "每次要做什麼",
schedInstructionPlaceholder: "例如:打開信箱,把未讀摘要寫回對話。",
schedWhen: "何時執行", schedEveryHour: "每小時", schedEveryDay: "每天", schedWeekdays: "工作日",
schedEveryWeek: "每週一", schedEveryMonth: "每月", schedInterval: "間隔", schedAdvanced: "進階 cron",
schedMinutes: "分鐘", schedHours: "小時", schedDays: "天",
computerLoginHint: "在這台機器人自己的瀏覽器登入。Session 留在它的電腦,不要把密碼打在聊天裡。",
language: "介面語言", languageZh: "繁體中文", languageEn: "English",
languageHint: "介面語言。對話內容與 Agent 回覆不會自動翻譯。",
helpLanguageTitle: "語言", helpLanguage: "在左下角工作區選單的設定裡,可切換繁體中文與英文。",
clipboardSynced: "剪貼簿已同步", clipboardSyncBlocked: "瀏覽器未允許同步,請按複製按鈕",
clipboardPasted: "已貼上文字", clipboardPasteFailed: "貼上失敗,請確認已接管電腦後重試",
clipboardCopiedSelection: "已複製選取文字", clipboardCopyFailed: "複製未同步;請按複製按鈕,或在遠端使用 Ctrl+Shift+C",
slashMode: "執行模式", slashFileSkill: "檔案技能", slashList: "指令與技能",
schedIntervalHint: "固定經過時間;一天為 24 小時,不因月底或日光節約時間重置。",
schedInvalidInterval: "間隔必須為 1365 的整數",
schedHumanDaily: "每天 {time}", schedHumanWeekdays: "工作日 {time}",
schedHumanWeekly: "每週一 {time}", schedHumanMonthly: "每月 {time}",
schedHumanInterval: "每 {n} {unit}",
stepThinking: "思考中", stepWatchingScreen: "看畫面", stepUsingSkill: "讀取技能 {name}",
stepFillingLogin: "填入已存帳號", stepListingAccounts: "列出已存帳號",
stepScheduling: "排程 {name}", stepListingSchedules: "列出排程", stepCancelingSchedule: "取消排程",
stepRecheckingScreen: "重新確認畫面",
schedEveryMinute: "每分鐘",
schedHumanEveryMinutes: "每 {n} 分鐘", schedHumanEveryHours: "每 {n} 小時",
schedHumanMonthlyDay: "每月 1 日 {time}",
schedHumanElapsed: "每隔 {n} 分鐘(固定間隔)",
schedCalendar: "日曆排程:{expr}",
teachInProgress: "示範進行中:先按「完成示範」或「取消」,再送訊息。",
aiTimeout: "AI 回應逾時120 秒)",
} as const;

View File

@ -4,5 +4,12 @@ import "blobatar/motion.css";
import "blobatar/gaze.css";
import "./styles.css";
import "./refinements.css";
import "./avatar.css";
import "./chat.css";
import "./computer.css";
import "./schedule.css";
import "./call.css";
import "./responsive.css";
import { App } from "./App";
ReactDOM.createRoot(document.getElementById("root")!).render(<App />);
import { LocaleRoot } from "./i18n";
ReactDOM.createRoot(document.getElementById("root")!).render(<LocaleRoot><App /></LocaleRoot>);

56
apps/web/src/markdown.css Normal file
View File

@ -0,0 +1,56 @@
.md-body{min-width:0;overflow-wrap:anywhere}
.md-body>:first-child,.md-body>:first-child>:first-child{margin-top:0}
.md-body>:last-child,.md-body>:last-child>:last-child{margin-bottom:0}
.md-body p,.md-body blockquote,.md-body ul,.md-body ol,.md-body table{margin:.65em 0}
.md-body h1,.md-body h2,.md-body h3,.md-body h4,.md-body h5,.md-body h6{
margin:.85em 0 .3em;color:var(--ink);font-weight:400;line-height:1.3;letter-spacing:.01em
}
.md-body h1{font-size:1.22em}
.md-body h2{font-size:1.12em}
.md-body h3,.md-body h4,.md-body h5,.md-body h6{font-size:1.04em}
.md-body ul,.md-body ol{padding-left:1.4em}
.md-body ul{list-style:disc}
.md-body ol{list-style:decimal}
.md-body li+li{margin-top:.28em}
.md-body li>p{margin:0}
.md-body .contains-task-list,.md-body .task-list-item{list-style:none}
.md-body .contains-task-list{padding-left:0}
.md-body a{
color:var(--accent);text-decoration:underline;
text-decoration-color:color-mix(in srgb,var(--accent) 55%,transparent);
text-underline-offset:.16em
}
.md-body a:hover{color:#7ee0c8}
.md-body code{
border:1px solid var(--border);border-radius:.35em;background:var(--inset);
padding:.08em .32em;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;
font-size:.88em;direction:ltr;unicode-bidi:isolate
}
.md-body pre{
max-width:100%;overflow-x:auto;margin:0;border:1px solid var(--border);
border-radius:12px;background:var(--input);padding:.75em .9em;direction:ltr;unicode-bidi:isolate
}
.md-body pre code{border:0;background:transparent;padding:0;font-size:.84em;white-space:pre}
.md-pre-wrap{position:relative;margin:.65em 0}
.md-copy{
position:absolute;top:.45em;right:.45em;display:grid;place-items:center;
width:28px;height:28px;border:1px solid var(--border);border-radius:8px;
background:#17171a;color:var(--muted);opacity:0;cursor:pointer
}
.md-pre-wrap:hover .md-copy,.md-copy:focus-visible{opacity:1}
.md-copy:hover{color:var(--ink);background:#202024}
.md-body blockquote{
margin:.65em 0;border-left:3px solid var(--accent);padding-left:.85em;color:var(--muted)
}
.md-body table{display:block;max-width:100%;overflow-x:auto;border-collapse:collapse;font-size:.92em}
.md-body th,.md-body td{
border:1px solid var(--border);padding:.4em .6em;text-align:left;overflow-wrap:break-word
}
.md-body th{background:#1a1a1d;color:var(--ink);font-weight:400}
.md-body hr{margin:.9em 0;border:0;border-top:1px solid var(--border)}
.md-body img{display:block;max-width:100%;height:auto;margin:.65em 0;border-radius:12px}
.md-body input[type="checkbox"]{margin-right:.4em;accent-color:var(--accent)}
.message-body.md{white-space:normal}
.user .md-body code,.user .md-body pre,.user .md-body th{background:#e8e8e6;border-color:#d4d4d2;color:var(--on-cream)}
.user .md-body a{color:#0f766e}
.user .md-copy{background:#ececea;color:#555}

110
apps/web/src/markdown.tsx Normal file
View File

@ -0,0 +1,110 @@
import { memo, useCallback, useRef, useState, type ComponentPropsWithoutRef } from "react";
import ReactMarkdown, { type Components } from "react-markdown";
import remarkBreaks from "remark-breaks";
import remarkGfm from "remark-gfm";
import { t } from "./i18n";
import "./markdown.css";
const protocolPattern = /^([a-z][a-z\d+.-]*):/i;
const safeProtocols = new Set(["http", "https", "mailto", "tel"]);
export function sanitizeMarkdownUrl(url: string): string | undefined {
const value = url.trim();
const protocol = value.match(protocolPattern)?.[1]?.toLowerCase();
if (protocol) return safeProtocols.has(protocol) ? value : undefined;
if (value.startsWith("#") && !value.toLowerCase().startsWith("#javascript")) return value;
return undefined;
}
function CopyIcon() {
return (
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" aria-hidden="true">
<rect x="9" y="9" width="12" height="12" rx="2" stroke="currentColor" strokeWidth="2" strokeLinejoin="round"/>
<path d="M5 15H4a1 1 0 0 1-1-1V4a1 1 0 0 1 1-1h10a1 1 0 0 1 1 1v1" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"/>
</svg>
);
}
function CheckIcon() {
return (
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" aria-hidden="true">
<path d="M4 12.5 9.5 18 20 6" stroke="currentColor" strokeWidth="2.25" strokeLinecap="round" strokeLinejoin="round"/>
</svg>
);
}
function useCopiedFlag() {
const [copied, setCopied] = useState(false);
const timerRef = useRef<number>(undefined);
const markCopied = useCallback(() => {
setCopied(true);
window.clearTimeout(timerRef.current);
timerRef.current = window.setTimeout(() => setCopied(false), 1500);
}, []);
return {copied, markCopied};
}
function copyText(text: string): Promise<boolean> {
if (!navigator.clipboard) return Promise.resolve(false);
return navigator.clipboard.writeText(text).then(() => true).catch(() => false);
}
function CodeBlock(props: ComponentPropsWithoutRef<"pre">) {
const preRef = useRef<HTMLPreElement>(null);
const {copied, markCopied} = useCopiedFlag();
const handleCopy = useCallback(() => {
void copyText(preRef.current?.textContent ?? "").then((ok) => { if (ok) markCopied(); });
}, [markCopied]);
return (
<div className="md-pre-wrap">
<pre {...props} ref={preRef}/>
<button type="button" className="md-copy" onClick={handleCopy} aria-label={copied ? t("copiedCode") : t("copyCode")}>
{copied ? <CheckIcon/> : <CopyIcon/>}
</button>
</div>
);
}
const components: Components = {
a({node: _node, ...props}) {
return <a {...props} target="_blank" rel="noreferrer noopener"/>;
},
img({node: _node, ...props}) {
return <img {...props} alt={props.alt ?? ""} loading="lazy"/>;
},
pre({node: _node, ...props}) {
return <CodeBlock {...props}/>;
},
};
export function CopyMessageButton({text}:{text:string}) {
const {copied, markCopied} = useCopiedFlag();
if (!text.trim()) return null;
return (
<button
type="button"
className={`copy-msg ${copied ? "copied" : ""}`}
title={copied ? t("copiedMessage") : t("copyMessage")}
aria-label={copied ? t("copiedMessage") : t("copyMessage")}
onClick={() => { void copyText(text).then((ok) => { if (ok) markCopied(); }); }}
>
{copied ? <CheckIcon/> : <CopyIcon/>}
<span>{copied ? t("copiedMessage") : t("copyMessage")}</span>
</button>
);
}
export const ChatMarkdown = memo(function ChatMarkdown({children}:{children:string}) {
return (
<div className="md-body">
<ReactMarkdown
components={components}
remarkPlugins={[remarkGfm, remarkBreaks]}
skipHtml
urlTransform={(url) => sanitizeMarkdownUrl(url) ?? ""}
>
{children}
</ReactMarkdown>
</div>
);
});

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,57 @@
@media(max-width:1050px){.app-shell{grid-template-columns:250px 1fr}.computer-panel{display:none}}
@media(max-width:700px){.app-shell{display:block}.sidebar{position:fixed;z-index:40;inset:0 auto 0 0;width:min(300px,88vw);transform:translateX(-105%);transition:.2s;box-shadow:20px 0 60px #000}.sidebar.open{transform:none}.chat-panel{height:100%}.mobile-menu{display:inline-flex}.topbar{padding:0 12px}.messages{padding:24px 18px 130px}.composer{left:12px;right:12px}.computer-overlay>header{height:auto;min-height:64px;flex-wrap:wrap;padding:10px}.computer-overlay>header>div:last-child{flex-wrap:wrap;justify-content:flex-end}.overlay-screen{padding:8px}.mode-grid{grid-template-columns:1fr}}
@media(max-width:1200px){.app-shell{grid-template-columns:230px minmax(0,1fr) 330px}}
@media(max-width:1050px){.app-shell{grid-template-columns:250px minmax(0,1fr)}}
@media(max-width:700px){.chat-panel{--chat-gutter:18px}.app-shell{display:block}.message{width:100%}.message>span,.message>.message-body,.message-stack{max-width:90%}.composer-dock{padding:20px var(--chat-gutter) 16px}.composer{width:var(--chat-col);min-height:60px}.composer-files{padding:2px 4px 8px 8px}.file-card{max-width:100%;height:52px}.file-card-remove{flex-basis:32px;width:32px;height:32px}.message.with-files .msg-attachments{max-width:100%}.messages{padding-bottom:118px}.chat-panel:has(.composer-dock.has-status) .messages{padding-bottom:178px}}
@media(prefers-reduced-motion:reduce){.avatar.blobatar.thinking:before,.avatar.blobatar.thinking:after,.thinking-dots i{animation:none!important}}
@media(max-width:1200px){.app-shell.right-open{grid-template-columns:230px minmax(0,1fr) 340px}.app-shell.right-collapsed{grid-template-columns:230px minmax(0,1fr) 52px}}
@media(max-width:1050px){
.panel-open-btn{display:inline-flex}
.app-shell.right-collapsed{grid-template-columns:250px minmax(0,1fr)}
.app-shell.right-collapsed .side-card{display:none}
.app-shell.right-open{grid-template-columns:250px minmax(0,1fr)}
.app-shell.right-open .side-card-backdrop{display:block;position:fixed;inset:0;z-index:44;background:rgba(0,0,0,.5)}
.app-shell.right-open .side-card{position:fixed;z-index:45;inset:0 0 0 auto;width:min(420px,100vw);box-shadow:-20px 0 60px #000}
}
@media(prefers-reduced-motion:reduce){.working-label{animation:none;color:var(--muted);background:none}}
@media(prefers-reduced-motion:reduce){.computer-hud .avatar.blobatar,.computer-hud-label{animation:none!important}.computer-hud-label{color:var(--muted);background:none}}
@media(max-width:700px){
.topbar{gap:8px}
.top-tools{gap:2px;padding:3px}
.top-tool-button{width:32px;height:32px}
.topbar>strong{min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.session-picker{margin-left:2px}
.session-current{max-width:min(32vw,150px)}
.message.assistant.spoken{grid-template-columns:24px minmax(0,1fr);column-gap:8px}
.computer-part .preview{max-height:none}
}
@media(prefers-reduced-motion:reduce){.computer-signal,.computer-signal::before,.computer-signal-face i,.avatar.blobatar.thinking,.avatar-stack .stack-item .avatar.thinking{animation:none!important}}
/* Mobile conversation drawer and two-row chat header. */
.mobile-nav-backdrop,.mobile-nav-close{display:none}
.brand-icon{width:30px;height:30px;flex:none;border-radius:9px;margin-right:9px}
@media(max-width:700px){
.mobile-nav-backdrop{display:block;position:fixed;inset:0;z-index:39;border:0;border-radius:0;
padding:0;background:#0008;backdrop-filter:blur(3px);touch-action:manipulation}
.sidebar{visibility:hidden;padding-top:env(safe-area-inset-top);padding-bottom:env(safe-area-inset-bottom)}
.sidebar.open{visibility:visible}
.mobile-nav-close{display:inline-flex;align-items:center;justify-content:center;min-width:44px;min-height:44px;
margin-left:auto;font-size:26px;line-height:1}
.brand .create-menu-wrap{margin-left:4px}
.topbar{display:grid;grid-template-columns:44px 32px minmax(0,1fr) minmax(0,104px);
grid-template-rows:44px auto;gap:8px;height:auto;flex:0 0 auto;
padding:calc(8px + env(safe-area-inset-top)) 12px 8px;align-items:center}
.topbar>.mobile-menu{grid-column:1;grid-row:1;width:44px;height:44px;justify-content:center}
.topbar>.avatar,.topbar>.avatar-stack{grid-column:2;grid-row:1;margin:0}
.topbar>strong{grid-column:3;grid-row:1;font-size:15px;line-height:1.3}
.topbar>.session-picker{grid-column:4;grid-row:1;margin:0;min-width:0}
.topbar .session-current{max-width:100%;min-height:44px;width:100%}
.topbar>.grow{display:none}
.topbar>.top-tools{grid-column:1 / -1;grid-row:2;display:flex;width:100%;min-width:0;
margin:0;padding:0;border:0;border-radius:0;gap:4px;background:transparent;box-shadow:none}
.topbar .top-tool-button,.topbar .call-entry{flex:1;min-width:0;width:auto}
.topbar .top-tool-button{height:44px;min-height:44px;border-radius:10px;background:var(--surface)}
.topbar .call-entry{display:flex}
.topbar .call-entry>.top-tool-button{width:100%}
.topbar .session-menu{right:0;left:auto;max-width:calc(100vw - 24px)}
}

37
apps/web/src/schedule.css Normal file
View File

@ -0,0 +1,37 @@
.login-chip,.sched-chip{display:grid;gap:6px;margin:8px 0;padding:12px 14px;border:1px solid var(--border);border-radius:12px;background:var(--inset);max-width:min(var(--chat-col),100%)}
.login-chip .login-label,.sched-chip .sched-label{color:var(--muted);font-size:12px}
.login-chip .login-site{font-weight:600}
.login-chip .login-why{color:var(--muted);font-size:13px}
.sched-list{display:grid;gap:8px;min-height:0;overflow:auto}
.sched-list-head{display:flex;align-items:center;justify-content:space-between;color:var(--muted);font-size:13px}
.sched-empty{margin:0;color:var(--faint);font-size:13px}
.sched-row{display:flex;align-items:center;gap:8px}
.sched-row-main{display:flex;align-items:center;gap:10px;min-width:0;flex:1;padding:8px;border:0;border-radius:10px;background:transparent;color:inherit;text-align:left;cursor:pointer}
.sched-row-main:hover{background:var(--surface)}
.sched-row-main span{min-width:0;display:grid}
.sched-row-main strong{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.sched-row-main small{color:var(--muted);font-size:12px}
.sched-dot{width:8px;height:8px;border-radius:50%;background:var(--faint)}
.sched-dot.on{background:var(--online)}
.sched-editor{display:grid;gap:12px;min-height:0;overflow:auto}
.sched-editor-head{display:flex;align-items:center;gap:8px}
.sched-when{display:grid;gap:8px;color:var(--muted);font-size:13px}
.sched-card{display:flex;flex-wrap:wrap;gap:8px;padding:10px;border:1px solid var(--border);border-radius:12px;background:var(--input)}
.sched-card select,.sched-cron{height:36px;border:1px solid var(--border);border-radius:8px;background:var(--surface);color:var(--ink);padding:0 8px;font:inherit}
.sched-cron{flex:1;min-width:140px;font-family:ui-monospace,monospace}
.sched-list,.sched-editor{flex:none;overflow:visible;border-top:1px solid var(--line);padding-top:16px;margin-top:6px;min-width:0}
.sched-row{display:grid;grid-template-columns:minmax(0,1fr) auto;gap:6px;padding:4px 0}
.sched-row>.outline{padding:0 10px;white-space:nowrap}
.sched-row-main small{overflow-wrap:anywhere;line-height:1.45}
.sched-dot{flex:0 0 8px}
.sched-editor>label{display:grid;gap:8px;color:var(--muted);font-size:13px;min-width:0}
.sched-editor>label.memory-toggle{display:flex;align-items:center}
.sched-editor input:not([type="checkbox"]),.sched-editor textarea{width:100%;min-width:0;border:1px solid var(--border);border-radius:10px;background:var(--inset);color:var(--ink);padding:10px 12px;font:inherit}
.sched-editor textarea{resize:vertical;min-height:100px;line-height:1.5}
.sched-editor input[type="checkbox"]{width:16px;height:16px;margin:0;accent-color:var(--accent)}
.sched-card>*{min-width:0;max-width:100%;flex:1 1 100px}
.sched-when>span{display:flex;flex-wrap:wrap;gap:6px;overflow-wrap:anywhere}
.sched-editor .dialog-actions{flex-wrap:wrap}
.login-chip,.sched-chip{width:100%;overflow-wrap:anywhere}
.message:has(>.sched-chip),.message:has(>.login-chip){flex-direction:column;gap:8px}
.message.spoken>.sched-chip,.message.spoken>.login-chip{grid-column:2}

251
apps/web/src/schedule.tsx Normal file
View File

@ -0,0 +1,251 @@
import { getLocale, t } from "./i18n";
export type CronFreq = "Every hour" | "Every day" | "Weekdays" | "Every week" | "Every month" | "Interval" | "Advanced";
export type CronUnit = "minutes" | "hours" | "days";
export type CronPreset = { freq: CronFreq; n: number; unit: CronUnit; time: string; cron: string };
export interface ScheduleItem {
id: string;
botId: string;
threadId?: string | null;
name: string;
cron: string;
timezone: string;
instructions: string;
enabled: boolean;
human: string;
lastRunAt?: string | null;
nextRunAt?: string | null;
}
export const CRON_FREQS: CronFreq[] = ["Every hour", "Every day", "Weekdays", "Every week", "Every month", "Interval", "Advanced"];
const TIMES = ["6:00 AM", "7:00 AM", "8:00 AM", "9:00 AM", "12:00 PM", "3:00 PM", "6:00 PM", "9:00 PM"];
const NUMBERS = [1, 2, 3, 5, 10, 15, 30, 45];
const TIMED: CronFreq[] = ["Every day", "Weekdays", "Every week", "Every month"];
export function defaultCronPreset(): CronPreset {
return { freq: "Every day", n: 3, unit: "minutes", time: "9:00 AM", cron: "" };
}
export function cronFromPreset(input: CronPreset): string {
if (input.freq === "Advanced") return input.cron.trim();
if (input.freq === "Every hour") return "0 * * * *";
if (input.freq === "Interval") {
if (!Number.isInteger(input.n) || input.n < 1 || input.n > 365) throw new Error(t("schedInvalidInterval"));
return `@every ${input.n}${({minutes:"m",hours:"h",days:"d"})[input.unit]}`;
}
const { hour, minute } = parseClock(input.time);
if (input.freq === "Weekdays") return `${minute} ${hour} * * 1-5`;
if (input.freq === "Every week") return `${minute} ${hour} * * 1`;
if (input.freq === "Every month") return `${minute} ${hour} 1 * *`;
return `${minute} ${hour} * * *`;
}
export function presetFromCron(cron: string): CronPreset {
const base = defaultCronPreset();
const interval = /^@every ([1-9]\d{0,2})([mhd])$/.exec(cron.trim());
if (interval && Number(interval[1]) <= 365) return {...base, freq:"Interval", n:Number(interval[1]), unit:({m:"minutes",h:"hours",d:"days"} as const)[interval[2] as "m"|"h"|"d"]};
const parts = cron.trim().split(/\s+/);
if (parts.length !== 5) return { ...base, freq: "Advanced", cron };
const [minute, hour, day, month, dow] = parts;
if (month !== "*") return { ...base, freq: "Advanced", cron };
if (minute === "0" && hour === "*" && day === "*" && dow === "*") return { ...base, freq: "Every hour" };
if (!/^\d+$/.test(minute) || !/^\d+$/.test(hour)) return { ...base, freq: "Advanced", cron };
if (Number(hour) > 23 || Number(minute) > 59) return {...base, freq:"Advanced", cron};
const time = formatClock(Number(hour), Number(minute));
if (day === "*" && dow === "1-5") return { ...base, freq: "Weekdays", time };
if (day === "*" && dow === "1") return { ...base, freq: "Every week", time };
if (day === "1" && dow === "*") return { ...base, freq: "Every month", time };
if (day === "*" && dow === "*") return { ...base, freq: "Every day", time };
return { ...base, freq: "Advanced", cron };
}
function parseClock(time: string): { hour: number; minute: number } {
const [rawH, rest] = time.split(":");
const minute = Number((rest ?? "00").slice(0, 2));
let hour = Number(rawH);
if (/pm/i.test(time) && hour < 12) hour += 12;
if (/am/i.test(time) && hour === 12) hour = 0;
return { hour, minute };
}
function formatClock(hour: number, minute: number): string {
const period = hour >= 12 ? "PM" : "AM";
const h12 = hour % 12 === 0 ? 12 : hour % 12;
return `${h12}:${String(minute).padStart(2, "0")} ${period}`;
}
export function clockLabel(time: string): string {
const { hour, minute } = parseClock(time);
if (getLocale() === "en") return formatClock(hour, minute);
return `${String(hour).padStart(2, "0")}:${String(minute).padStart(2, "0")}`;
}
export function describeCron(cron: string): string {
const preset = presetFromCron(cron);
const time = clockLabel(preset.time);
if (preset.freq === "Every hour") return t("schedEveryHour");
if (preset.freq === "Every day") return t("schedHumanDaily", { time });
if (preset.freq === "Weekdays") return t("schedHumanWeekdays", { time });
if (preset.freq === "Every week") return t("schedHumanWeekly", { time });
if (preset.freq === "Every month") return t("schedHumanMonthly", { time });
if (preset.freq === "Interval") {
const unit =
preset.unit === "hours" ? t("schedHours") : preset.unit === "days" ? t("schedDays") : t("schedMinutes");
return t("schedHumanInterval", { n: preset.n, unit });
}
return cron;
}
export function describeCronHuman(human: string): string {
const text = human.trim();
if (!text) return "";
if (text === "每小時" || text === "Every hour") return t("schedEveryHour");
if (text === "每分鐘" || text === "Every minute") return t("schedEveryMinute");
let match: RegExpExecArray | null;
if ((match = /^(?:每天|Daily at) (.+)$/.exec(text))) return t("schedHumanDaily", { time: clockLabel(match[1]) });
if ((match = /^(?:工作日|Weekdays at) (.+)$/.exec(text))) return t("schedHumanWeekdays", { time: clockLabel(match[1]) });
if ((match = /^(?:每週一|Mondays at) (.+)$/.exec(text))) return t("schedHumanWeekly", { time: clockLabel(match[1]) });
if ((match = /^(?:每月 1 日|Monthly on the 1st at) (.+)$/.exec(text))) {
return t("schedHumanMonthlyDay", { time: clockLabel(match[1]) });
}
if ((match = /^(?:每月|Monthly at) (.+)$/.exec(text))) return t("schedHumanMonthly", { time: clockLabel(match[1]) });
if ((match = /^(?:每隔 (\d+) 分鐘(固定間隔)|Every (\d+) minutes \(elapsed\))$/.exec(text))) {
return t("schedHumanElapsed", { n: match[1] || match[2] });
}
if ((match = /^(?:每 (\d+) 分鐘|Every (\d+) minutes)$/.exec(text))) {
return t("schedHumanEveryMinutes", { n: match[1] || match[2] });
}
if ((match = /^(?:每 (\d+) 小時|Every (\d+) hours)$/.exec(text))) {
return t("schedHumanEveryHours", { n: match[1] || match[2] });
}
if ((match = /^(?:日曆排程:|Calendar schedule: )(.+)$/.exec(text))) return t("schedCalendar", { expr: match[1] });
return text;
}
export function scheduleWhen(cron?: string, human?: string): string {
if (cron) return describeCron(cron);
return describeCronHuman(human || "");
}
function freqLabel(freq: CronFreq): string {
return ({
"Every hour": t("schedEveryHour"),
"Every day": t("schedEveryDay"),
"Weekdays": t("schedWeekdays"),
"Every week": t("schedEveryWeek"),
"Every month": t("schedEveryMonth"),
"Interval": t("schedInterval"),
"Advanced": t("schedAdvanced"),
})[freq];
}
export function ScheduleList({
items,
onCreate,
onOpen,
onRun,
runningId,
}: {
items: ScheduleItem[];
onCreate: () => void;
onOpen: (item: ScheduleItem) => void;
onRun: (item: ScheduleItem) => void;
runningId?: string | null;
}) {
return (
<div className="sched-list">
<div className="sched-list-head">
<span>{t("schedules")}</span>
<button type="button" className="icon-button" title={t("schedCreate")} onClick={onCreate}>+</button>
</div>
{items.length === 0 ? <p className="sched-empty">{t("schedEmpty")}</p> : items.map(item => (
<div className="sched-row" key={item.id}>
<button type="button" className="sched-row-main" onClick={() => onOpen(item)}>
<i className={`sched-dot ${item.enabled ? "on" : "off"}`} />
<span>
<strong>{item.name}</strong>
<small>{item.enabled ? describeCron(item.cron) : t("schedPaused")}</small>
</span>
</button>
<button type="button" className="outline" disabled={runningId === item.id} onClick={() => onRun(item)}>
{runningId === item.id ? t("schedRunning") : t("schedRunNow")}
</button>
</div>
))}
</div>
);
}
export function ScheduleEditor({
draft,
timezone,
saving,
error,
onChange,
onBack,
onSave,
onDelete,
}: {
draft: { name: string; instructions: string; enabled: boolean; preset: CronPreset };
timezone: string;
saving: boolean;
error: string | null;
onChange: (next: { name: string; instructions: string; enabled: boolean; preset: CronPreset }) => void;
onBack: () => void;
onSave: () => void;
onDelete?: () => void;
}) {
const preset = draft.preset;
const times = TIMES.includes(preset.time) ? TIMES : [...TIMES, preset.time];
const numbers = NUMBERS.includes(preset.n) ? NUMBERS : [...NUMBERS, preset.n].sort((a, b) => a - b);
function patchPreset(partial: Partial<CronPreset>) {
onChange({ ...draft, preset: { ...preset, ...partial } });
}
return (
<div className="sched-editor">
<div className="sched-editor-head">
<button type="button" className="outline" onClick={onBack}>{t("back")}</button>
<strong>{t("schedule")}</strong>
</div>
<label className="memory-toggle">
<input type="checkbox" checked={draft.enabled} onChange={e => onChange({ ...draft, enabled: e.target.checked })} />
{t("schedActive")}
</label>
<label>{t("name")}<input value={draft.name} onChange={e => onChange({ ...draft, name: e.target.value })} placeholder={t("schedNamePlaceholder")} /></label>
<label>{t("schedInstruction")}<textarea rows={4} value={draft.instructions} onChange={e => onChange({ ...draft, instructions: e.target.value })} placeholder={t("schedInstructionPlaceholder")} /></label>
<div className="sched-when">
<span>{t("schedWhen")} <small>{timezone}</small></span>
<div className="sched-card">
<select value={preset.freq} onChange={e => patchPreset({ freq: e.target.value as CronFreq })} aria-label={t("schedWhen")}>
{CRON_FREQS.map(freq => <option key={freq} value={freq}>{freqLabel(freq)}</option>)}
</select>
{preset.freq === "Interval" && <>
<select value={String(preset.n)} onChange={e => patchPreset({ n: Number(e.target.value) })}>
{numbers.map(n => <option key={n} value={n}>{n}</option>)}
</select>
<select value={preset.unit} onChange={e => patchPreset({ unit: e.target.value as CronUnit })}>
<option value="minutes">{t("schedMinutes")}</option>
<option value="hours">{t("schedHours")}</option>
<option value="days">{t("schedDays")}</option>
</select>
</>}
{TIMED.includes(preset.freq) && (
<select value={preset.time} onChange={e => patchPreset({ time: e.target.value })}>
{times.map(time => <option key={time} value={time}>{clockLabel(time)}</option>)}
</select>
)}
{preset.freq === "Advanced" && (
<input className="sched-cron" value={preset.cron} placeholder="0 9 * * 1-5" onChange={e => patchPreset({ cron: e.target.value })} />
)}
</div>
</div>
{preset.freq === "Interval" && <small className="memory-help">{t("schedIntervalHint")}</small>}
{error && <div className="pane-error">{error}</div>}
<div className="pane-actions">
{onDelete && <button type="button" className="danger" disabled={saving} onClick={onDelete}>{t("delete")}</button>}
<button type="button" className="primary" disabled={saving || !draft.name.trim() || !draft.instructions.trim() || (preset.freq === "Advanced" && !preset.cron.trim())} onClick={onSave}>{saving ? t("saving") : t("save")}</button>
</div>
</div>
);
}

File diff suppressed because one or more lines are too long

View File

@ -9,12 +9,13 @@ export interface Message { id:string; sessionId?:string; seq?:number; role:strin
export interface MessageFile { kind:"image"|"file"; name:string; mimeType?:string; size?:number }
export interface RoomMember { id:string; name:string; avatarColor:string; avatarShape:AvatarShape }
export interface Room { id:string; name:string; members:RoomMember[]; lastMessageAt:string|null; lastPreview:string|null; unreadCount:number }
export interface ComputerStatus { botId:string; mode:ComputerMode; state:ComputerState; controlHolder:"none"|"bot"|"user"; takeoverRequested:boolean; busyBotName:string|null; busySessionId:string|null; busyRunId:string|null; busyStep?:string|null; waitingRunId?:string|null; waitingSessionId?:string|null; queuedRuns?:number; display:string|null; profileMode:string; screenAvailable:boolean }
export interface ComputerStatus { botId:string; mode:ComputerMode; state:ComputerState; controlHolder:"none"|"bot"|"user"; takeoverRequested:boolean; busyBotName:string|null; busySessionId:string|null; busyRunId:string|null; busyStep?:string|null; usingComputer?:boolean; waitingRunId?:string|null; waitingSessionId?:string|null; queuedRuns?:number; display:string|null; profileMode:string; screenAvailable:boolean }
export interface PlaybookStep { do:string; expect?:string; note?:string }
export interface PlaybookInput { name:string; description?:string; example?:string }
export interface Playbook { name?:string; whenToUse?:string; intent?:string; inputs?:PlaybookInput[]; preconditions?:string[]; steps?:(PlaybookStep|string)[]; howToCheck?:string; whatToReturn?:string; cautions?:string[] }
export type TaughtSkillStatus = "recording"|"drafting"|"draft"|"saved"|"failed"|"cancelled";
export interface TaughtSkill { id:string; botId:string; threadId:string|null; name:string; goal:string; status:TaughtSkillStatus; playbook:Playbook; error:string|null; startedAt:string|null; expiresAt:string|null; stoppedAt:string|null; createdAt:string; updatedAt:string; eventCount:number; frameCount:number }
export interface FileSkill { name:string; description:string }
export interface MemoryItem { id:string; sessionId:string|null; sourceRunId:string|null; sourceMessageId:string|null; content:string; importance:number; revision:number; createdAt:string; updatedAt:string }
export type McpTransport = "stdio" | "http" | "sse";
export interface McpTool { name:string; exposedName:string; description:string }
@ -22,6 +23,22 @@ export interface McpServer { id:string; name:string; transport:McpTransport; com
export interface McpSecretField { name:string; required:boolean; secret:boolean; hint:string }
export interface McpCatalogEntry { id:string; title:string; description:string; transport:McpTransport; command:string|null; args:string[]; url:string|null; envKeys:McpSecretField[]; headerKeys:McpSecretField[]; source:"featured"|"registry"; remote:boolean }
export type ModelProviderId = "xai" | "opencode-go" | "openai-compatible";
export type VoiceProviderId = "xai" | "openai" | "scripted";
export interface VoiceSettings {
enabled: boolean;
provider: VoiceProviderId;
modelId: string;
voiceId: string;
ready: boolean;
missing?: string | null;
apiKeySet: boolean;
envKeySet: boolean;
envKeyName: string;
reusesTextKey: boolean;
providers: { id: VoiceProviderId; name: string; envKeyName: string }[];
models: { id: string; name: string }[];
voices: { id: string; name: string }[];
}
export interface WorkspaceProvider { id:ModelProviderId; name:string; needsBaseUrl:boolean; needsKey:boolean; defaultBaseUrl:string|null; defaultModel:string|null }
export interface WorkspaceModel { id:string; name:string }
export interface WorkspaceSettings { provider:ModelProviderId; modelId:string; baseUrl:string; apiKeySet:boolean; envKeySet:boolean; envKeyName:string; providers:WorkspaceProvider[]; models:WorkspaceModel[] }

View File

@ -0,0 +1,7 @@
import { lazy, Suspense, type ComponentProps } from "react";
import type Animation from "react-useanimations";
const AnimationPlayer = lazy(() => import("react-useanimations"));
export default function UseAnimations(props: ComponentProps<typeof Animation>) {
const size=props.size ?? 24;
return <Suspense fallback={<span aria-hidden="true" style={{display:"inline-block",width:size,height:size,flexShrink:0}}/>}><AnimationPlayer {...props}/></Suspense>;
}

View File

@ -0,0 +1,144 @@
import { useEffect, useState } from "react";
import { api } from "./api";
import { t } from "./i18n";
import { X } from "./animated-icons";
import type { VoiceProviderId, VoiceSettings } from "./types";
export type { VoiceProviderId, VoiceSettings };
export function VoiceSettingsDialog({ close }: { close: () => void }) {
const [settings, setSettings] = useState<VoiceSettings | null>(null);
const [enabled, setEnabled] = useState(false);
const [provider, setProvider] = useState<VoiceProviderId>("xai");
const [modelId, setModelId] = useState("");
const [voiceId, setVoiceId] = useState("");
const [apiKey, setApiKey] = useState("");
const [clearKey, setClearKey] = useState(false);
const [busy, setBusy] = useState(false);
const [error, setError] = useState("");
useEffect(() => {
api<VoiceSettings>("/api/voice/settings")
.then((value) => {
setSettings(value);
setEnabled(value.enabled);
setProvider(value.provider);
setModelId(value.modelId);
setVoiceId(value.voiceId);
})
.catch((err) => setError(err instanceof Error ? err.message : t("loadFailed")));
}, []);
const current = settings?.providers.find((item) => item.id === provider);
const models = (settings?.provider === provider ? settings.models : null)
?? (provider === "openai"
? [{ id: "gpt-realtime", name: "GPT Realtime" }]
: [{ id: "grok-voice-latest", name: "Grok Voice Latest" }, { id: "grok-voice-think-fast-2.0", name: "Grok Voice Think Fast 2.0" }]);
const voices = (settings?.provider === provider ? settings.voices : null)
?? (provider === "openai"
? [{ id: "marin", name: "Marin" }, { id: "alloy", name: "Alloy" }, { id: "verse", name: "Verse" }]
: [{ id: "eve", name: "Eve" }, { id: "ara", name: "Ara" }, { id: "leo", name: "Leo" }, { id: "rex", name: "Rex" }, { id: "sal", name: "Sal" }]);
function pickProvider(id: VoiceProviderId) {
setProvider(id);
setError("");
if (id === "openai") {
setModelId("gpt-realtime");
setVoiceId("marin");
return;
}
setModelId("grok-voice-latest");
setVoiceId("eve");
}
return (
<div className="modal-backdrop">
<form
className="dialog settings-dialog"
onSubmit={async (event) => {
event.preventDefault();
setBusy(true);
setError("");
try {
await api("/api/voice/settings", {
method: "PATCH",
body: JSON.stringify({
enabled,
provider,
modelId: modelId.trim(),
voiceId: voiceId.trim(),
apiKey: clearKey ? "" : apiKey.trim() || null,
clearApiKey: clearKey,
}),
});
close();
} catch (err) {
setError(err instanceof Error ? err.message : t("settingsFailed"));
} finally {
setBusy(false);
}
}}
>
<div className="dialog-title">
<h2>{t("voiceSettings")}</h2>
<button type="button" onClick={close}><X /></button>
</div>
<p className="dialog-lead">{t("voiceSettingsHint")}</p>
<label className="memory-toggle">
<input type="checkbox" role="switch" checked={enabled} disabled={!settings || busy} onChange={(event) => setEnabled(event.target.checked)} />
{t("voiceEnabled")}
</label>
<p className="dialog-lead">{t("voiceEnabledHint")}</p>
{enabled ? <>
<fieldset className="provider-fieldset">
<legend>{t("voiceProvider")}</legend>
<div className="provider-grid">
{(settings?.providers || [
{ id: "xai" as const, name: t("providerXai"), envKeyName: "XAI_API_KEY" },
{ id: "openai" as const, name: t("providerOpenai"), envKeyName: "OPENAI_API_KEY" },
]).map((item) => (
<button type="button" key={item.id} className={provider === item.id ? "picked" : ""} onClick={() => pickProvider(item.id)}>
{item.id === "xai" ? t("providerXai") : item.id === "openai" ? t("providerOpenai") : item.name}
</button>
))}
</div>
<p className="dialog-lead">{provider === "openai" ? t("providerOpenaiVoiceHint") : t("providerXaiVoiceHint")}</p>
</fieldset>
<label>
{t("voiceModel")}
{models.length ? (
<select value={models.some((item) => item.id === modelId) ? modelId : modelId} onChange={(event) => setModelId(event.target.value)}>
{models.map((item) => <option key={item.id} value={item.id}>{item.name}</option>)}
{!models.some((item) => item.id === modelId) && modelId ? <option value={modelId}>{modelId}</option> : null}
</select>
) : (
<input value={modelId} onChange={(event) => setModelId(event.target.value)} />
)}
</label>
<label>
{t("voiceName")}
{voices.length ? (
<select value={voiceId} onChange={(event) => setVoiceId(event.target.value)}>
{voices.map((item) => <option key={item.id} value={item.id}>{item.name}</option>)}
</select>
) : (
<input value={voiceId} onChange={(event) => setVoiceId(event.target.value)} />
)}
</label>
<label>
{t("apiKey")}
<input type="password" autoComplete="off" value={apiKey} onChange={(event) => { setApiKey(event.target.value); setClearKey(false); }} placeholder={settings?.apiKeySet ? t("apiKeyStored") : t("apiKeyPlaceholder")} />
</label>
{settings?.reusesTextKey && !apiKey && !clearKey ? <p className="dialog-lead">{t("voiceReusesTextKey")}</p> : null}
{settings?.envKeySet && !apiKey && !clearKey ? <p className="dialog-lead">{t("usingEnvKey", { name: current?.envKeyName || settings.envKeyName })}</p> : null}
{settings?.apiKeySet ? <label className="memory-toggle"><input type="checkbox" checked={clearKey} onChange={(event) => setClearKey(event.target.checked)} /> {t("clearApiKey")}</label> : null}
</> : null}
{error ? <div className="pane-error">{error}</div> : null}
<div className="dialog-actions">
<button type="button" className="outline" onClick={close}>{t("cancel")}</button>
<button className="primary" disabled={busy || !settings || !modelId.trim() || !voiceId.trim()}>{busy ? t("saving") : t("save")}</button>
</div>
</form>
</div>
);
}

View File

@ -1,3 +1,4 @@
import { readFileSync } from "node:fs";
import { defineConfig } from "vite";
import react from "@vitejs/plugin-react";
export default defineConfig({plugins:[react()],build:{outDir:"dist",emptyOutDir:true},server:{port:5173,proxy:{"/api":"http://127.0.0.1:3101","/view":{target:"http://127.0.0.1:3101",ws:true}}}});
export default defineConfig({plugins:[react(),{name:"desktop-viewer",generateBundle(){this.emitFile({type:"asset",fileName:"vnc.html",source:readFileSync(new URL("./vnc.html",import.meta.url),"utf8")})}}],build:{outDir:"dist",emptyOutDir:true},server:{port:5173,proxy:{"/api":{target:"http://127.0.0.1:3101",ws:true},"/view":{target:"http://127.0.0.1:3101",ws:true}}}});

View File

@ -2,6 +2,7 @@
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>LazyBoy desktop</title>
<style>
html, body, #screen {
@ -12,18 +13,28 @@
overflow: hidden;
}
#status {
position: absolute;
left: 8px;
top: 8px;
z-index: 2;
padding: 4px 8px;
border-radius: 6px;
background: rgba(0, 0, 0, 0.45);
color: #fff;
font: 12px/1.3 ui-sans-serif, system-ui, sans-serif;
pointer-events: none;
display: none;
}
body { display:flex; flex-direction:column; }
#screen { flex:1; min-height:0; height:auto; }
#screen canvas { cursor: default; }
#mobile-controls { display:none; flex:none; padding:6px 8px max(6px,env(safe-area-inset-bottom));
background:#172033; color:#f8fafc; font:12px system-ui; border-top:1px solid #334155; }
#mobile-controls .buttons { display:flex; gap:5px; overflow-x:auto; }
#mobile-controls button { flex:1 0 auto; min-height:44px; padding:8px 10px;
border:1px solid #64748b; border-radius:8px; background:#25334a; color:inherit; font:inherit; touch-action:manipulation; }
#mobile-controls button[aria-pressed="true"] { background:#155e75; border-color:#67e8f9; }
#mobile-controls button:disabled { opacity:.4; }
#pointer-help { margin:5px 0 0; line-height:1.4; }
#mobile-cursor { display:none; position:fixed; width:18px; height:18px; border:2px solid white;
border-radius:50%; box-shadow:0 0 0 1px #000,0 1px 5px #000; transform:translate(-50%,-50%);
pointer-events:none; z-index:5; }
#mobile-cursor::after { content:""; position:absolute; width:4px; height:4px; background:#fff;
border-radius:50%; left:7px; top:7px; }
@media(any-pointer:coarse) { #mobile-controls { display:block; } }
/* Keep a real editable element on screen for iOS/Android keyboards. */
#mobile-keyboard { position:fixed; bottom:0; left:0; width:1px; height:1px;
padding:0; border:0; opacity:.01; font-size:16px; pointer-events:none; }
</style>
<script type="module">
import RFB from "./core/rfb.js";
@ -57,20 +68,225 @@
let rfb = null;
let reconnectTimer = null;
const parentOrigin = window.location.origin;
const keyboard = document.getElementById("mobile-keyboard");
const sentinel = "\u200b";
let composing = false;
let touchStart = null;
const controls = document.getElementById("mobile-controls");
const cursor = document.getElementById("mobile-cursor");
const modeButton = document.getElementById("pointer-mode");
const dragButton = document.getElementById("pointer-drag");
const pointerHelp = document.getElementById("pointer-help");
let trackpad = false;
let dragging = false;
let gesture = null;
let pointer = { x: .5, y: .5 };
function pointerGeometry() {
const canvas = document.querySelector("#screen canvas");
const rect = canvas?.getBoundingClientRect();
return rect && rect.width > 0 && rect.height > 0 ? rect : null;
}
function paintPointer() {
const rect = pointerGeometry();
if (!cursor?.style) return;
cursor.style.display = trackpad && rect && rfb && !rfb.viewOnly ? "block" : "none";
if (rect) {
cursor.style.left = `${rect.left + pointer.x * rect.width}px`;
cursor.style.top = `${rect.top + pointer.y * rect.height}px`;
}
}
// Pointer adapter for the pinned noVNC 1.7.0. These methods take scaled
// canvas coordinates and preserve noVNC's throttling/protocol handling.
function sendPointer(mask = null) {
const rect = pointerGeometry();
if (!rect || !rfb || rfb.viewOnly) return;
const x = Math.min(rect.width - 1, Math.max(0, pointer.x * rect.width));
const y = Math.min(rect.height - 1, Math.max(0, pointer.y * rect.height));
if (mask === null) rfb._handleMouseMove(x, y);
else rfb._handleMouseButton(x, y, mask);
paintPointer();
}
function releaseDrag() {
if (dragging) sendPointer(0);
dragging = false;
dragButton?.setAttribute?.("aria-pressed", "false");
}
function pointerClick(mask) {
releaseDrag();
sendPointer(mask);
sendPointer(0);
}
function scrollPointer(deltaX, deltaY) {
const rect = pointerGeometry();
if (!rect || !rfb || rfb.viewOnly) return;
rfb._handleWheel({ clientX:rect.left + pointer.x * rect.width,
clientY:rect.top + pointer.y * rect.height, deltaX, deltaY, deltaMode:0,
buttons:0, preventDefault(){}, stopPropagation(){} });
}
function updatePointerControls() {
modeButton?.setAttribute?.("aria-pressed", String(trackpad));
if (modeButton) modeButton.textContent = trackpad ? "觸控板" : "直接點選";
if (pointerHelp) pointerHelp.textContent = trackpad
? "滑動移游標・輕點左鍵・雙指捲動/輕點右鍵・拖曳按完再按一次放開"
: "點畫面定位並開鍵盤・長按右鍵・精準操作可切換觸控板";
for (const button of controls?.querySelectorAll?.("button[data-action]") || []) {
button.disabled = !rfb || rfb.viewOnly;
}
paintPointer();
}
function showKeyboard() {
if (!rfb || rfb.viewOnly) return;
rfb.focusOnClick = false;
keyboard.focus({ preventScroll: true });
keyboard.setSelectionRange(keyboard.value.length, keyboard.value.length);
}
window.addEventListener("click", event => {
const button = event.target.closest?.("#mobile-controls button");
if (!button) return;
const action = button.dataset.action;
if (action === "mode") {
releaseDrag(); gesture = null; touchStart = null;
trackpad = !trackpad; keyboard.blur(); updatePointerControls(); return;
}
if (!rfb || rfb.viewOnly) return;
if (action === "keyboard") { releaseDrag(); showKeyboard(); return; }
if (action === "left") pointerClick(1);
if (action === "right") pointerClick(4);
if (action === "up") { releaseDrag(); scrollPointer(0, -100); }
if (action === "down") { releaseDrag(); scrollPointer(0, 100); }
if (action === "drag") {
if (!trackpad) { trackpad = true; keyboard.blur(); }
dragging = !dragging; sendPointer(dragging ? 1 : 0);
dragButton.setAttribute("aria-pressed", String(dragging));
updatePointerControls();
}
});
function stopTouch(event) { event.preventDefault(); event.stopImmediatePropagation(); }
function touchCenter(touches) {
const points = Array.from(touches);
return { x:points.reduce((sum,p)=>sum+p.clientX,0)/points.length,
y:points.reduce((sum,p)=>sum+p.clientY,0)/points.length };
}
function startPointerTouch(event) {
if (!trackpad || !event.target.closest?.("#screen")) return false;
stopTouch(event);
if (!rfb || rfb.viewOnly) {
window.parent.postMessage({ type:"lazyboy-request-control" }, parentOrigin);
return true;
}
const center = touchCenter(event.touches);
if (event.touches.length > 1) releaseDrag();
gesture = { last:center, start:center, fingers:event.touches.length,
moved:false, time:Date.now() };
return true;
}
function movePointerTouch(event) {
if (!trackpad || !gesture) return false;
stopTouch(event);
if (!rfb || rfb.viewOnly) { gesture = null; return true; }
const center = touchCenter(event.touches);
if (event.touches.length !== gesture.fingers) {
gesture.last = center; gesture.moved = true; return true;
}
const dx = center.x - gesture.last.x, dy = center.y - gesture.last.y;
gesture.last = center;
if (Math.hypot(center.x - gesture.start.x, center.y - gesture.start.y) > 5) gesture.moved = true;
if (gesture.fingers === 2) { scrollPointer(-dx, -dy); return true; }
const rect = pointerGeometry();
if (rect && gesture.fingers === 1 && gesture.moved) {
pointer.x = Math.max(0, Math.min(1, pointer.x + dx / rect.width));
pointer.y = Math.max(0, Math.min(1, pointer.y + dy / rect.height));
sendPointer();
}
return true;
}
function endPointerTouch(event) {
if (!trackpad || !gesture) return false;
stopTouch(event);
if (event.touches.length) return true;
if (!gesture.moved && Date.now() - gesture.time < 450 && !dragging) {
if (gesture.fingers === 1) pointerClick(1);
if (gesture.fingers === 2) pointerClick(4);
}
gesture = null;
return true;
}
window.addEventListener("blur", () => { gesture = null; releaseDrag(); });
window.addEventListener("pagehide", () => { gesture = null; releaseDrag(); });
function resetKeyboard() {
keyboard.value = sentinel;
composing = false;
}
function mobileKey(key) {
if (!rfb || rfb.viewOnly) return;
window.parent.postMessage({ type: "lazyboy-mobile-key", key }, parentOrigin);
}
function commitKeyboard() {
if (composing) return;
const value = keyboard.value;
if (!rfb || rfb.viewOnly) { resetKeyboard(); return; }
if (!value) mobileKey("BackSpace");
else {
const text = value.startsWith(sentinel) ? value.slice(1) : value;
if (text) pasteIntoDesktop(text);
}
resetKeyboard();
}
resetKeyboard();
window.addEventListener("compositionstart", event => {
if (event.target === keyboard) composing = true;
});
window.addEventListener("compositionend", event => {
if (event.target !== keyboard) return;
composing = false;
commitKeyboard();
});
window.addEventListener("input", event => {
if (event.target === keyboard && !event.isComposing) commitKeyboard();
});
window.addEventListener("beforeinput", event => {
if (event.target !== keyboard || composing || event.isComposing) return;
if (event.inputType === "insertLineBreak" || event.inputType === "insertParagraph") {
event.preventDefault(); mobileKey("Return"); resetKeyboard();
}
});
// Focus synchronously in the user's tap, before iOS loses activation.
// VNC pixels cannot reveal whether the remote target is a text field.
window.addEventListener("touchstart", event => {
if (event.target === keyboard) return;
if (startPointerTouch(event)) { touchStart = null; return; }
const point = event.touches[0];
const rect = pointerGeometry();
if (point && rect && event.target.closest?.("#screen")) {
pointer.x = Math.max(0, Math.min(1, (point.clientX - rect.left) / rect.width));
pointer.y = Math.max(0, Math.min(1, (point.clientY - rect.top) / rect.height));
}
touchStart = event.touches.length === 1 && event.target.closest?.("#screen")
? { x: point.clientX, y: point.clientY } : null;
if (rfb) rfb.focusOnClick = false;
}, { passive: false, capture: true });
window.addEventListener("touchmove", event => {
if (movePointerTouch(event)) return;
const point = event.touches[0];
if (touchStart && (!point || event.touches.length !== 1 ||
Math.hypot(point.clientX - touchStart.x, point.clientY - touchStart.y) > 10)) touchStart = null;
}, { passive: false, capture: true });
window.addEventListener("touchcancel", () => { touchStart = null; gesture = null; releaseDrag(); }, true);
window.addEventListener("touchend", event => {
if (endPointerTouch(event)) return;
const tapped = touchStart && event.touches.length === 0;
touchStart = null;
if (!tapped || !rfb || rfb.viewOnly) return;
keyboard.focus({ preventScroll: true });
keyboard.setSelectionRange(keyboard.value.length, keyboard.value.length);
}, { passive: false, capture: true });
function pasteIntoDesktop(text) {
if (!rfb || rfb.viewOnly || !text) return;
try { rfb.clipboardPasteFrom(text); } catch (_) {}
// x11vnc applies ClientCutText asynchronously. Give X11 a moment before
// sending Ctrl+V so pastes are reliable for Chromium and terminals.
setTimeout(() => {
try {
rfb.sendKey(0xffe3, "ControlLeft", true);
rfb.sendKey(0x0076, "KeyV", true);
rfb.sendKey(0x0076, "KeyV", false);
rfb.sendKey(0xffe3, "ControlLeft", false);
} catch (_) {}
}, 100);
window.parent.postMessage({type:"lazyboy-paste-text", text}, parentOrigin);
}
function pinTaskbar() {
@ -87,23 +303,35 @@
}
}
function applyViewOnly(value) {
if (!rfb) return;
if (value) { releaseDrag(); gesture = null; }
rfb.viewOnly = Boolean(value);
updatePointerControls();
if (rfb.viewOnly) { keyboard.blur(); resetKeyboard(); }
}
function connect() {
setStatus("Connecting to desktop…");
statusEl.style.display = "block";
window.parent.postMessage({ type: "lazyboy-desktop-lost" }, parentOrigin);
rfb = new RFB(document.getElementById("screen"), url);
rfb.viewOnly = flag("view_only", false);
rfb.viewOnly = flag("view_only", true);
rfb.scaleViewport = true;
rfb.clipViewport = false;
rfb.background = "#0f172a";
pinTaskbar();
updatePointerControls();
rfb.addEventListener("connect", () => {
pinTaskbar();
setStatus(rfb.viewOnly ? "View only — click to take control" : "Click the desktop");
updatePointerControls();
try { rfb.focus(); } catch (_) {}
setTimeout(() => { statusEl.style.display = "none"; }, 2500);
window.parent.postMessage({ type: "lazyboy-desktop-ready" }, parentOrigin);
});
rfb.addEventListener("disconnect", (event) => {
statusEl.style.display = "block";
releaseDrag(); gesture = null;
applyViewOnly(true);
keyboard.blur(); resetKeyboard();
window.parent.postMessage({ type: "lazyboy-desktop-lost" }, parentOrigin);
const clean = event && event.detail && event.detail.clean;
setStatus(clean ? "Disconnected — retrying" : "Desktop connection lost — retrying");
if (reconnectTimer) clearTimeout(reconnectTimer);
@ -117,10 +345,39 @@
});
}
// Capture before noVNC forwards the shortcut; otherwise stale remote
// clipboard contents can be pasted once before the local text arrives.
window.addEventListener("keydown", async (event) => {
if (event.target === keyboard) {
if (event.isComposing || composing || event.keyCode === 229) return;
const keys = { Enter: "Return", Backspace: "BackSpace", Tab: "Tab", Escape: "Escape",
ArrowLeft: "Left", ArrowRight: "Right", ArrowUp: "Up", ArrowDown: "Down" };
if (keys[event.key]) {
event.preventDefault(); event.stopImmediatePropagation();
mobileKey(keys[event.key]); resetKeyboard();
}
return;
}
if (rfb && !rfb.viewOnly && event.metaKey && event.code === "KeyC") {
event.preventDefault(); event.stopImmediatePropagation();
window.parent.postMessage({type:"lazyboy-copy-request"},parentOrigin); return;
}
if (!rfb || rfb.viewOnly || !(event.ctrlKey || event.metaKey) || event.altKey || event.code !== "KeyV") return;
event.preventDefault();
event.stopImmediatePropagation();
const target = rfb;
try {
const text = await navigator.clipboard.readText();
if (rfb === target && !target.viewOnly) pasteIntoDesktop(text);
} catch (_) {
window.parent.postMessage({ type: "lazyboy-paste-request" }, parentOrigin);
}
}, true);
connect();
window.addEventListener("resize", pinTaskbar);
window.addEventListener("pointerdown", () => {
try { if (rfb) rfb.focus(); } catch (_) {}
window.addEventListener("resize", () => { pinTaskbar(); paintPointer(); });
window.addEventListener("pointerdown", (event) => {
if (event.target === keyboard || event.target.closest?.("#mobile-controls")) return;
try { if (rfb && event.pointerType !== "touch") { rfb.focusOnClick = true; rfb.focus(); } } catch (_) {}
if (rfb && rfb.viewOnly) {
window.parent.postMessage({ type: "lazyboy-request-control" }, parentOrigin);
}
@ -131,16 +388,37 @@
if (!text) return;
event.preventDefault();
pasteIntoDesktop(text);
if (event.target === keyboard) resetKeyboard();
});
window.addEventListener("message", (event) => {
if (event.origin !== parentOrigin) return;
if (!event.data || event.data.type !== "lazyboy-host-clipboard") return;
if (event.origin !== parentOrigin || event.source !== window.parent) return;
if (!event.data) return;
if (event.data.type === "lazyboy-view-only") {
applyViewOnly(event.data.viewOnly);
return;
}
if (event.data.type !== "lazyboy-host-clipboard") return;
pasteIntoDesktop(String(event.data.text || ""));
});
</script>
</head>
<body>
<div id="status">Loading desktop…</div>
<div id="status" hidden>Loading desktop…</div>
<div id="screen"></div>
<div id="mobile-cursor" aria-hidden="true"></div>
<nav id="mobile-controls" aria-label="遠端滑鼠">
<div class="buttons">
<button id="pointer-mode" data-action="mode" aria-pressed="false" title="切換直接點選與觸控板">直接點選</button>
<button data-action="left">左鍵</button>
<button data-action="right">右鍵</button>
<button id="pointer-drag" data-action="drag" aria-pressed="false">拖曳</button>
<button data-action="up" aria-label="向上捲動"></button>
<button data-action="down" aria-label="向下捲動"></button>
<button data-action="keyboard">鍵盤</button>
</div>
<p id="pointer-help"></p>
</nav>
<textarea id="mobile-keyboard" aria-label="Remote desktop keyboard" tabindex="-1"
autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"></textarea>
</body>
</html>

View File

@ -32,6 +32,12 @@ tokio-tungstenite.workspace = true
futures-util = "0.3"
http-body-util = "0.1"
dotenvy = "0.15"
fastembed = "6.0.2"
fastembed = { version = "6.0.2", default-features = false, features = ["hf-hub-rustls-tls", "ort-load-dynamic"] }
rmcp = { version = "3.2", default-features = false, features = ["client", "transport-child-process", "transport-streamable-http-client-reqwest"] }
http = "1"
aes-gcm = "0.10"
cron = "0.15"
chrono-tz = "0.10"
rand = "0.8"
cap-std = "3"

View File

@ -5,7 +5,7 @@
//! mounted home and delete anything older than [`INBOX_TTL`].
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime};
use std::time::Duration;
use base64::Engine;
use lazyboy_contracts::SessionAttachment;
@ -13,7 +13,6 @@ use lazyboy_control::resolve_bot_workspace_path;
use rig_core::completion::message::{ImageDetail, ImageMediaType, UserContent};
use serde::Serialize;
use serde_json::{Value, json};
use tokio::fs;
use crate::computer;
use crate::db::{Actor, parse_mode};
@ -41,6 +40,7 @@ pub struct StoredAttachment {
#[derive(Debug, Clone)]
pub struct DecodedAttachment {
pub name: String,
pub stored_name: String,
pub mime_type: String,
pub bytes: Vec<u8>,
}
@ -68,8 +68,11 @@ pub fn decode_incoming(items: &[IncomingAttachment]) -> Result<Vec<DecodedAttach
if total > MAX_TOTAL_BYTES {
return Err("附件合計超過 20 MB".into());
}
let (stem, ext) = split_ext(&name);
let stored_name = format!("{stem}-{}{ext}", uuid::Uuid::new_v4());
out.push(DecodedAttachment {
name,
stored_name,
mime_type: mime,
bytes,
});
@ -86,7 +89,7 @@ pub fn stored_blocks(files: &[DecodedAttachment]) -> Vec<Value> {
"name": file.name,
"mimeType": file.mime_type,
"size": file.bytes.len(),
"path": format!("inbox/{}", file.name),
"path": format!("inbox/{}", file.stored_name),
})
})
.collect()
@ -117,15 +120,30 @@ pub async fn stage_for_bots(
let Some(dir) = inbox_dir_for_bot(state, actor, bot_id).await? else {
continue;
};
fs::create_dir_all(&dir)
.await
.map_err(|error| error.to_string())?;
let root = PathBuf::from(&state.data_dir).join("homes");
let relative = dir
.strip_prefix(&root)
.map_err(|_| "invalid inbox root")?
.to_path_buf();
let files = files.to_vec();
tokio::task::spawn_blocking(move || -> Result<(), String> {
use std::io::Write;
let root = cap_std::fs::Dir::open_ambient_dir(root, cap_std::ambient_authority())
.map_err(|e| e.to_string())?;
root.create_dir_all(&relative).map_err(|e| e.to_string())?;
let inbox = root.open_dir(&relative).map_err(|e| e.to_string())?;
for file in files {
let path = unique_path(&dir, &file.name).await;
fs::write(&path, &file.bytes)
.await
.map_err(|error| error.to_string())?;
let mut opts = cap_std::fs::OpenOptions::new();
opts.write(true).create_new(true);
let mut output = inbox
.open_with(&file.stored_name, &opts)
.map_err(|e| e.to_string())?;
output.write_all(&file.bytes).map_err(|e| e.to_string())?;
}
Ok(())
})
.await
.map_err(|e| e.to_string())??;
}
Ok(())
}
@ -151,8 +169,8 @@ pub async fn llm_parts(
}
for (stored, bytes) in files {
notes.push(format!(
"- {} ({}, {} bytes) at inbox/{}",
stored.name, stored.mime_type, stored.size, stored.name
"- {} ({}, {} bytes) at {}",
stored.name, stored.mime_type, stored.size, stored.path
));
if is_image(&stored.mime_type) {
if vision {
@ -189,21 +207,52 @@ pub async fn llm_parts(
pub async fn sweep_all_inboxes(data_dir: &str) {
let homes = PathBuf::from(data_dir).join("homes");
let Ok(mut spaces) = fs::read_dir(&homes).await else {
let _ = tokio::task::spawn_blocking(move || {
let Ok(root) = cap_std::fs::Dir::open_ambient_dir(homes, cap_std::ambient_authority())
else {
return;
};
while let Ok(Some(entry)) = spaces.next_entry().await {
let path = entry.path();
if !path.is_dir() {
continue;
}
sweep_dir(&path.join("inbox")).await;
let bots = path.join("bots");
let Ok(mut bots_dir) = fs::read_dir(&bots).await else {
let Ok(entries) = root.entries() else {
return;
};
for entry in entries.flatten() {
let Ok(home) = root.open_dir(entry.file_name()) else {
continue;
};
while let Ok(Some(bot)) = bots_dir.next_entry().await {
sweep_dir(&bot.path().join("inbox")).await;
sweep_cap_inbox(&home, "inbox");
if let Ok(bots) = home.open_dir("bots") {
if let Ok(entries) = bots.entries() {
for bot in entries.flatten() {
if let Ok(dir) = bots.open_dir(bot.file_name()) {
sweep_cap_inbox(&dir, "inbox");
}
}
}
}
}
})
.await;
}
fn sweep_cap_inbox(root: &cap_std::fs::Dir, path: &str) {
let Ok(dir) = root.open_dir(path) else {
return;
};
let Ok(entries) = dir.entries() else {
return;
};
for entry in entries.flatten() {
let Ok(meta) = entry.metadata() else {
continue;
};
if meta.is_file()
&& meta
.modified()
.ok()
.and_then(|t| t.into_std().elapsed().ok())
.is_some_and(|age| age > INBOX_TTL)
{
let _ = dir.remove_file(entry.file_name());
}
}
}
@ -256,8 +305,31 @@ async fn load_from_blocks(
let Some(file) = parse_stored(block) else {
continue;
};
let path = dir.join(&file.name);
let bytes = fs::read(&path).await.ok();
let root = PathBuf::from(&state.data_dir).join("homes");
let stored = file.path.strip_prefix("inbox/").unwrap_or(&file.name);
let relative = dir.strip_prefix(&root).ok().map(|p| p.join(stored));
let bytes = if let Some(relative) = relative {
tokio::task::spawn_blocking(move || {
use std::io::Read;
let root =
cap_std::fs::Dir::open_ambient_dir(root, cap_std::ambient_authority()).ok()?;
let input = root.open(relative).ok()?;
if !input.metadata().ok()?.is_file() {
return None;
}
let mut bytes = Vec::new();
input
.take(MAX_BYTES as u64 + 1)
.read_to_end(&mut bytes)
.ok()?;
(bytes.len() <= MAX_BYTES).then_some(bytes)
})
.await
.ok()
.flatten()
} else {
None
};
out.push((file, bytes));
}
out
@ -269,6 +341,23 @@ fn parse_stored(value: &Value) -> Option<StoredAttachment> {
return None;
}
let name = value.get("name").and_then(Value::as_str)?;
if name.is_empty()
|| name == "."
|| name == ".."
|| name.contains(['/', '\\'])
|| name.chars().any(char::is_control)
{
return None;
}
let path = value.get("path").and_then(Value::as_str).unwrap_or("");
if !path.is_empty() {
let Some(stored) = path.strip_prefix("inbox/") else {
return None;
};
if stored.is_empty() || stored == "." || stored == ".." || stored.contains(['/', '\\']) {
return None;
}
}
Some(StoredAttachment {
kind: if kind == "image" { "image" } else { "file" },
name: name.to_string(),
@ -286,41 +375,6 @@ fn parse_stored(value: &Value) -> Option<StoredAttachment> {
})
}
async fn sweep_dir(dir: &Path) {
let Ok(mut entries) = fs::read_dir(dir).await else {
return;
};
let cutoff = SystemTime::now() - INBOX_TTL;
while let Ok(Some(entry)) = entries.next_entry().await {
let path = entry.path();
let Ok(meta) = fs::metadata(&path).await else {
continue;
};
if !meta.is_file() {
continue;
}
let modified = meta.modified().unwrap_or(SystemTime::UNIX_EPOCH);
if modified < cutoff {
let _ = fs::remove_file(&path).await;
}
}
}
async fn unique_path(dir: &Path, name: &str) -> PathBuf {
let candidate = dir.join(name);
if fs::metadata(&candidate).await.is_err() {
return candidate;
}
let (stem, ext) = split_ext(name);
for n in 2..1000 {
let next = dir.join(format!("{stem}-{n}{ext}"));
if fs::metadata(&next).await.is_err() {
return next;
}
}
dir.join(format!("{stem}-{}.bin", uuid::Uuid::new_v4()))
}
fn split_ext(name: &str) -> (String, String) {
match name.rfind('.') {
Some(index) if index > 0 => (name[..index].to_string(), name[index..].to_string()),
@ -459,6 +513,7 @@ mod tests {
fn stored_blocks_drop_bytes() {
let files = [DecodedAttachment {
name: "a.png".into(),
stored_name: "a.png".into(),
mime_type: "image/png".into(),
bytes: vec![1, 2, 3],
}];
@ -479,3 +534,30 @@ mod tests {
assert!(decode_incoming(&many).is_err());
}
}
#[cfg(test)]
mod boundary_tests {
use super::*;
#[test]
fn stored_metadata_cannot_escape_inbox() {
for name in ["../secret", "/etc/passwd", "..", "x\\secret"] {
assert!(parse_stored(&json!({"kind":"file","name":name})).is_none());
}
}
#[cfg(unix)]
#[test]
fn symlink_from_computer_home_cannot_read_host_secret() {
let base = std::env::temp_dir().join(format!("lazyboy-boundary-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(base.join("homes")).unwrap();
std::fs::write(base.join("secret"), b"private").unwrap();
std::os::unix::fs::symlink(base.join("secret"), base.join("homes/link")).unwrap();
let root =
cap_std::fs::Dir::open_ambient_dir(base.join("homes"), cap_std::ambient_authority())
.unwrap();
assert!(root.read("link").is_err());
assert!(root.write("link", b"overwritten").is_err());
assert_eq!(std::fs::read(base.join("secret")).unwrap(), b"private");
drop(root);
std::fs::remove_dir_all(base).unwrap();
}
}

View File

@ -6,6 +6,11 @@ use axum::{Json, Router};
use serde::Deserialize;
use serde_json::json;
use sha2::{Digest, Sha256};
use std::{
collections::HashMap,
sync::{Arc, Mutex},
time::{Duration, Instant},
};
use crate::state::AppState;
@ -14,7 +19,7 @@ const COOKIE_NAME: &str = "lazyboy_session";
#[derive(Clone)]
pub struct AuthConfig {
token: Option<String>,
session_value: Option<String>,
sessions: Arc<Mutex<HashMap<String, Instant>>>,
secure_cookie: bool,
}
@ -23,18 +28,12 @@ impl AuthConfig {
let token = std::env::var("LAZYBOY_APP_TOKEN")
.ok()
.filter(|value| !value.trim().is_empty());
let session_value = token.as_ref().map(|value| {
let mut hasher = Sha256::new();
hasher.update(b"lazyboy-session-v1:");
hasher.update(value.as_bytes());
hex::encode(hasher.finalize())
});
let secure_cookie = std::env::var("LAZYBOY_SECURE_COOKIE")
.map(|value| matches!(value.as_str(), "1" | "true" | "yes"))
.unwrap_or(false);
Self {
token,
session_value,
sessions: Arc::new(Mutex::new(HashMap::new())),
secure_cookie,
}
}
@ -58,21 +57,55 @@ impl AuthConfig {
}
fn valid_session(&self, headers: &HeaderMap) -> bool {
let Some(expected) = &self.session_value else {
if !self.enabled() {
return true;
}
let Some(value) = cookie_value(headers, COOKIE_NAME) else {
return false;
};
cookie_value(headers, COOKIE_NAME)
.map(|supplied| constant_time_eq(expected.as_bytes(), supplied.as_bytes()))
.unwrap_or(false)
let key = hex::encode(Sha256::digest(value.as_bytes()));
self.sessions
.lock()
.unwrap()
.get(&key)
.is_some_and(|expires| *expires > Instant::now())
}
fn session_cookie(&self) -> Option<String> {
self.session_value.as_ref().map(|value| {
format!(
if !self.enabled() {
return None;
}
let value = format!(
"{}{}",
uuid::Uuid::new_v4().simple(),
uuid::Uuid::new_v4().simple()
);
let key = hex::encode(Sha256::digest(value.as_bytes()));
let mut sessions = self.sessions.lock().unwrap();
sessions.retain(|_, expires| *expires > Instant::now());
if sessions.len() >= 4096 {
if let Some(oldest) = sessions
.iter()
.min_by_key(|(_, t)| **t)
.map(|(k, _)| k.clone())
{
sessions.remove(&oldest);
}
}
sessions.insert(key, Instant::now() + Duration::from_secs(604800));
Some(format!(
"{COOKIE_NAME}={value}; Path=/; HttpOnly; SameSite=Strict; Max-Age=604800{}",
if self.secure_cookie { "; Secure" } else { "" }
)
})
))
}
fn revoke_session(&self, headers: &HeaderMap) {
if let Some(value) = cookie_value(headers, COOKIE_NAME) {
self.sessions
.lock()
.unwrap()
.remove(&hex::encode(Sha256::digest(value.as_bytes())));
}
}
}
@ -143,7 +176,8 @@ async fn login(
Ok(response)
}
async fn logout() -> Response {
async fn logout(State(state): State<AppState>, headers: HeaderMap) -> Response {
state.auth.revoke_session(&headers);
let mut response = Json(json!({"ok": true})).into_response();
response.headers_mut().insert(
header::SET_COOKIE,
@ -175,3 +209,126 @@ mod tests {
assert!(!constant_time_eq(b"correct", b"correct-longer"));
}
}
/// SameSite cookies do not replace Origin checks (including WebSockets).
pub async fn browser_boundary(
State(state): State<AppState>,
request: Request,
next: Next,
) -> Response {
if !allowed_browser_request(request.headers(), state.auth.enabled()) {
return (
StatusCode::FORBIDDEN,
Json(json!({"message":"cross-origin request rejected"})),
)
.into_response();
}
let private =
request.uri().path().starts_with("/api/") || request.uri().path().starts_with("/view/");
let mut response = next.run(request).await;
let headers = response.headers_mut();
headers.insert(
header::X_CONTENT_TYPE_OPTIONS,
HeaderValue::from_static("nosniff"),
);
headers.insert(
header::X_FRAME_OPTIONS,
HeaderValue::from_static("SAMEORIGIN"),
);
headers.insert(
header::REFERRER_POLICY,
HeaderValue::from_static("same-origin"),
);
if private {
headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
}
response
}
fn allowed_browser_request(headers: &HeaderMap, authenticated_mode: bool) -> bool {
if headers.get("sec-fetch-site").and_then(|v| v.to_str().ok()) == Some("cross-site") {
return false;
}
let Some(host) = headers.get(header::HOST).and_then(|v| v.to_str().ok()) else {
return false;
};
let Ok(destination) = reqwest::Url::parse(&format!("http://{host}")) else {
return false;
};
if !authenticated_mode
&& !matches!(
destination.host_str(),
Some("localhost" | "127.0.0.1" | "[::1]")
)
{
return false;
}
if let Some(origin) = headers.get(header::ORIGIN) {
let Ok(origin) = origin
.to_str()
.ok()
.and_then(|s| reqwest::Url::parse(s).ok())
.ok_or(())
else {
return false;
};
let Ok(expected) = reqwest::Url::parse(&format!("{}://{host}", origin.scheme())) else {
return false;
};
if !matches!(origin.scheme(), "http" | "https") || origin.origin() != expected.origin() {
return false;
}
}
true
}
#[cfg(test)]
mod origin_tests {
use super::*;
#[test]
fn blocks_cross_origin_and_dns_rebinding() {
let mut headers = HeaderMap::new();
headers.insert(header::HOST, "localhost:3101".parse().unwrap());
assert!(allowed_browser_request(&headers, false));
headers.insert(header::ORIGIN, "https://evil.example".parse().unwrap());
assert!(!allowed_browser_request(&headers, true));
headers.insert(header::ORIGIN, "http://localhost:3101".parse().unwrap());
assert!(allowed_browser_request(&headers, true));
headers.remove(header::ORIGIN);
headers.insert(header::HOST, "rebinding.example".parse().unwrap());
assert!(!allowed_browser_request(&headers, false));
}
}
#[cfg(test)]
mod session_tests {
use super::*;
#[test]
fn sessions_are_distinct_expire_and_are_revoked_on_logout() {
let config = AuthConfig {
token: Some("secret".into()),
sessions: Arc::new(Mutex::new(HashMap::new())),
secure_cookie: true,
};
let first = config.session_cookie().unwrap();
let second = config.session_cookie().unwrap();
assert_ne!(first, second);
let mut headers = HeaderMap::new();
headers.insert(
header::COOKIE,
first.split(';').next().unwrap().parse().unwrap(),
);
assert!(config.valid_session(&headers));
config.revoke_session(&headers);
assert!(!config.valid_session(&headers));
headers.insert(
header::COOKIE,
second.split(';').next().unwrap().parse().unwrap(),
);
assert!(config.valid_session(&headers));
for expiry in config.sessions.lock().unwrap().values_mut() {
*expiry = Instant::now() - Duration::from_secs(1);
}
assert!(!config.valid_session(&headers));
}
}

View File

@ -19,6 +19,10 @@ use crate::db::{
};
use crate::state::AppState;
pub const STEP_BOOTING: &str = "電腦啟動中";
pub const STEP_WAKING: &str = "喚醒中";
pub const STEP_HANDOFF: &str = "換手中";
pub fn status_from(
bot_id: &str,
computer: &ComputerRow,
@ -55,6 +59,7 @@ pub fn status_from(
busy_session_id: None,
busy_run_id: None,
busy_step: None,
using_computer: false,
waiting_run_id: None,
waiting_session_id: None,
queued_runs: 0,
@ -320,6 +325,7 @@ async fn probe_computer_container(
],
cwd: None,
timeout_ms: Some(5_000),
stdin: None,
},
&adapter_context(actor, bot_id, "probe"),
)
@ -500,6 +506,21 @@ pub async fn boot(state: &AppState, actor: &Actor, bot_id: &str) -> Result<Compu
.await
.map_err(|error| error.to_string())?;
}
if computer.state == "suspended" && computer.provider_ref.is_some() {
match resume_paused(state, actor, bot_id, &computer).await {
Ok(status) => return Ok(status),
Err(error) => {
tracing::warn!("computer {computer_id} resume failed: {error}");
let _ = sqlx::query(
"UPDATE computers SET state = 'stopped', updated_at = now()
WHERE id = $1 AND state = 'suspended'",
)
.bind(&computer_id)
.execute(state.pool())
.await;
}
}
}
let claimed = sqlx::query(
"UPDATE computers SET state = 'booting', updated_at = now()
WHERE id = $1 AND state IN ('stopped','suspended','error')",
@ -563,6 +584,7 @@ pub async fn boot(state: &AppState, actor: &Actor, bot_id: &str) -> Result<Compu
argv: vec!["mkdir".into(), "-p".into(), "shared".into(), folder],
cwd: None,
timeout_ms: Some(10_000),
stdin: None,
},
&ctx,
),
@ -604,6 +626,71 @@ pub async fn boot(state: &AppState, actor: &Actor, bot_id: &str) -> Result<Compu
Ok(status_from(bot_id, &computer, screen.as_ref(), None))
}
async fn resume_paused(
state: &AppState,
actor: &Actor,
bot_id: &str,
computer: &ComputerRow,
) -> Result<ComputerStatus, String> {
let ctx = adapter_context(actor, bot_id, "resume");
let home = home_path(&state.data_dir, &computer.home_key);
let resumed = match tokio::time::timeout(
Duration::from_secs(20),
state.sandbox.resume(
ProvisionRequest {
home_key: computer.home_key.clone(),
home_path: home.to_string_lossy().into_owned(),
provider_ref: computer.provider_ref.clone(),
},
&ctx,
),
)
.await
{
Ok(Ok(resumed)) => resumed,
Ok(Err(error)) => return Err(error.to_string()),
Err(_) => return Err("computer resume timed out after 20 seconds".into()),
};
let running = sqlx::query(
"UPDATE computers SET state = 'running', provider_ref = $2, kind = $3, updated_at = now()
WHERE id = $1 AND state = 'suspended'",
)
.bind(&computer.id)
.bind(&resumed.provider_ref)
.bind(resumed.kind.as_str())
.execute(state.pool())
.await
.map_err(|error| error.to_string())?;
if running.rows_affected() != 1 {
let current = state
.db
.get_computer(&computer.id)
.await
.map_err(|error| error.to_string())?
.ok_or_else(|| "computer not found".to_string())?;
if current.state == "running" {
let screen = ensure_bot_screen(state, actor, bot_id, &current, None)
.await
.ok()
.and_then(|bound| bound.row);
return Ok(status_from(bot_id, &current, screen.as_ref(), None));
}
return Err("computer resume was superseded".into());
}
let computer = state
.db
.get_computer(&computer.id)
.await
.map_err(|error| error.to_string())?
.ok_or_else(|| "computer not found".to_string())?;
let screen = ensure_bot_screen(state, actor, bot_id, &computer, None)
.await
.ok()
.and_then(|bound| bound.row);
restore_computer_screens(state, actor, &computer, bot_id).await;
Ok(status_from(bot_id, &computer, screen.as_ref(), None))
}
async fn mark_boot_error(state: &AppState, computer_id: &str, provider_ref: Option<&str>) {
if let Err(error) = sqlx::query(
"UPDATE computers SET state = 'error', provider_ref = COALESCE($2, provider_ref), updated_at = now()
@ -635,9 +722,12 @@ pub async fn stop(state: &AppState, actor: &Actor, bot_id: &str) -> Result<Compu
.await
.map_err(|error| error.to_string())?
.ok_or_else(|| "computer not found".to_string())?;
if computer_has_active_work(state, &computer_id).await {
return Err("電腦仍有工作或示範進行中,請先停止工作再關閉電腦。".into());
}
if let Some(provider_ref) = &computer.provider_ref {
let ctx = adapter_context(actor, bot_id, "stop");
let _ = state
state
.sandbox
.stop(
&lazyboy_control::ComputerRef {
@ -649,7 +739,8 @@ pub async fn stop(state: &AppState, actor: &Actor, bot_id: &str) -> Result<Compu
},
&ctx,
)
.await;
.await
.map_err(|error| format!("無法關閉電腦:{error}"))?;
}
sqlx::query(
"UPDATE computers SET state = 'stopped', control_holder = 'none', control_lease_id = NULL,
@ -813,6 +904,7 @@ pub async fn takeover(
.await
.map_err(|error| error.to_string())?;
for (run_id, thread_id) in paused {
crate::runs::set_run_step(state, &run_id, STEP_HANDOFF).await;
let _ = crate::runs::append_bot_message(
state,
&thread_id,
@ -878,6 +970,11 @@ pub async fn heartbeat(state: &AppState, actor: &Actor, bot_id: &str) -> Result<
return Ok(());
};
let expires = Utc::now() + TimeDelta::minutes(15);
sqlx::query("UPDATE computers SET updated_at = now() WHERE id = $1")
.bind(&computer_id)
.execute(state.pool())
.await
.map_err(|error| error.to_string())?;
sqlx::query(
"UPDATE computer_screens SET control_lease_expires_at = $2, updated_at = now()
WHERE computer_id = $1 AND bot_id = $3 AND control_holder = 'user'",
@ -933,6 +1030,36 @@ pub async fn idle_loop(state: AppState) {
loop {
tokio::time::sleep(Duration::from_secs(60)).await;
crate::attachments::sweep_all_inboxes(&state.data_dir).await;
pause_idle_computers(&state).await;
stop_parked_computers(&state).await;
}
}
async fn computer_has_active_work(state: &AppState, computer_id: &str) -> bool {
let active: Result<Option<(i64,)>, _> = sqlx::query_as(
"SELECT 1 FROM runs WHERE status IN ('queued','leased','running','waiting_input','waiting_takeover')
AND bot_id IN (SELECT id FROM bots WHERE computer_id = $1)
UNION ALL
SELECT 1 FROM taught_skills WHERE status IN ('recording','drafting')
AND bot_id IN (SELECT id FROM bots WHERE computer_id = $1)
LIMIT 1",
)
.bind(computer_id)
.fetch_optional(state.pool())
.await;
matches!(active, Ok(Some(_)))
}
fn idle_adapter(computer: &ComputerRow, operation: &str) -> AdapterContext {
AdapterContext {
operation_id: operation.into(),
space_id: computer.space_id.clone(),
user_id: computer.user_id.clone(),
..Default::default()
}
}
async fn pause_idle_computers(state: &AppState) {
let cutoff = Utc::now() - TimeDelta::minutes(10);
let rows = sqlx::query_as::<_, ComputerRow>(
"SELECT id, space_id, user_id, scope, scope_key, home_key, home_revision, kind, provider_ref, state,
@ -944,51 +1071,62 @@ pub async fn idle_loop(state: AppState) {
.bind(cutoff)
.fetch_all(state.pool())
.await;
let Ok(rows) = rows else { continue };
let Ok(rows) = rows else { return };
for computer in rows {
let active: Result<Option<(i64,)>, _> = sqlx::query_as(
"SELECT 1 FROM runs WHERE status IN ('queued','leased','running','waiting_input','waiting_takeover')
AND bot_id IN (SELECT id FROM bots WHERE computer_id = $1)
UNION ALL
SELECT 1 FROM taught_skills WHERE status IN ('recording','drafting')
AND bot_id IN (SELECT id FROM bots WHERE computer_id = $1)
LIMIT 1",
)
.bind(&computer.id)
.fetch_optional(state.pool())
.await;
if matches!(active, Ok(Some(_))) {
if computer_has_active_work(state, &computer.id).await {
continue;
}
if let Some(provider_ref) = &computer.provider_ref {
let ctx = AdapterContext {
operation_id: "idle".into(),
space_id: computer.space_id.clone(),
user_id: computer.user_id.clone(),
..Default::default()
};
let _ = state
if let Some(computer_ref) = computer_ref(&computer) {
if state
.sandbox
.stop(
&lazyboy_control::ComputerRef {
id: provider_ref.clone(),
home_key: computer.home_key.clone(),
kind: parse_kind(&computer.kind),
provider_ref: provider_ref.clone(),
fresh: false,
},
&ctx,
)
.await;
.suspend(&computer_ref, &idle_adapter(&computer, "idle"))
.await
.is_err()
{
continue;
}
}
let _ = sqlx::query(
"UPDATE computers SET state = 'stopped', updated_at = now() WHERE id = $1",
"UPDATE computers SET state = 'suspended', updated_at = now()
WHERE id = $1 AND state = 'running'",
)
.bind(&computer.id)
.execute(state.pool())
.await;
}
}
async fn stop_parked_computers(state: &AppState) {
let cutoff = Utc::now() - TimeDelta::hours(6);
let rows = sqlx::query_as::<_, ComputerRow>(
"SELECT id, space_id, user_id, scope, scope_key, home_key, home_revision, kind, provider_ref, state,
control_holder, control_lease_id, control_lease_expires_at, control_bot_id, control_run_id,
execution_run_id, execution_bot_id, execution_lease_expires_at, execution_fence,
browser_profile_mode
FROM computers WHERE state = 'suspended' AND updated_at < $1",
)
.bind(cutoff)
.fetch_all(state.pool())
.await;
let Ok(rows) = rows else { return };
for computer in rows {
if computer_has_active_work(state, &computer.id).await {
continue;
}
if let Some(computer_ref) = computer_ref(&computer) {
let _ = state
.sandbox
.stop(&computer_ref, &idle_adapter(&computer, "parked"))
.await;
}
let _ = sqlx::query(
"UPDATE computers SET state = 'stopped', updated_at = now()
WHERE id = $1 AND state = 'suspended'",
)
.bind(&computer.id)
.execute(state.pool())
.await;
}
}
pub fn computer_ref(computer: &ComputerRow) -> Option<lazyboy_control::ComputerRef> {
@ -1069,6 +1207,11 @@ pub async fn current_status(
status.busy_run_id = Some(run.id.clone());
status.busy_session_id = Some(run.thread_id.clone());
status.busy_step = run.step.clone();
status.using_computer = screen
.as_ref()
.and_then(|row| row.execution_run_id.as_deref())
== Some(run.id.as_str())
|| computer.execution_run_id.as_deref() == Some(run.id.as_str());
}
if let Some(run) = waiting {
status.waiting_run_id = Some(run.id.clone());

View File

@ -95,6 +95,11 @@ pub struct SpaceRow {
pub default_model_id: String,
pub default_model_base_url: Option<String>,
pub default_model_api_key: Option<String>,
pub voice_enabled: bool,
pub voice_provider: Option<String>,
pub voice_model_id: Option<String>,
pub voice_id: Option<String>,
pub voice_api_key: Option<String>,
}
impl Db {
@ -125,7 +130,8 @@ impl Db {
pub async fn get_space(&self, actor: &Actor) -> Result<Option<SpaceRow>, sqlx::Error> {
sqlx::query_as(
"SELECT id, user_id, name, default_model_provider, default_model_id,
default_model_base_url, default_model_api_key
default_model_base_url, default_model_api_key,
voice_enabled, voice_provider, voice_model_id, voice_id, voice_api_key
FROM spaces WHERE id = $1 AND user_id = $2",
)
.bind(&actor.space_id)
@ -178,6 +184,51 @@ impl Db {
self.get_space(actor).await?.ok_or(sqlx::Error::RowNotFound)
}
pub async fn update_voice_settings(
&self,
actor: &Actor,
enabled: Option<bool>,
provider: &str,
model_id: &str,
voice_id: &str,
api_key: Option<Option<&str>>,
) -> Result<SpaceRow, sqlx::Error> {
match api_key {
Some(key) => {
sqlx::query(
"UPDATE spaces
SET voice_provider = $3, voice_model_id = $4, voice_id = $5, voice_api_key = $6, voice_enabled = COALESCE($7, voice_enabled)
WHERE id = $1 AND user_id = $2",
)
.bind(&actor.space_id)
.bind(&actor.user_id)
.bind(provider)
.bind(model_id)
.bind(voice_id)
.bind(key)
.bind(enabled)
.execute(&self.pool)
.await?;
}
None => {
sqlx::query(
"UPDATE spaces
SET voice_provider = $3, voice_model_id = $4, voice_id = $5, voice_enabled = COALESCE($6, voice_enabled)
WHERE id = $1 AND user_id = $2",
)
.bind(&actor.space_id)
.bind(&actor.user_id)
.bind(provider)
.bind(model_id)
.bind(voice_id)
.bind(enabled)
.execute(&self.pool)
.await?;
}
}
self.get_space(actor).await?.ok_or(sqlx::Error::RowNotFound)
}
pub async fn list_bots(
&self,
actor: &Actor,

View File

@ -0,0 +1,96 @@
//! Read-only, workspace-wide SKILL.md commands.
use std::path::{Path, PathBuf};
use serde::Serialize;
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct FileSkill {
pub name: String,
pub description: String,
#[serde(skip_serializing)]
pub instructions: String,
}
fn valid_name(name: &str) -> bool {
!name.is_empty()
&& name != "goal"
&& name != "skills"
&& name != "help"
&& name.bytes().all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
}
fn parse_file(path: PathBuf, name: String) -> Option<FileSkill> {
let file_type = std::fs::symlink_metadata(&path).ok()?.file_type();
if file_type.is_symlink() || !file_type.is_file() {
return None;
}
if std::fs::metadata(&path).ok()?.len() > 512 * 1024 {
return None;
}
let text = std::fs::read_to_string(&path).ok()?;
let (frontmatter, instructions) = if let Some(rest) = text.strip_prefix("---\n") {
let (header, body) = rest.split_once("\n---")?;
(
header,
body.trim_start_matches(|ch| ch == '\n' || ch == '\r')
.trim()
.to_string(),
)
} else {
("", text.trim().to_string())
};
let description = frontmatter.lines().find_map(|line| {
let (key, value) = line.split_once(':')?;
(key.trim() == "description")
.then(|| value.trim().trim_matches(|ch| ch == '"' || ch == '\'').to_string())
}).unwrap_or_else(|| instructions.lines().next().unwrap_or("自訂技能").chars().take(120).collect());
(!instructions.is_empty()).then_some(FileSkill { name, description, instructions })
}
pub fn list(data_dir: &str) -> Vec<FileSkill> {
let root = Path::new(data_dir).join("skills");
let Ok(entries) = std::fs::read_dir(&root) else { return Vec::new() };
let mut skills = entries.filter_map(Result::ok).filter_map(|entry| {
let file_type = entry.file_type().ok()?;
if !file_type.is_dir() { return None; }
let name = entry.file_name().to_string_lossy().to_string();
valid_name(&name).then(|| parse_file(entry.path().join("SKILL.md"), name)).flatten()
}).collect::<Vec<_>>();
skills.sort_by(|left, right| left.name.cmp(&right.name));
skills
}
pub fn slash(prompt: &str, data_dir: &str) -> Option<(FileSkill, String)> {
let mut words = prompt.trim().splitn(2, char::is_whitespace);
let command = words.next()?.strip_prefix('/')?;
if !valid_name(command) { return None; }
let skill = list(data_dir).into_iter().find(|skill| skill.name == command)?;
Some((skill, words.next().unwrap_or("").trim().to_string()))
}
pub fn index(data_dir: &str) -> String {
let skills = list(data_dir);
if skills.is_empty() { return String::new(); }
let lines = skills.iter().map(|skill| format!("/{:<18} {}", skill.name, skill.description)).collect::<Vec<_>>().join("\n");
format!("可用的檔案技能(唯讀):\n{lines}")
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
#[test]
fn loads_only_safe_skill_names_and_resolves_arguments() {
let root = std::env::temp_dir().join(format!("lazyboy-file-skills-{}", uuid::Uuid::new_v4()));
fs::create_dir_all(root.join("skills/open-site")).unwrap();
fs::write(root.join("skills/open-site/SKILL.md"), "---\ndescription: Open a site\n---\nUse the browser.\n").unwrap();
fs::create_dir_all(root.join("skills/goal")).unwrap();
let root_text = root.to_str().unwrap();
assert_eq!(slash("/open-site example.com", root_text).unwrap().1, "example.com");
assert!(slash("/goal do it", root_text).is_none());
assert!(slash("/missing", root_text).is_none());
let _ = fs::remove_dir_all(root);
}
}

View File

@ -2,26 +2,32 @@ mod attachments;
mod auth;
mod computer;
mod db;
mod file_skills;
mod mcp;
mod mcp_catalog;
mod memory;
mod retention;
mod rooms;
mod routes;
mod runs;
mod schedules;
mod screen_proxy;
mod sessions;
mod skills;
mod state;
mod tools;
mod vault;
mod voice;
mod voice_call;
mod web_static;
mod workspace;
use std::net::SocketAddr;
use std::sync::Arc;
use axum::Router;
use axum::extract::DefaultBodyLimit;
use state::AppState;
use tower_http::services::ServeDir;
use tracing_subscriber::EnvFilter;
#[tokio::main]
@ -49,6 +55,9 @@ async fn main() {
mcp_state.mcp.reconnect_all(mcp_state.pool(), &actor).await;
});
let retention_state = state.clone();
tokio::spawn(async move { retention::retention_loop(retention_state).await; });
let worker_state = state.clone();
tokio::spawn(async move {
runs::worker_loop(worker_state).await;
@ -57,6 +66,10 @@ async fn main() {
tokio::spawn(async move {
computer::idle_loop(idle_state).await;
});
let schedule_state = state.clone();
tokio::spawn(async move {
schedules::tick_loop(schedule_state).await;
});
let bind = std::env::var("API_BIND").unwrap_or_else(|_| "127.0.0.1:3101".into());
let addr: SocketAddr = bind.parse().expect("API_BIND");
@ -66,16 +79,19 @@ async fn main() {
);
}
let web_dir = std::env::var("LAZYBOY_WEB_DIR").unwrap_or_else(|_| "apps/web".into());
let app = Router::new()
let web_dir = std::env::var("LAZYBOY_WEB_DIR").unwrap_or_else(|_| "apps/web/dist".into());
let app = web_static::static_router(&web_dir)
.route(
"/api/health",
axum::routing::get(|| async { axum::Json(serde_json::json!({"ok": true})) }),
)
.merge(auth::public_router(state.clone()))
.merge(routes::router(state))
.fallback_service(ServeDir::new(web_dir))
.layer(DefaultBodyLimit::max(24 * 1024 * 1024));
.merge(routes::router(state.clone()))
.layer(DefaultBodyLimit::max(28 * 1024 * 1024))
.layer(axum::middleware::from_fn_with_state(
state,
auth::browser_boundary,
));
tracing::info!("api listening on {addr}");
let listener = tokio::net::TcpListener::bind(addr).await.expect("bind");

View File

@ -254,6 +254,21 @@ async fn connect_client(row: &McpRow) -> Result<LiveClient, String> {
.filter(|value| !value.is_empty())
.ok_or_else(|| "stdio 需要 command".to_string())?;
let mut cmd = Command::new(command);
cmd.env_clear();
for key in [
"PATH",
"HOME",
"LANG",
"LC_ALL",
"TMPDIR",
"PYTHONPATH",
"NODE_EXTRA_CA_CERTS",
] {
if let Some(value) = std::env::var_os(key) {
cmd.env(key, value);
}
}
cmd.kill_on_drop(true);
cmd.args(&row.args);
cmd.stdin(std::process::Stdio::piped());
cmd.stdout(std::process::Stdio::piped());

View File

@ -0,0 +1,84 @@
//! Bounded maintenance of expendable diagnostics, never user-authored content.
use std::time::{Duration};
use sqlx::PgPool;
use crate::state::AppState;
fn days(name: &str, default: i32) -> i32 {
std::env::var(name).ok().and_then(|v| v.parse::<i32>().ok())
.filter(|v| (1..=3650).contains(v)).unwrap_or(default)
}
pub async fn retention_loop(state: AppState) {
let rules = [
("events", include_str!("retention/events.sql"), days("LAZYBOY_EVENT_RETENTION_DAYS", 30)),
("checkpoints", include_str!("retention/checkpoints.sql"), days("LAZYBOY_CHECKPOINT_RETENTION_DAYS", 7)),
("runs", include_str!("retention/runs.sql"), days("LAZYBOY_RUN_RETENTION_DAYS", 90)),
("recordings", include_str!("retention/recordings.sql"), days("LAZYBOY_RECORDING_RETENTION_DAYS", 30)),
("revisions", include_str!("retention/revisions.sql"), days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90)),
("deleted_memories", include_str!("retention/deleted_memories.sql"), days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90)),
("leases", include_str!("retention/leases.sql"), 7),
("profile_locks", include_str!("retention/profile_locks.sql"), 7),
];
loop {
for (name, query, age) in rules {
let mut removed = 0;
// Limit both transaction size and work per hour; defer excess backlog.
for _ in 0..20 {
match batch(state.pool(), query, age).await {
Ok(count) => { removed += count; if count < 1000 { break; } }
Err(error) => { tracing::warn!(name, %error, "retention batch failed; retry next hour"); break; }
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
if removed > 0 { tracing::info!(name, rows=removed, "retention cleaned expired diagnostics"); }
}
if let Err(error) = clean_frames(&state, rules[3].2).await {
tracing::warn!(%error, "recording file retention failed; retry next hour");
}
if let Ok(bytes) = sqlx::query_scalar::<_, i64>("SELECT pg_database_size(current_database())").fetch_one(state.pool()).await {
let warn_mb = std::env::var("LAZYBOY_DB_WARN_MB").ok().and_then(|v| v.parse::<i64>().ok()).filter(|v| *v > 0 && *v < 1_000_000).unwrap_or(1024);
tracing::info!(bytes, "database size after retention");
if bytes > warn_mb * 1024 * 1024 { tracing::warn!(bytes, warn_mb, "database exceeds configured size warning; review retained conversations and memories"); }
}
tokio::time::sleep(Duration::from_secs(3600)).await;
}
}
async fn batch(pool: &PgPool, query: &str, age: i32) -> Result<u64, sqlx::Error> {
let mut tx = pool.begin().await?;
// One maintenance writer, even when multiple API processes start together.
let acquired: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(72189431)").fetch_one(&mut *tx).await?;
if !acquired { return Ok(0); }
sqlx::query("SET LOCAL statement_timeout = '10s'").execute(&mut *tx).await?;
sqlx::query("SET LOCAL lock_timeout = '1s'").execute(&mut *tx).await?;
let result = sqlx::query(query).bind(age).bind(1000_i64).execute(&mut *tx).await?;
tx.commit().await?;
Ok(result.rows_affected())
}
async fn clean_frames(state: &AppState, age: i32) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let root = std::path::Path::new(&state.data_dir).join("teach");
let mut dirs = match tokio::fs::read_dir(&root).await {
Ok(dirs) => dirs,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error.into()),
};
let mut cleaned = 0;
while let Some(entry) = dirs.next_entry().await? {
if cleaned >= 1000 { break; }
// Ignore symlinks and unexpected names; never traverse a user's home.
if !entry.file_type().await?.is_dir() { continue; }
let id = entry.file_name().to_string_lossy().into_owned();
if uuid::Uuid::parse_str(&id).is_err() { continue; }
let eligible: Option<bool> = sqlx::query_scalar(
"SELECT status IN ('saved','failed','draft') AND updated_at < now() - make_interval(days => $2) FROM taught_skills WHERE id=$1"
).bind(&id).bind(age).fetch_optional(state.pool()).await?;
let old_orphan = eligible.is_none() && entry.metadata().await?.modified()?.elapsed().unwrap_or(Duration::ZERO) > Duration::from_secs(age as u64 * 86400);
if eligible == Some(true) || old_orphan {
tokio::fs::remove_dir_all(entry.path()).await?;
cleaned += 1;
}
}
if cleaned > 0 { tracing::info!(cleaned, "removed expired teaching frame directories"); }
Ok(())
}

View File

@ -0,0 +1,6 @@
UPDATE runs SET checkpoint = '{}'::jsonb WHERE id IN (
SELECT id FROM runs WHERE status IN ('completed','failed','cancelled')
AND COALESCE(completed_at,updated_at) < now() - make_interval(days => $1)
AND checkpoint <> '{}'::jsonb
ORDER BY COALESCE(completed_at,updated_at) LIMIT $2 FOR UPDATE SKIP LOCKED
)

View File

@ -0,0 +1,4 @@
DELETE FROM memory_items WHERE id IN (
SELECT id FROM memory_items WHERE deleted_at < now() - make_interval(days => $1)
ORDER BY deleted_at LIMIT $2 FOR UPDATE SKIP LOCKED
)

View File

@ -0,0 +1,4 @@
DELETE FROM events WHERE id IN (
SELECT id FROM events WHERE created_at < now() - make_interval(days => $1)
ORDER BY created_at LIMIT $2 FOR UPDATE SKIP LOCKED
)

View File

@ -0,0 +1,5 @@
DELETE FROM computer_execution_leases WHERE id IN (
SELECT l.id FROM computer_execution_leases l WHERE l.expires_at < now() - make_interval(days => $1)
AND NOT EXISTS (SELECT 1 FROM runs r WHERE r.id=l.run_id AND r.status NOT IN ('completed','failed','cancelled'))
ORDER BY l.expires_at LIMIT $2 FOR UPDATE SKIP LOCKED
)

View File

@ -0,0 +1,6 @@
DELETE FROM computer_profile_locks WHERE (computer_id,profile_key) IN (
SELECT l.computer_id,l.profile_key FROM computer_profile_locks l
WHERE l.expires_at < now() - make_interval(days => $1)
AND NOT EXISTS (SELECT 1 FROM runs r WHERE r.id=l.run_id AND r.status NOT IN ('completed','failed','cancelled'))
ORDER BY l.expires_at LIMIT $2 FOR UPDATE SKIP LOCKED
)

View File

@ -0,0 +1,5 @@
UPDATE taught_skills SET recording = '{}'::jsonb WHERE id IN (
SELECT id FROM taught_skills WHERE status IN ('saved','failed','draft')
AND updated_at < now() - make_interval(days => $1) AND recording <> '{}'::jsonb
ORDER BY updated_at LIMIT $2 FOR UPDATE SKIP LOCKED
)

View File

@ -0,0 +1,7 @@
DELETE FROM memory_revisions WHERE (memory_id,revision) IN (
SELECT r.memory_id,r.revision FROM memory_revisions r JOIN memory_items m ON m.id=r.memory_id
WHERE r.revision < m.revision AND (
r.created_at < now() - make_interval(days => $1)
OR r.revision <= m.revision - 10
) ORDER BY r.created_at LIMIT $2 FOR UPDATE OF r SKIP LOCKED
)

View File

@ -0,0 +1,5 @@
DELETE FROM runs WHERE id IN (
SELECT id FROM runs WHERE status IN ('completed','failed','cancelled')
AND COALESCE(completed_at,updated_at) < now() - make_interval(days => $1)
ORDER BY COALESCE(completed_at,updated_at) LIMIT $2 FOR UPDATE SKIP LOCKED
)

View File

@ -18,7 +18,11 @@ pub fn router(state: AppState) -> Router {
.merge(crate::rooms::router())
.merge(crate::mcp::router())
.merge(crate::workspace::router())
.merge(crate::voice::router())
.merge(crate::skills::router())
.route("/api/file-skills", get(file_skills))
.merge(crate::vault::router())
.merge(crate::schedules::router())
.route("/api/bots", get(list_bots).post(create_bot))
.route(
"/api/bots/{id}",
@ -49,6 +53,10 @@ pub fn router(state: AppState) -> Router {
.with_state(state)
}
async fn file_skills(State(state): State<AppState>) -> Json<Vec<crate::file_skills::FileSkill>> {
Json(crate::file_skills::list(&state.data_dir))
}
async fn actor(state: &AppState) -> Result<Actor, StatusCode> {
state
.bootstrap()
@ -600,12 +608,12 @@ async fn restart(
async fn stop(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Json<Value>, StatusCode> {
let actor = actor(&state).await?;
) -> Result<Json<Value>, (StatusCode, Json<Value>)> {
let actor = actor(&state).await.map_err(|status| (status, Json(json!({"message":"無法取得工作區"}))))?;
computer::stop(&state, &actor, &id)
.await
.map(|status| Json(serde_json::to_value(status).unwrap()))
.map_err(|_| StatusCode::BAD_REQUEST)
.map_err(|error| (StatusCode::BAD_REQUEST, Json(json!({"message":error}))))
}
async fn screen_url(
@ -649,7 +657,7 @@ async fn screen_url(
.await
.map_err(|_| StatusCode::BAD_GATEWAY)?;
Ok(Json(json!({
"url": format!("/view/{id}/vnc.html?view_only={}", !interactive)
"url": format!("/view/{id}/vnc.html")
})))
}
@ -722,6 +730,39 @@ async fn input(
return Err(StatusCode::CONFLICT);
}
let computer_ref = computer::computer_ref(&computer).ok_or(StatusCode::BAD_REQUEST)?;
if body.kind == "clipboard" || body.kind == "copy" {
let text = body.text.unwrap_or_default();
if text.len() > 1024 * 1024 {
return Err(StatusCode::PAYLOAD_TOO_LARGE);
}
let context =
computer::adapter_context_for(&actor, &id, "clipboard", screen.as_ref(), None);
let mut argv =
lazyboy_control::paste_command_on(context.display.as_deref().unwrap_or(":1"));
if body.kind == "copy" {
argv.push("copy".into());
}
let result = state
.sandbox
.execute(
&computer_ref,
lazyboy_control::CommandRequest {
argv,
cwd: None,
timeout_ms: Some(10_000),
stdin: Some(text),
},
&context,
)
.await
.map_err(|_| StatusCode::BAD_GATEWAY)?;
if result.code != 0 {
return Err(StatusCode::BAD_GATEWAY);
}
return Ok(Json(
json!({"ok":true,"text":if body.kind=="copy" {Some(result.stdout)} else {None}}),
));
}
let action = match body.kind.as_str() {
"key" => lazyboy_contracts::ComputerAction::Key {
key: body.key.unwrap_or_default(),

File diff suppressed because it is too large Load Diff

794
crates/api/src/schedules.rs Normal file
View File

@ -0,0 +1,794 @@
//! Recurring bot runs. Chat or the computer panel can create them; a tick
//! loop turns due rows into ordinary queued runs.
use std::str::FromStr;
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::routing::{get, post};
use axum::{Json, Router};
use chrono::{DateTime, Utc};
use chrono_tz::Tz;
use cron::Schedule;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use sqlx::FromRow;
use uuid::Uuid;
use crate::db::Actor;
use crate::state::AppState;
#[derive(Debug, Clone, Serialize, FromRow)]
#[serde(rename_all = "camelCase")]
pub struct ScheduleRow {
pub id: String,
pub bot_id: String,
pub thread_id: Option<String>,
pub name: String,
pub cron: String,
pub timezone: String,
pub instructions: String,
pub enabled: bool,
pub last_run_at: Option<DateTime<Utc>>,
pub next_run_at: Option<DateTime<Utc>>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CreateSchedule {
pub name: String,
pub cron: String,
#[serde(default)]
pub timezone: String,
pub instructions: String,
#[serde(default)]
pub thread_id: Option<String>,
#[serde(default = "default_enabled")]
pub enabled: bool,
}
fn default_enabled() -> bool {
true
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct UpdateSchedule {
pub name: Option<String>,
pub cron: Option<String>,
pub timezone: Option<String>,
pub instructions: Option<String>,
pub enabled: Option<bool>,
pub thread_id: Option<String>,
}
pub fn router() -> Router<AppState> {
Router::new()
.route(
"/api/bots/{bot_id}/schedules",
get(list_http).post(create_http),
)
.route(
"/api/schedules/{id}",
axum::routing::patch(update_http).delete(delete_http),
)
.route("/api/schedules/{id}/run", post(run_now_http))
}
async fn scoped_actor(state: &AppState, bot_id: &str) -> Result<Actor, StatusCode> {
let actor = state
.bootstrap()
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
state
.db
.get_bot(&actor, bot_id)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
.ok_or(StatusCode::NOT_FOUND)?;
Ok(actor)
}
async fn list_http(
State(state): State<AppState>,
Path(bot_id): Path<String>,
) -> Result<Json<Vec<Value>>, StatusCode> {
let actor = scoped_actor(&state, &bot_id).await?;
let rows = list(&state, &actor, &bot_id)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
Ok(Json(rows.into_iter().map(public_json).collect()))
}
async fn create_http(
State(state): State<AppState>,
Path(bot_id): Path<String>,
Json(input): Json<CreateSchedule>,
) -> Result<Json<Value>, (StatusCode, Json<Value>)> {
let actor = scoped_actor(&state, &bot_id)
.await
.map_err(|status| (status, Json(json!({"message":"bot not found"}))))?;
create(&state, &actor, &bot_id, input)
.await
.map(|row| Json(public_json(row)))
.map_err(|error| (StatusCode::BAD_REQUEST, Json(json!({"message": error}))))
}
async fn update_http(
State(state): State<AppState>,
Path(id): Path<String>,
Json(input): Json<UpdateSchedule>,
) -> Result<Json<Value>, (StatusCode, Json<Value>)> {
let actor = state.bootstrap().await.map_err(|_| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"message":"actor"})),
)
})?;
update(&state, &actor, &id, input)
.await
.map_err(|error| (StatusCode::BAD_REQUEST, Json(json!({"message": error}))))?
.map(|row| Json(public_json(row)))
.ok_or((
StatusCode::NOT_FOUND,
Json(json!({"message":"schedule not found"})),
))
}
async fn delete_http(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<StatusCode, StatusCode> {
let actor = state
.bootstrap()
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let deleted = sqlx::query("DELETE FROM schedules WHERE id=$1 AND space_id=$2 AND user_id=$3")
.bind(&id)
.bind(&actor.space_id)
.bind(&actor.user_id)
.execute(state.pool())
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
if deleted.rows_affected() == 0 {
return Err(StatusCode::NOT_FOUND);
}
Ok(StatusCode::NO_CONTENT)
}
async fn run_now_http(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Json<Value>, (StatusCode, Json<Value>)> {
let actor = state.bootstrap().await.map_err(|_| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"message":"actor"})),
)
})?;
let row = get_row(&state, &actor, &id)
.await
.map_err(|error| (StatusCode::BAD_REQUEST, Json(json!({"message": error}))))?
.ok_or((
StatusCode::NOT_FOUND,
Json(json!({"message":"schedule not found"})),
))?;
enqueue(&state, &row, false)
.await
.map(|run_id| Json(json!({"ok": true, "runId": run_id})))
.map_err(|error| (StatusCode::BAD_REQUEST, Json(json!({"message": error}))))
}
pub fn public_json(row: ScheduleRow) -> Value {
json!({
"id": row.id,
"botId": row.bot_id,
"threadId": row.thread_id,
"name": row.name,
"cron": row.cron,
"timezone": row.timezone,
"instructions": row.instructions,
"enabled": row.enabled,
"human": describe_cron(&row.cron),
"lastRunAt": row.last_run_at,
"nextRunAt": row.next_run_at,
"createdAt": row.created_at,
"updatedAt": row.updated_at,
})
}
pub async fn list(
state: &AppState,
actor: &Actor,
bot_id: &str,
) -> Result<Vec<ScheduleRow>, String> {
sqlx::query_as(
"SELECT id, bot_id, thread_id, name, cron, timezone, instructions, enabled,
last_run_at, next_run_at, created_at, updated_at
FROM schedules
WHERE bot_id=$1 AND space_id=$2 AND user_id=$3
ORDER BY enabled DESC, next_run_at ASC NULLS LAST, updated_at DESC",
)
.bind(bot_id)
.bind(&actor.space_id)
.bind(&actor.user_id)
.fetch_all(state.pool())
.await
.map_err(|error| error.to_string())
}
pub async fn get_row(
state: &AppState,
actor: &Actor,
id: &str,
) -> Result<Option<ScheduleRow>, String> {
sqlx::query_as(
"SELECT id, bot_id, thread_id, name, cron, timezone, instructions, enabled,
last_run_at, next_run_at, created_at, updated_at
FROM schedules WHERE id=$1 AND space_id=$2 AND user_id=$3",
)
.bind(id)
.bind(&actor.space_id)
.bind(&actor.user_id)
.fetch_optional(state.pool())
.await
.map_err(|error| error.to_string())
}
pub async fn create(
state: &AppState,
actor: &Actor,
bot_id: &str,
input: CreateSchedule,
) -> Result<ScheduleRow, String> {
validate_thread(state, actor, bot_id, input.thread_id.as_deref()).await?;
let name = clean_name(&input.name)?;
let instructions = clean_instructions(&input.instructions)?;
let cron = validate_cron(&input.cron)?;
let timezone = validate_timezone(&input.timezone)?;
let next = if input.enabled {
Some(next_fire(&cron, &timezone, Utc::now())?)
} else {
None
};
let id = Uuid::new_v4().to_string();
sqlx::query_as(
"INSERT INTO schedules
(id, space_id, user_id, bot_id, thread_id, name, cron, timezone, instructions, enabled, next_run_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)
RETURNING id, bot_id, thread_id, name, cron, timezone, instructions, enabled,
last_run_at, next_run_at, created_at, updated_at",
)
.bind(&id)
.bind(&actor.space_id)
.bind(&actor.user_id)
.bind(bot_id)
.bind(input.thread_id.as_deref())
.bind(name)
.bind(&cron)
.bind(&timezone)
.bind(instructions)
.bind(input.enabled)
.bind(next)
.fetch_one(state.pool())
.await
.map_err(|error| error.to_string())
}
async fn update(
state: &AppState,
actor: &Actor,
id: &str,
input: UpdateSchedule,
) -> Result<Option<ScheduleRow>, String> {
let existing = match get_row(state, actor, id).await? {
Some(row) => row,
None => return Ok(None),
};
let name = match input.name {
Some(value) => clean_name(&value)?,
None => existing.name.clone(),
};
let instructions = match input.instructions {
Some(value) => clean_instructions(&value)?,
None => existing.instructions.clone(),
};
let cron = match input.cron {
Some(value) => validate_cron(&value)?,
None => existing.cron.clone(),
};
let timezone = match input.timezone {
Some(value) => validate_timezone(&value)?,
None => existing.timezone.clone(),
};
let enabled = input.enabled.unwrap_or(existing.enabled);
let thread_id = input.thread_id.or(existing.thread_id);
validate_thread(state, actor, &existing.bot_id, thread_id.as_deref()).await?;
let next = if enabled {
Some(next_fire(&cron, &timezone, Utc::now())?)
} else {
None
};
sqlx::query_as(
"UPDATE schedules
SET name=$1, cron=$2, timezone=$3, instructions=$4, enabled=$5, thread_id=$6,
next_run_at=$7, updated_at=now()
WHERE id=$8 AND space_id=$9 AND user_id=$10
RETURNING id, bot_id, thread_id, name, cron, timezone, instructions, enabled,
last_run_at, next_run_at, created_at, updated_at",
)
.bind(name)
.bind(cron)
.bind(timezone)
.bind(instructions)
.bind(enabled)
.bind(thread_id)
.bind(next)
.bind(id)
.bind(&actor.space_id)
.bind(&actor.user_id)
.fetch_optional(state.pool())
.await
.map_err(|error| error.to_string())
}
pub async fn tick_loop(state: AppState) {
loop {
tokio::time::sleep(std::time::Duration::from_secs(15)).await;
if let Err(error) = fire_due(&state).await {
tracing::warn!("schedule tick failed: {error}");
}
}
}
async fn fire_due(state: &AppState) -> Result<(), String> {
let due: Vec<ScheduleRow> = sqlx::query_as(
"SELECT id, bot_id, thread_id, name, cron, timezone, instructions, enabled,
last_run_at, next_run_at, created_at, updated_at
FROM schedules
WHERE enabled AND next_run_at IS NOT NULL AND next_run_at <= now()
ORDER BY next_run_at ASC
LIMIT 8",
)
.fetch_all(state.pool())
.await
.map_err(|error| error.to_string())?;
for row in due {
if let Err(error) = enqueue(state, &row, true).await {
tracing::warn!("schedule {} failed to enqueue: {error}", row.id);
}
}
Ok(())
}
async fn enqueue(state: &AppState, row: &ScheduleRow, from_tick: bool) -> Result<String, String> {
let mut tx = state.pool().begin().await.map_err(|e| e.to_string())?;
let locked: Option<ScheduleRow> = if from_tick {
sqlx::query_as("SELECT id,bot_id,thread_id,name,cron,timezone,instructions,enabled,last_run_at,next_run_at,created_at,updated_at FROM schedules WHERE id=$1 AND enabled AND next_run_at<=now() FOR UPDATE SKIP LOCKED")
.bind(&row.id).fetch_optional(&mut *tx).await.map_err(|e| e.to_string())?
} else {
Some(row.clone())
};
let Some(row) = locked.as_ref() else {
return Ok(String::new());
};
let bot = sqlx::query_as::<_, (String, String, String)>(
"SELECT space_id, user_id, id FROM bots WHERE id=$1",
)
.bind(&row.bot_id)
.fetch_optional(state.pool())
.await
.map_err(|error| error.to_string())?
.ok_or_else(|| "bot not found".to_string())?;
let thread_id = match row.thread_id.as_deref() {
Some(id) => id.to_string(),
None => latest_thread(state, &row.bot_id).await?,
};
validate_thread(
state,
&Actor {
space_id: bot.0.clone(),
user_id: bot.1.clone(),
},
&row.bot_id,
Some(&thread_id),
)
.await?;
let prompt = format!("Scheduled task 「{}」:\n{}", row.name, row.instructions);
let seq: i32 = sqlx::query_scalar(
"UPDATE threads SET next_message_seq=next_message_seq+1, updated_at=now()
WHERE id=$1 RETURNING next_message_seq-1",
)
.bind(&thread_id)
.fetch_one(&mut *tx)
.await
.map_err(|error| error.to_string())?;
let run_id = Uuid::new_v4().to_string();
let message_id = Uuid::new_v4().to_string();
let body = if from_tick {
format!("[排程] {}", row.name)
} else {
format!("[排程試跑] {}", row.name)
};
sqlx::query(
"INSERT INTO messages (id,thread_id,seq,role,body,blocks,run_id)
VALUES ($1,$2,$3,'user',$4,$5,$6)",
)
.bind(&message_id)
.bind(&thread_id)
.bind(seq)
.bind(&body)
.bind(json!([{"kind":"scheduleRun","scheduleId":row.id,"name":row.name,"cron":row.cron,"human":describe_cron(&row.cron)}]))
.bind(&run_id)
.execute(&mut *tx)
.await
.map_err(|error| error.to_string())?;
sqlx::query(
"INSERT INTO runs (id,space_id,bot_id,thread_id,user_id,status,prompt,checkpoint)
VALUES ($1,$2,$3,$4,$5,'queued',$6,$7)",
)
.bind(&run_id)
.bind(&bot.0)
.bind(&row.bot_id)
.bind(&thread_id)
.bind(&bot.1)
.bind(&prompt)
.bind(json!({"messageSeq": seq, "scheduleId": row.id}))
.execute(&mut *tx)
.await
.map_err(|error| error.to_string())?;
if from_tick {
let next = next_after_due(
&row.cron,
&row.timezone,
row.next_run_at.unwrap_or_else(Utc::now),
Utc::now(),
)?;
sqlx::query(
"UPDATE schedules SET last_run_at=now(),next_run_at=$2,updated_at=now() WHERE id=$1",
)
.bind(&row.id)
.bind(next)
.execute(&mut *tx)
.await
.map_err(|e| e.to_string())?;
}
tx.commit().await.map_err(|error| error.to_string())?;
Ok(run_id)
}
async fn latest_thread(state: &AppState, bot_id: &str) -> Result<String, String> {
sqlx::query_scalar(
"SELECT id FROM threads
WHERE bot_id=$1 AND status='active' AND room_id IS NULL
ORDER BY updated_at DESC LIMIT 1",
)
.bind(bot_id)
.fetch_optional(state.pool())
.await
.map_err(|error| error.to_string())?
.ok_or_else(|| "bot has no conversation to run this schedule in".into())
}
fn clean_name(name: &str) -> Result<String, String> {
let value = name.trim();
if value.is_empty() || value.chars().count() > 80 {
return Err("name must be 180 characters".into());
}
Ok(value.to_string())
}
fn clean_instructions(text: &str) -> Result<String, String> {
let value = text.trim();
if value.is_empty() || value.chars().count() > 8_000 {
return Err("instructions must be 18000 characters".into());
}
Ok(value.to_string())
}
pub fn validate_cron(expr: &str) -> Result<String, String> {
let trimmed = expr.trim();
if interval_seconds(trimmed)?.is_none() {
parse_schedule(trimmed)?;
}
Ok(trimmed.to_string())
}
fn validate_timezone(value: &str) -> Result<String, String> {
let trimmed = value.trim();
if trimmed.is_empty() {
return Ok("Asia/Taipei".into());
}
Tz::from_str(trimmed).map_err(|_| "invalid IANA timezone".to_string())?;
Ok(trimmed.to_string())
}
fn interval_seconds(expr: &str) -> Result<Option<i64>, String> {
let Some(raw) = expr.strip_prefix("@every ") else {
return Ok(None);
};
if !raw.is_ascii() || raw.len() < 2 {
return Err("invalid fixed interval".into());
}
let (number, unit) = raw.split_at(raw.len() - 1);
let n: i64 = number.parse().map_err(|_| "invalid interval")?;
if !(1..=365).contains(&n) {
return Err("interval must be 1365".into());
}
let scale = match unit {
"m" => 60,
"h" => 3600,
"d" => 86400,
_ => return Err("interval unit must be m, h, or d".into()),
};
Ok(Some(n * scale))
}
fn parse_schedule(expr: &str) -> Result<Schedule, String> {
let fields: Vec<&str> = expr.split_whitespace().collect();
if fields.len() != 5 {
return Err("cron must have exactly 5 fields".into());
}
if fields[2] != "*" && fields[4] != "*" {
return Err("use either day-of-month or day-of-week, not both".into());
}
let weekday = standard_weekdays(fields[4])?;
let six = format!(
"0 {} {} {} {} {}",
fields[0], fields[1], fields[2], fields[3], weekday
);
Schedule::from_str(&six).map_err(|error| format!("invalid cron: {error}"))
}
// UI/API use Unix weekdays (0/7=Sun, 1=Mon); cron crate uses 1=Sun.
fn standard_weekdays(field: &str) -> Result<String, String> {
if field == "*" {
return Ok("*".into());
}
fn value(raw: &str) -> Result<u32, String> {
match raw.to_ascii_uppercase().as_str() {
"SUN" => Ok(0),
"MON" => Ok(1),
"TUE" => Ok(2),
"WED" => Ok(3),
"THU" => Ok(4),
"FRI" => Ok(5),
"SAT" => Ok(6),
_ => raw
.parse::<u32>()
.ok()
.filter(|n| *n <= 7)
.ok_or_else(|| "invalid weekday".into()),
}
}
let mut days = std::collections::BTreeSet::new();
for part in field.split(',') {
let (base, step) = if let Some((base, n)) = part.split_once('/') {
(
base,
n.parse::<u32>()
.ok()
.filter(|n| (1..=7).contains(n))
.ok_or("invalid weekday step")?,
)
} else {
(part, 1)
};
let (start, end) = if base == "*" {
(0, 6)
} else if let Some((a, b)) = base.split_once('-') {
(value(a)?, value(b)?)
} else {
let n = value(base)?;
(n, n)
};
if start > end {
return Err("weekday range must be ascending".into());
}
for n in (start..=end).step_by(step as usize) {
days.insert(n % 7 + 1);
}
}
Ok(days
.into_iter()
.map(|n| n.to_string())
.collect::<Vec<_>>()
.join(","))
}
pub fn next_fire(expr: &str, timezone: &str, from: DateTime<Utc>) -> Result<DateTime<Utc>, String> {
if let Some(seconds) = interval_seconds(expr)? {
return Ok(from + chrono::TimeDelta::seconds(seconds));
}
let schedule = parse_schedule(expr)?;
let tz: Tz = Tz::from_str(timezone).map_err(|_| "invalid IANA timezone")?;
let local = from.with_timezone(&tz);
schedule
.after(&local)
.next()
.map(|when| when.with_timezone(&Utc))
.ok_or_else(|| "cron has no future run".into())
}
pub fn describe_cron(expr: &str) -> String {
if let Ok(Some(seconds)) = interval_seconds(expr) {
return format!("每隔 {} 分鐘(固定間隔)", seconds / 60);
}
let parts: Vec<&str> = expr.trim().split_whitespace().collect();
if parts.len() != 5 {
return expr.to_string();
}
let (min, hour, dom, month, dow) = (parts[0], parts[1], parts[2], parts[3], parts[4]);
if expr.trim() == "* * * * *" {
return "每分鐘".into();
}
if let Some(rest) = min.strip_prefix("*/") {
if hour == "*" && dom == "*" && month == "*" && dow == "*" {
return if rest
.parse::<u32>()
.ok()
.is_some_and(|n| n > 0 && 60 % n == 0)
{
format!("{rest} 分鐘")
} else {
format!("日曆排程:{expr}")
};
}
}
if min == "0" && hour == "*" && dom == "*" && month == "*" && dow == "*" {
return "每小時".into();
}
if min == "0" {
if let Some(rest) = hour.strip_prefix("*/") {
if dom == "*" && month == "*" && dow == "*" {
return if rest
.parse::<u32>()
.ok()
.is_some_and(|n| n > 0 && 24 % n == 0)
{
format!("{rest} 小時")
} else {
format!("日曆排程:{expr}")
};
}
}
}
if min.parse::<u32>().is_ok() && hour.parse::<u32>().is_ok() && month == "*" {
let at = format!("{hour:0>2}:{min:0>2}");
if dom == "*" && dow == "*" {
return format!("每天 {at}");
}
if dom == "*" && dow == "1-5" {
return format!("工作日 {at}");
}
if dom == "*" && dow == "1" {
return format!("每週一 {at}");
}
if dom == "1" && dow == "*" {
return format!("每月 1 日 {at}");
}
}
expr.to_string()
}
#[cfg(test)]
mod tests {
use super::{describe_cron, next_fire, validate_cron};
use chrono::TimeZone;
#[test]
fn weekday_nine_is_valid() {
assert_eq!(validate_cron("0 9 * * 1-5").unwrap(), "0 9 * * 1-5");
assert!(validate_cron("not cron").is_err());
}
#[test]
fn describes_common_patterns() {
assert_eq!(describe_cron("0 9 * * *"), "每天 09:00");
assert_eq!(describe_cron("0 9 * * 1-5"), "工作日 09:00");
assert_eq!(describe_cron("*/15 * * * *"), "每 15 分鐘");
}
#[test]
fn next_fire_is_in_the_future() {
let from = chrono::Utc.with_ymd_and_hms(2026, 9, 5, 0, 0, 0).unwrap();
let next = next_fire("0 9 * * *", "Asia/Taipei", from).unwrap();
assert!(next > from);
}
}
async fn validate_thread(
state: &AppState,
actor: &Actor,
bot: &str,
thread: Option<&str>,
) -> Result<(), String> {
let owns: bool = sqlx::query_scalar(
"SELECT EXISTS(SELECT 1 FROM bots WHERE id=$1 AND space_id=$2 AND user_id=$3)",
)
.bind(bot)
.bind(&actor.space_id)
.bind(&actor.user_id)
.fetch_one(state.pool())
.await
.map_err(|e| e.to_string())?;
if !owns {
return Err("bot not found".into());
}
if let Some(id) = thread {
let valid: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM threads WHERE id=$1 AND bot_id=$2 AND status='active' AND room_id IS NULL)")
.bind(id).bind(bot).fetch_one(state.pool()).await.map_err(|e| e.to_string())?;
if !valid {
return Err("schedule conversation must belong to this bot and be active".into());
}
}
Ok(())
}
fn next_after_due(
expr: &str,
timezone: &str,
due: DateTime<Utc>,
now: DateTime<Utc>,
) -> Result<DateTime<Utc>, String> {
if let Some(seconds) = interval_seconds(expr)? {
let elapsed = (now - due).num_seconds().max(0);
return Ok(due + chrono::TimeDelta::seconds((elapsed / seconds + 1) * seconds));
}
next_fire(expr, timezone, now)
}
#[cfg(test)]
mod regression_tests {
use super::*;
use chrono::{Datelike, TimeZone};
#[test]
fn intervals_cross_month_and_dst_without_reset() {
let from = Utc.with_ymd_and_hms(2026, 1, 31, 9, 0, 0).unwrap();
assert_eq!(
(next_fire("@every 3d", "America/New_York", from).unwrap() - from).num_hours(),
72
);
let dst = Utc.with_ymd_and_hms(2026, 3, 7, 9, 0, 0).unwrap();
assert_eq!(
(next_fire("@every 2d", "America/New_York", dst).unwrap() - dst).num_hours(),
48
);
}
#[test]
fn weekday_means_monday_through_friday() {
let friday = Utc.with_ymd_and_hms(2026, 9, 4, 2, 0, 0).unwrap();
let next = next_fire("0 9 * * 1-5", "Asia/Taipei", friday).unwrap();
assert_eq!(next.weekday(), chrono::Weekday::Mon);
}
#[test]
fn invalid_formats_and_timezones_fail_closed() {
for expr in [
"0 0 9 * * 1",
"99 25 * * *",
"@every 0d",
"@every 366h",
"@every 5z",
] {
assert!(validate_cron(expr).is_err(), "{expr}");
}
assert!(validate_timezone("Asia/Typo").is_err());
}
#[test]
fn missed_intervals_preserve_phase() {
let due = Utc.with_ymd_and_hms(2026, 9, 5, 0, 0, 0).unwrap();
assert_eq!(
next_after_due(
"@every 3m",
"UTC",
due,
due + chrono::TimeDelta::seconds(400)
)
.unwrap(),
due + chrono::TimeDelta::seconds(540)
);
}
}

View File

@ -31,6 +31,18 @@ async fn proxy(state: AppState, bot_id: String, rest: String, req: Request) -> R
.and_then(|value| value.to_str().ok())
.map(|value| value.eq_ignore_ascii_case("websocket"))
.unwrap_or(false);
if !upgrade {
if req.method() != axum::http::Method::GET && req.method() != axum::http::Method::HEAD {
return StatusCode::METHOD_NOT_ALLOWED.into_response();
}
if is_viewer_page(&rest) {
return viewer_page().await;
}
return trusted_asset(&rest).await;
}
if rest != "websockify" {
return StatusCode::NOT_FOUND.into_response();
}
let ensure = upgrade || is_viewer_page(&rest) || rest.contains("websockify");
let port = match upstream_port(&state, &bot_id, ensure).await {
Ok(port) => port,
@ -47,7 +59,7 @@ async fn proxy(state: AppState, bot_id: String, rest: String, req: Request) -> R
if is_viewer_page(&rest) {
return viewer_page().await;
}
http_proxy(port, &rest, req).await
StatusCode::NOT_FOUND.into_response()
}
fn is_viewer_page(rest: &str) -> bool {
@ -71,7 +83,6 @@ async fn viewer_page() -> Response {
"text/html; charset=utf-8".parse().unwrap(),
);
headers.insert(header::CACHE_CONTROL, "no-store".parse().unwrap());
headers.insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
(StatusCode::OK, headers, html).into_response()
}
@ -129,49 +140,36 @@ fn rewrite_upstream(url: &str) -> String {
url.replace("127.0.0.1", &host).replace("localhost", &host)
}
async fn http_proxy(port: u16, rest: &str, req: Request) -> Response {
let host = std::env::var("LAZYBOY_SCREEN_UPSTREAM").unwrap_or_else(|_| "127.0.0.1".into());
let path = if rest.is_empty() {
"vnc_lite.html"
fn safe_asset(rest: &str) -> bool {
(rest.starts_with("core/") || rest.starts_with("vendor/"))
&& rest.ends_with(".js")
&& rest
.split('/')
.all(|p| !p.is_empty() && p != "." && p != "..")
&& rest
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b"/._-".contains(&b))
}
async fn trusted_asset(rest: &str) -> Response {
if !safe_asset(rest) {
return StatusCode::NOT_FOUND.into_response();
}
let root = std::env::var("LAZYBOY_NOVNC_DIR").unwrap_or_else(|_| {
let web = std::env::var("LAZYBOY_WEB_DIR").unwrap_or_else(|_| "apps/web".into());
if std::path::Path::new(&web).join("novnc").is_dir() {
format!("{web}/novnc")
} else {
rest
};
let query = req.uri().query().unwrap_or_default();
let url = if query.is_empty() {
format!("http://{host}:{port}/{path}")
} else {
format!("http://{host}:{port}/{path}?{query}")
};
let client = reqwest::Client::new();
let method = reqwest::Method::from_bytes(req.method().as_str().as_bytes())
.unwrap_or(reqwest::Method::GET);
match client.request(method, url).send().await {
Ok(upstream) => {
let status =
StatusCode::from_u16(upstream.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY);
let content_type = upstream
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.unwrap_or("application/octet-stream")
.to_string();
match upstream.bytes().await {
Ok(bytes) => {
let mut headers = HeaderMap::new();
if let Ok(value) = content_type.parse() {
headers.insert(header::CONTENT_TYPE, value);
"apps/web/node_modules/@novnc/novnc".into()
}
headers.insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
headers.insert(
header::HeaderName::from_static("cross-origin-resource-policy"),
"cross-origin".parse().unwrap(),
);
(status, headers, bytes).into_response()
}
Err(_) => StatusCode::BAD_GATEWAY.into_response(),
}
}
Err(_) => StatusCode::BAD_GATEWAY.into_response(),
});
match tokio::fs::read(std::path::Path::new(&root).join(rest)).await {
Ok(bytes) => (
[(header::CONTENT_TYPE, "text/javascript; charset=utf-8")],
bytes,
)
.into_response(),
Err(_) => StatusCode::NOT_FOUND.into_response(),
}
}
@ -229,3 +227,21 @@ async fn proxy_socket(mut client: WebSocket, port: u16, rest: String) {
}
}
}
#[cfg(test)]
mod asset_tests {
use super::*;
#[test]
fn only_trusted_novnc_scripts_are_served() {
assert!(safe_asset("core/rfb.js"));
for path in [
"../secret.js",
"core/../../secret.js",
"core/%2e%2e/x.js",
"evil.html",
"/core/rfb.js",
] {
assert!(!safe_asset(path));
}
}
}

View File

@ -292,6 +292,7 @@ async fn start_skill(
argv,
cwd: None,
timeout_ms: Some(10_000),
stdin: None,
},
&ctx,
)
@ -902,6 +903,7 @@ async fn stop_recorder(
argv: cdp_record_stop_command(skill_id),
cwd: None,
timeout_ms: Some(5_000),
stdin: None,
},
ctx,
)
@ -928,6 +930,7 @@ async fn collect_browser_events(
],
cwd: None,
timeout_ms: Some(10_000),
stdin: None,
},
ctx,
)

View File

@ -1,3 +1,4 @@
use std::collections::HashMap;
use std::sync::Arc;
use lazyboy_control::SandboxProvider;
@ -10,6 +11,48 @@ use crate::db::{Actor, Db};
use crate::mcp::McpHub;
use crate::memory::MemoryService;
#[derive(Clone, Default)]
pub struct CallRegistry {
inner: Arc<std::sync::Mutex<HashMap<String, String>>>,
}
impl CallRegistry {
/// Returns a lease that releases the slot when dropped, so a websocket
/// upgrade that never completes cannot leave the bot marked as busy.
pub fn try_begin(&self, bot_id: &str, call_id: &str) -> Option<CallLease> {
let mut map = self.lock();
if map.contains_key(bot_id) {
return None;
}
map.insert(bot_id.to_string(), call_id.to_string());
drop(map);
Some(CallLease {
registry: self.clone(),
bot_id: bot_id.to_string(),
call_id: call_id.to_string(),
})
}
fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, String>> {
self.inner.lock().unwrap_or_else(|error| error.into_inner())
}
}
pub struct CallLease {
registry: CallRegistry,
bot_id: String,
call_id: String,
}
impl Drop for CallLease {
fn drop(&mut self) {
let mut map = self.registry.lock();
if map.get(&self.bot_id).is_some_and(|held| *held == self.call_id) {
map.remove(&self.bot_id);
}
}
}
#[derive(Clone)]
pub struct AppState {
pub db: Db,
@ -18,6 +61,7 @@ pub struct AppState {
pub auth: AuthConfig,
pub memory: MemoryService,
pub mcp: McpHub,
pub calls: CallRegistry,
}
impl AppState {
@ -39,6 +83,7 @@ impl AppState {
auth: AuthConfig::from_env(),
memory: MemoryService::from_env(),
mcp: McpHub::new(),
calls: CallRegistry::default(),
})
}

File diff suppressed because it is too large Load Diff

383
crates/api/src/vault.rs Normal file
View File

@ -0,0 +1,383 @@
//! Per-bot login vault. Humans store credentials; the model only sees ids.
use aes_gcm::aead::{Aead, KeyInit};
use aes_gcm::{Aes256Gcm, Nonce};
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::routing::{delete, get, patch, post};
use axum::{Json, Router};
use chrono::{DateTime, Utc};
use rand::RngCore;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
use sqlx::FromRow;
use uuid::Uuid;
use crate::db::Actor;
use crate::state::AppState;
#[derive(Debug, Clone, Serialize, FromRow)]
#[serde(rename_all = "camelCase")]
pub struct VaultAccount {
pub id: String,
pub bot_id: String,
pub site: String,
pub host: String,
pub username: String,
pub notes: String,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct UpsertAccount {
pub site: String,
#[serde(default)]
pub host: String,
pub username: String,
#[serde(default)]
pub password: String,
#[serde(default)]
pub notes: String,
}
pub fn router() -> Router<AppState> {
Router::new()
.route(
"/api/bots/{bot_id}/accounts",
get(list_accounts).post(create_account),
)
.route(
"/api/bots/{bot_id}/accounts/{account_id}",
patch(update_account).delete(delete_account),
)
}
async fn scoped_actor(state: &AppState, bot_id: &str) -> Result<Actor, StatusCode> {
let actor = state
.bootstrap()
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
state
.db
.get_bot(&actor, bot_id)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
.ok_or(StatusCode::NOT_FOUND)?;
Ok(actor)
}
async fn list_accounts(
State(state): State<AppState>,
Path(bot_id): Path<String>,
) -> Result<Json<Vec<VaultAccount>>, StatusCode> {
let actor = scoped_actor(&state, &bot_id).await?;
list(&state, &actor, &bot_id)
.await
.map(Json)
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)
}
async fn create_account(
State(state): State<AppState>,
Path(bot_id): Path<String>,
Json(input): Json<UpsertAccount>,
) -> Result<Json<VaultAccount>, (StatusCode, Json<Value>)> {
let actor = scoped_actor(&state, &bot_id)
.await
.map_err(|status| (status, Json(json!({"message":"bot not found"}))))?;
insert(&state, &actor, &bot_id, input)
.await
.map(Json)
.map_err(|error| (StatusCode::BAD_REQUEST, Json(json!({"message": error}))))
}
async fn update_account(
State(state): State<AppState>,
Path((bot_id, account_id)): Path<(String, String)>,
Json(input): Json<UpsertAccount>,
) -> Result<Json<VaultAccount>, (StatusCode, Json<Value>)> {
let actor = scoped_actor(&state, &bot_id)
.await
.map_err(|status| (status, Json(json!({"message":"bot not found"}))))?;
update_row(&state, &actor, &bot_id, &account_id, input)
.await
.map_err(|error| (StatusCode::BAD_REQUEST, Json(json!({"message": error}))))?
.map(Json)
.ok_or((
StatusCode::NOT_FOUND,
Json(json!({"message":"account not found"})),
))
}
async fn delete_account(
State(state): State<AppState>,
Path((bot_id, account_id)): Path<(String, String)>,
) -> Result<StatusCode, StatusCode> {
let actor = scoped_actor(&state, &bot_id).await?;
let deleted = sqlx::query(
"DELETE FROM vault_accounts
WHERE id=$1 AND bot_id=$2 AND space_id=$3 AND user_id=$4",
)
.bind(&account_id)
.bind(&bot_id)
.bind(&actor.space_id)
.bind(&actor.user_id)
.execute(state.pool())
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
if deleted.rows_affected() == 0 {
return Err(StatusCode::NOT_FOUND);
}
Ok(StatusCode::NO_CONTENT)
}
pub async fn list(
state: &AppState,
actor: &Actor,
bot_id: &str,
) -> Result<Vec<VaultAccount>, String> {
list_on(state.pool(), actor, bot_id).await
}
pub async fn list_on(
pool: &sqlx::PgPool,
actor: &Actor,
bot_id: &str,
) -> Result<Vec<VaultAccount>, String> {
sqlx::query_as(
"SELECT id, bot_id, site, host, username, notes, created_at, updated_at
FROM vault_accounts
WHERE bot_id=$1 AND space_id=$2 AND user_id=$3
ORDER BY updated_at DESC",
)
.bind(bot_id)
.bind(&actor.space_id)
.bind(&actor.user_id)
.fetch_all(pool)
.await
.map_err(|error| error.to_string())
}
pub async fn get_secret(
state: &AppState,
actor: &Actor,
bot_id: &str,
account_id: &str,
) -> Result<Option<(VaultAccount, String, String)>, String> {
get_secret_on(state.pool(), actor, bot_id, account_id).await
}
pub async fn get_secret_on(
pool: &sqlx::PgPool,
actor: &Actor,
bot_id: &str,
account_id: &str,
) -> Result<Option<(VaultAccount, String, String)>, String> {
let row: Option<(String, String, String, String, String, String, DateTime<Utc>, DateTime<Utc>, String)> =
sqlx::query_as(
"SELECT id, bot_id, site, host, username, notes, created_at, updated_at, password_ciphertext
FROM vault_accounts
WHERE id=$1 AND bot_id=$2 AND space_id=$3 AND user_id=$4",
)
.bind(account_id)
.bind(bot_id)
.bind(&actor.space_id)
.bind(&actor.user_id)
.fetch_optional(pool)
.await
.map_err(|error| error.to_string())?;
let Some(row) = row else {
return Ok(None);
};
let password = decrypt(&row.8)?;
Ok(Some((
VaultAccount {
id: row.0,
bot_id: row.1,
site: row.2,
host: row.3,
username: row.4.clone(),
notes: row.5,
created_at: row.6,
updated_at: row.7,
},
row.4,
password,
)))
}
async fn insert(
state: &AppState,
actor: &Actor,
bot_id: &str,
input: UpsertAccount,
) -> Result<VaultAccount, String> {
let site = clean_site(&input.site)?;
let username = clean_username(&input.username)?;
if input.password.is_empty() {
return Err("password is required".into());
}
let host = normalize_host(&input.host, &site);
let id = Uuid::new_v4().to_string();
let ciphertext = encrypt(&input.password)?;
sqlx::query_as(
"INSERT INTO vault_accounts
(id, space_id, user_id, bot_id, site, host, username, password_ciphertext, notes)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
RETURNING id, bot_id, site, host, username, notes, created_at, updated_at",
)
.bind(&id)
.bind(&actor.space_id)
.bind(&actor.user_id)
.bind(bot_id)
.bind(site)
.bind(host)
.bind(username)
.bind(ciphertext)
.bind(input.notes.trim())
.fetch_one(state.pool())
.await
.map_err(|error| error.to_string())
}
async fn update_row(
state: &AppState,
actor: &Actor,
bot_id: &str,
account_id: &str,
input: UpsertAccount,
) -> Result<Option<VaultAccount>, String> {
let site = clean_site(&input.site)?;
let username = clean_username(&input.username)?;
let host = normalize_host(&input.host, &site);
if input.password.is_empty() {
sqlx::query_as(
"UPDATE vault_accounts
SET site=$1, host=$2, username=$3, notes=$4, updated_at=now()
WHERE id=$5 AND bot_id=$6 AND space_id=$7 AND user_id=$8
RETURNING id, bot_id, site, host, username, notes, created_at, updated_at",
)
.bind(site)
.bind(host)
.bind(username)
.bind(input.notes.trim())
.bind(account_id)
.bind(bot_id)
.bind(&actor.space_id)
.bind(&actor.user_id)
.fetch_optional(state.pool())
.await
.map_err(|error| error.to_string())
} else {
let ciphertext = encrypt(&input.password)?;
sqlx::query_as(
"UPDATE vault_accounts
SET site=$1, host=$2, username=$3, notes=$4, password_ciphertext=$5, updated_at=now()
WHERE id=$6 AND bot_id=$7 AND space_id=$8 AND user_id=$9
RETURNING id, bot_id, site, host, username, notes, created_at, updated_at",
)
.bind(site)
.bind(host)
.bind(username)
.bind(input.notes.trim())
.bind(ciphertext)
.bind(account_id)
.bind(bot_id)
.bind(&actor.space_id)
.bind(&actor.user_id)
.fetch_optional(state.pool())
.await
.map_err(|error| error.to_string())
}
}
fn clean_site(site: &str) -> Result<String, String> {
let value = site.trim();
if value.is_empty() || value.chars().count() > 80 {
return Err("site name must be 180 characters".into());
}
Ok(value.to_string())
}
fn clean_username(username: &str) -> Result<String, String> {
let value = username.trim();
if value.is_empty() || value.chars().count() > 200 {
return Err("username must be 1200 characters".into());
}
Ok(value.to_string())
}
fn normalize_host(host: &str, site: &str) -> String {
let raw = host.trim();
if raw.is_empty() {
return site.to_lowercase();
}
raw.trim_start_matches("https://")
.trim_start_matches("http://")
.split('/')
.next()
.unwrap_or(raw)
.trim()
.to_lowercase()
}
fn vault_key() -> Result<[u8; 32], String> {
let material = std::env::var("LAZYBOY_VAULT_KEY")
.ok()
.filter(|value| !value.is_empty())
.or_else(|| std::env::var("LAZYBOY_APP_TOKEN").ok().filter(|v| !v.is_empty()))
.ok_or_else(|| "set LAZYBOY_VAULT_KEY or LAZYBOY_APP_TOKEN to encrypt saved passwords".to_string())?;
let digest = Sha256::digest(material.as_bytes());
let mut key = [0u8; 32];
key.copy_from_slice(&digest);
Ok(key)
}
fn encrypt(plaintext: &str) -> Result<String, String> {
let cipher = Aes256Gcm::new_from_slice(&vault_key()?).map_err(|error| error.to_string())?;
let mut nonce_bytes = [0u8; 12];
rand::thread_rng().fill_bytes(&mut nonce_bytes);
let nonce = Nonce::from_slice(&nonce_bytes);
let encrypted = cipher
.encrypt(nonce, plaintext.as_bytes())
.map_err(|error| error.to_string())?;
let mut packed = Vec::with_capacity(12 + encrypted.len());
packed.extend_from_slice(&nonce_bytes);
packed.extend_from_slice(&encrypted);
Ok(hex::encode(packed))
}
fn decrypt(packed: &str) -> Result<String, String> {
let bytes = hex::decode(packed).map_err(|error| error.to_string())?;
if bytes.len() < 13 {
return Err("corrupt vault entry".into());
}
let cipher = Aes256Gcm::new_from_slice(&vault_key()?).map_err(|error| error.to_string())?;
let nonce = Nonce::from_slice(&bytes[..12]);
let plain = cipher
.decrypt(nonce, &bytes[12..])
.map_err(|_| "could not decrypt vault entry".to_string())?;
String::from_utf8(plain).map_err(|error| error.to_string())
}
#[cfg(test)]
mod tests {
use super::{decrypt, encrypt, normalize_host};
#[test]
fn round_trips_a_password() {
unsafe { std::env::set_var("LAZYBOY_VAULT_KEY", "test-vault-key-for-unit-tests") };
let packed = encrypt("s3cret!").unwrap();
assert!(!packed.contains("s3cret"));
assert_eq!(decrypt(&packed).unwrap(), "s3cret!");
}
#[test]
fn host_strips_urls() {
assert_eq!(normalize_host("https://mail.google.com/inbox", "Gmail"), "mail.google.com");
assert_eq!(normalize_host("", "Gmail"), "gmail");
}
}

297
crates/api/src/voice.rs Normal file
View File

@ -0,0 +1,297 @@
use axum::extract::State;
use axum::http::StatusCode;
use axum::routing::get;
use axum::{Json, Router};
use lazyboy_contracts::{VoiceProvider, catalog_voices, computer_voice_tools};
use lazyboy_harness::{
CredentialChain, ResolveVoiceRequest, resolve_voice, scripted_voice_enabled, voice_catalog,
};
use serde::Deserialize;
use serde_json::{Value, json};
use crate::db::{Actor, SpaceRow};
use crate::state::AppState;
use crate::voice_call;
pub fn router() -> Router<AppState> {
Router::new()
.route("/api/voice/settings", get(get_settings).patch(update_settings))
.route("/api/sessions/{id}/call", get(voice_call::call_ws))
}
async fn actor(state: &AppState) -> Result<Actor, StatusCode> {
state
.bootstrap()
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)
}
pub fn voice_credential_chain(
provider: VoiceProvider,
voice_api_key: Option<&str>,
text_provider: &str,
text_api_key: Option<&str>,
env_key: Option<&str>,
) -> CredentialChain {
let nonempty = |value: Option<&str>| {
value
.map(str::trim)
.filter(|item| !item.is_empty())
.map(str::to_string)
};
let space = nonempty(voice_api_key).or_else(|| {
if text_provider == provider.as_str() {
nonempty(text_api_key)
} else {
None
}
});
CredentialChain {
bot: None,
space,
env: nonempty(env_key),
}
}
pub fn voice_instructions(bot_name: &str, bot_instructions: &str) -> String {
let mut text = format!(
"You are {bot_name}, on a live voice call. Speak shortly and naturally in the user's language. Do not read markdown aloud.\n\
You cannot see the screen. Computer progress arrives as text from tools.\n\
If the user wants you to operate the computer, answer yes or no immediately, then call start_computer_task. Never pretend you already clicked.\n\
If a task is already running and they change their mind, call follow_up_computer_task or stop_computer_task.\n\
If a tool returns blocked because they have the screen, tell them to finish on the right-hand desktop."
);
let extra = bot_instructions.trim();
if !extra.is_empty() {
text.push_str("\n\nBot instructions:\n");
text.push_str(extra);
}
text
}
fn listed_providers() -> Vec<VoiceProvider> {
let mut providers = VoiceProvider::selectable().to_vec();
if scripted_voice_enabled() {
providers.push(VoiceProvider::Scripted);
}
providers
}
fn settings_json(space: &SpaceRow) -> Value {
let requested = space
.voice_provider
.as_deref()
.unwrap_or("xai")
.parse::<VoiceProvider>()
.unwrap_or(VoiceProvider::Xai);
let provider = if requested == VoiceProvider::Scripted && !scripted_voice_enabled() {
VoiceProvider::Xai
} else {
requested
};
let env_key = std::env::var(provider.env_key_name())
.ok()
.filter(|value| !value.is_empty());
let credentials = voice_credential_chain(
provider,
space.voice_api_key.as_deref(),
&space.default_model_provider,
space.default_model_api_key.as_deref(),
env_key.as_deref(),
);
let resolved = resolve_voice(ResolveVoiceRequest {
provider,
model_id: space.voice_model_id.clone(),
voice_id: space.voice_id.clone(),
credentials,
});
let (model_id, voice_id, ready, missing) = match &resolved {
Ok(value) => (value.model_id.clone(), value.voice_id.clone(), true, None),
Err(error) => (
space
.voice_model_id
.clone()
.filter(|value| !value.is_empty())
.unwrap_or_else(|| provider.default_model_id().to_string()),
space
.voice_id
.clone()
.filter(|value| !value.is_empty())
.unwrap_or_else(|| provider.default_voice_id().to_string()),
false,
Some(error.to_string()),
),
};
let catalog = voice_catalog(provider);
json!({
"provider": provider.as_str(),
"modelId": model_id,
"voiceId": voice_id,
"enabled": space.voice_enabled,
"ready": space.voice_enabled && ready,
"missing": missing,
"apiKeySet": space.voice_api_key.as_deref().is_some_and(|value| !value.is_empty()),
"envKeySet": env_key.is_some(),
"envKeyName": provider.env_key_name(),
"reusesTextKey": provider.as_str() == space.default_model_provider
&& space.voice_api_key.as_deref().is_none_or(|value| value.is_empty())
&& space.default_model_api_key.as_deref().is_some_and(|value| !value.is_empty()),
"providers": listed_providers().iter().map(|item| json!({
"id": item.as_str(),
"name": match item {
VoiceProvider::Xai => "xAI",
VoiceProvider::Openai => "OpenAI",
VoiceProvider::Scripted => "Scripted",
},
"envKeyName": item.env_key_name(),
})).collect::<Vec<_>>(),
"models": catalog.models.iter().map(|item| json!({"id": item.id, "name": item.name})).collect::<Vec<_>>(),
"voices": catalog.voices.iter().map(|item| json!({"id": item.id, "name": item.name})).collect::<Vec<_>>(),
"tools": computer_voice_tools(),
})
}
async fn get_settings(State(state): State<AppState>) -> Result<Json<Value>, StatusCode> {
let actor = actor(&state).await?;
let space = state
.db
.get_space(&actor)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
.ok_or(StatusCode::NOT_FOUND)?;
Ok(Json(settings_json(&space)))
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct UpdateVoiceSettings {
#[serde(default)]
enabled: Option<bool>,
provider: String,
model_id: String,
voice_id: String,
#[serde(default)]
api_key: Option<String>,
#[serde(default)]
clear_api_key: bool,
}
async fn update_settings(
State(state): State<AppState>,
Json(input): Json<UpdateVoiceSettings>,
) -> Result<Json<Value>, StatusCode> {
let actor = actor(&state).await?;
let provider: VoiceProvider = input
.provider
.parse()
.map_err(|_| StatusCode::BAD_REQUEST)?;
if !listed_providers().contains(&provider) {
return Err(StatusCode::BAD_REQUEST);
}
let model_id = input.model_id.trim();
let voice_id = input.voice_id.trim();
if model_id.is_empty() || voice_id.is_empty() {
return Err(StatusCode::BAD_REQUEST);
}
if catalog_voices(provider).iter().all(|(id, _)| *id != voice_id)
&& provider != VoiceProvider::Scripted
{
return Err(StatusCode::BAD_REQUEST);
}
let supplied = input
.api_key
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty());
let api_key = if input.clear_api_key {
Some(None)
} else {
supplied.map(Some)
};
state
.db
.update_voice_settings(&actor, input.enabled, provider.as_str(), model_id, voice_id, api_key)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
get_settings(State(state)).await
}
pub fn resolve_space_voice(
space: &SpaceRow,
) -> Result<(VoiceProvider, lazyboy_harness::ResolvedVoice), String> {
if !space.voice_enabled {
return Err("語音通話尚未啟用,請到「設定 → 語音」開啟。".into());
}
let requested = space
.voice_provider
.as_deref()
.unwrap_or("xai")
.parse::<VoiceProvider>()
.unwrap_or(VoiceProvider::Xai);
let provider = if requested == VoiceProvider::Scripted && !scripted_voice_enabled() {
VoiceProvider::Xai
} else {
requested
};
let env_key = std::env::var(provider.env_key_name())
.ok()
.filter(|value| !value.is_empty());
let credentials = voice_credential_chain(
provider,
space.voice_api_key.as_deref(),
&space.default_model_provider,
space.default_model_api_key.as_deref(),
env_key.as_deref(),
);
let resolved = resolve_voice(ResolveVoiceRequest {
provider,
model_id: space.voice_model_id.clone(),
voice_id: space.voice_id.clone(),
credentials,
})
.map_err(|error| error.to_string())?;
Ok((provider, resolved))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn voice_key_reuses_matching_text_provider_and_does_not_cross_providers() {
let reused = voice_credential_chain(
VoiceProvider::Xai,
None,
"xai",
Some("text-key"),
Some("env-key"),
);
assert_eq!(reused.resolve(), Some("text-key"));
let env_only = voice_credential_chain(
VoiceProvider::Xai,
None,
"opencode-go",
Some("go-key"),
Some("env-xai"),
);
assert_eq!(env_only.resolve(), Some("env-xai"));
let dedicated = voice_credential_chain(
VoiceProvider::Openai,
Some("voice-openai"),
"xai",
Some("text-xai"),
Some("env-openai"),
);
assert_eq!(dedicated.resolve(), Some("voice-openai"));
}
#[test]
fn instructions_include_bot_name_and_custom_text() {
let text = voice_instructions("阿明", "Always prefer the browser.");
assert!(text.contains("阿明"));
assert!(text.contains("start_computer_task"));
assert!(text.contains("Always prefer the browser."));
}
}

View File

@ -0,0 +1,657 @@
use std::sync::Arc;
use std::time::Duration;
use axum::extract::ws::{Message as AxumMessage, WebSocket, WebSocketUpgrade};
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::response::IntoResponse;
use axum::Json;
use futures_util::{SinkExt, StreamExt};
use lazyboy_contracts::SessionAttachment;
use lazyboy_harness::{VoiceConnectRequest, VoiceEvent, VoiceSocket, create_voice};
use serde_json::{Value, json};
use tokio::sync::Mutex;
use uuid::Uuid;
use crate::db::Actor;
use crate::state::AppState;
use crate::voice::{resolve_space_voice, voice_instructions};
struct PreparedCall {
call_id: String,
bot_id: String,
bot_name: String,
provider: lazyboy_contracts::VoiceProvider,
connect: VoiceConnectRequest,
}
pub async fn call_ws(
State(state): State<AppState>,
Path(session_id): Path<String>,
ws: WebSocketUpgrade,
) -> axum::response::Response {
let actor = match state.bootstrap().await {
Ok(actor) => actor,
Err(_) => {
return (
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"message":"internal error"})),
)
.into_response();
}
};
match prepare_call(&state, &actor, &session_id).await {
Ok(prep) => {
let Some(lease) = state.calls.try_begin(&prep.bot_id, &prep.call_id) else {
return (
StatusCode::CONFLICT,
Json(json!({"message":"already on a call"})),
)
.into_response();
};
let provider = create_voice(prep.provider);
let socket = match provider.connect(prep.connect.clone()).await {
Ok(socket) => socket,
Err(error) => {
return (
StatusCode::BAD_GATEWAY,
Json(json!({"message": error.to_string()})),
)
.into_response();
}
};
ws.on_upgrade(move |client| {
run_call(state, actor, session_id, prep, client, socket, lease)
})
.into_response()
}
Err((status, message)) => (status, Json(json!({"message": message}))).into_response(),
}
}
/// The client asks for a barge-in over the text channel; audio frames stay binary.
fn is_interrupt_request(text: &str) -> bool {
serde_json::from_str::<Value>(text)
.ok()
.and_then(|value| {
value
.get("type")
.and_then(Value::as_str)
.map(str::to_string)
})
.is_some_and(|kind| kind == "interrupt")
}
async fn prepare_call(
state: &AppState,
actor: &Actor,
session_id: &str,
) -> Result<PreparedCall, (StatusCode, String)> {
let row: Option<(String, Option<String>, String, String)> = sqlx::query_as(
"SELECT t.bot_id, t.room_id, b.name, b.instructions
FROM threads t JOIN bots b ON b.id=t.bot_id
WHERE t.id=$1 AND t.space_id=$2 AND t.user_id=$3 AND t.status='active'",
)
.bind(session_id)
.bind(&actor.space_id)
.bind(&actor.user_id)
.fetch_optional(state.pool())
.await
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?;
let Some((bot_id, room_id, bot_name, bot_instructions)) = row else {
return Err((StatusCode::NOT_FOUND, "session not found".into()));
};
if room_id.is_some() {
return Err((
StatusCode::BAD_REQUEST,
"group calls are not supported".into(),
));
}
let space = state
.db
.get_space(actor)
.await
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?
.ok_or((StatusCode::NOT_FOUND, "workspace not found".into()))?;
let (provider, resolved) =
resolve_space_voice(&space).map_err(|message| (StatusCode::CONFLICT, message))?;
let history = recent_text_history(state, session_id).await.unwrap_or_default();
let mut connect = VoiceConnectRequest::from_resolved(
&resolved,
voice_instructions(&bot_name, &bot_instructions),
);
connect.history = history;
Ok(PreparedCall {
call_id: Uuid::new_v4().to_string(),
bot_id,
bot_name,
provider,
connect,
})
}
async fn recent_text_history(
state: &AppState,
session_id: &str,
) -> Result<Vec<(String, String)>, sqlx::Error> {
let rows: Vec<(String, String)> = sqlx::query_as(
"SELECT role, body FROM messages
WHERE thread_id=$1 AND role IN ('user','assistant') AND body <> ''
ORDER BY seq DESC LIMIT 10",
)
.bind(session_id)
.fetch_all(state.pool())
.await?;
Ok(rows.into_iter().rev().collect())
}
async fn run_call(
state: AppState,
actor: Actor,
session_id: String,
prep: PreparedCall,
client: WebSocket,
mut provider: Box<dyn VoiceSocket>,
_lease: crate::state::CallLease,
) {
let call_id = prep.call_id.clone();
let bot_id = prep.bot_id.clone();
let (client_write, mut client_read) = client.split();
let client_write = Arc::new(Mutex::new(client_write));
let _ = send_json(
&client_write,
json!({
"type": "ready",
"callId": call_id,
"botId": bot_id,
"botName": prep.bot_name,
"provider": prep.provider.as_str(),
"voice": prep.connect.voice_id,
}),
)
.await;
let mut user_partial = String::new();
let mut assistant_partial = String::new();
let mut last_progress = String::new();
let mut response_active = false;
let mut last_spoken_at = std::time::Instant::now()
.checked_sub(Duration::from_secs(30))
.unwrap_or_else(std::time::Instant::now);
// A fresh `sleep` inside `select!` would be cancelled by every microphone
// frame (one every few milliseconds), so the poll must own its own timer.
let mut watchdog = tokio::time::interval(Duration::from_millis(800));
watchdog.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
tokio::select! {
client_msg = client_read.next() => {
match client_msg {
None | Some(Ok(AxumMessage::Close(_))) => break,
Some(Err(_)) => break,
Some(Ok(AxumMessage::Ping(payload))) => {
let _ = client_write.lock().await.send(AxumMessage::Pong(payload)).await;
}
Some(Ok(AxumMessage::Pong(_))) => {}
Some(Ok(AxumMessage::Binary(bytes))) => {
if provider.send(VoiceEvent::AudioPcm(bytes.to_vec())).await.is_err() {
break;
}
}
Some(Ok(AxumMessage::Text(text))) => {
if is_interrupt_request(&text) && response_active {
response_active = false;
if provider.send(VoiceEvent::CancelResponse).await.is_err() {
break;
}
}
}
}
}
provider_msg = provider.recv() => {
match provider_msg {
Ok(None) | Err(_) => break,
Ok(Some(event)) => {
if handle_provider_event(
&state,
&actor,
&session_id,
&prep,
&mut provider,
&client_write,
event,
&mut user_partial,
&mut assistant_partial,
&mut response_active,
)
.await
.is_err()
{
break;
}
}
}
}
_ = watchdog.tick() => {
let _ = watch_computer(
&state,
&actor,
&session_id,
&bot_id,
prep.provider,
&mut provider,
&client_write,
&mut last_progress,
&mut last_spoken_at,
)
.await;
}
}
}
}
async fn handle_provider_event(
state: &AppState,
actor: &Actor,
session_id: &str,
prep: &PreparedCall,
provider: &mut Box<dyn VoiceSocket>,
client_write: &Arc<Mutex<futures_util::stream::SplitSink<WebSocket, AxumMessage>>>,
event: VoiceEvent,
user_partial: &mut String,
assistant_partial: &mut String,
response_active: &mut bool,
) -> Result<(), ()> {
match event {
VoiceEvent::ResponseStarted => *response_active = true,
VoiceEvent::ResponseFinished => *response_active = false,
VoiceEvent::AudioPcm(bytes) => {
// Not every provider announces `response.created`; flowing audio is
// proof enough that there is something to cancel on a barge-in.
*response_active = true;
client_write
.lock()
.await
.send(AxumMessage::Binary(bytes.into()))
.await
.map_err(|_| ())?;
}
VoiceEvent::SpeechStarted => {
send_json(client_write, json!({"type":"speech","state":"started"})).await?;
}
VoiceEvent::SpeechStopped => {
send_json(client_write, json!({"type":"speech","state":"stopped"})).await?;
}
VoiceEvent::InputTranscript { text, final_ } => {
if final_ {
let body = if text.trim().is_empty() {
user_partial.trim().to_string()
} else {
text.trim().to_string()
};
user_partial.clear();
if !body.is_empty() {
let _ = persist_transcript(state, session_id, "user", &body, &prep.call_id).await;
send_json(
client_write,
json!({"type":"transcript","role":"user","text":body,"final":true}),
)
.await?;
}
} else {
user_partial.push_str(&text);
send_json(
client_write,
json!({"type":"transcript","role":"user","text":user_partial,"final":false}),
)
.await?;
}
}
VoiceEvent::OutputTranscript { text, final_ } => {
if final_ {
*response_active = false;
let body = if text.trim().is_empty() {
assistant_partial.trim().to_string()
} else {
text.trim().to_string()
};
assistant_partial.clear();
if !body.is_empty() {
let _ =
persist_transcript(state, session_id, "assistant", &body, &prep.call_id)
.await;
send_json(
client_write,
json!({"type":"transcript","role":"assistant","text":body,"final":true}),
)
.await?;
}
} else {
assistant_partial.push_str(&text);
send_json(
client_write,
json!({"type":"transcript","role":"assistant","text":assistant_partial,"final":false}),
)
.await?;
}
}
VoiceEvent::FunctionCall {
call_id,
name,
arguments,
} => {
let output = dispatch_voice_tool(state, actor, &prep.bot_id, session_id, &prep.call_id, &name, &arguments)
.await;
let _ = provider
.send(VoiceEvent::FunctionCallOutput {
call_id,
output: output.to_string(),
})
.await;
tokio::time::sleep(Duration::from_millis(250)).await;
let _ = provider.send(VoiceEvent::ResponseCreate).await;
if let Some(status) = output.get("status").and_then(Value::as_str) {
send_json(
client_write,
json!({
"type":"computer",
"status": status,
"step": output.get("step"),
"takeover": output.get("takeover").and_then(Value::as_bool).unwrap_or(false)
}),
)
.await?;
}
}
VoiceEvent::Error { message } => {
send_json(client_write, json!({"type":"error","message":message})).await?;
}
_ => {}
}
Ok(())
}
async fn dispatch_voice_tool(
state: &AppState,
actor: &Actor,
bot_id: &str,
session_id: &str,
call_id: &str,
name: &str,
arguments: &str,
) -> Value {
match name {
"computer_status" => computer_tool_status(state, actor, bot_id, session_id).await,
"stop_computer_task" => match crate::sessions::cancel_session_runs(state, session_id).await
{
Ok(ids) => json!({"status":"cancelled","runIds": ids}),
Err(error) => json!({"status":"error","reason": error}),
},
"start_computer_task" | "follow_up_computer_task" => {
let prompt = serde_json::from_str::<Value>(arguments)
.ok()
.and_then(|value| {
value
.get("prompt")
.and_then(Value::as_str)
.map(str::to_string)
})
.unwrap_or_default();
let prompt = prompt.trim().to_string();
if prompt.is_empty() {
return json!({"status":"error","reason":"missing prompt"});
}
start_or_follow(state, actor, bot_id, session_id, call_id, &prompt).await
}
other => json!({"status":"error","reason": format!("unknown tool {other}")}),
}
}
async fn start_or_follow(
state: &AppState,
actor: &Actor,
bot_id: &str,
session_id: &str,
call_id: &str,
prompt: &str,
) -> Value {
if crate::skills::recording_skill(state.pool(), bot_id)
.await
.is_some()
{
return json!({"status":"blocked","reason":"teaching"});
}
let status = computer_tool_status(state, actor, bot_id, session_id).await;
if status
.get("takeover")
.and_then(Value::as_bool)
.unwrap_or(false)
|| status.get("status").and_then(Value::as_str) == Some("takeover")
{
return json!({"status":"blocked","reason":"takeover","takeover":true});
}
if status.get("status").and_then(Value::as_str) == Some("user_control") {
return json!({"status":"blocked","reason":"user_control"});
}
let blocks = vec![json!({"kind":"voice","callId": call_id})];
match crate::runs::send(
state,
actor,
bot_id,
session_id,
prompt,
Some(&Uuid::new_v4().to_string()),
&blocks,
&[] as &[SessionAttachment],
)
.await
{
Ok(result) => json!({
"status": if result.get("queuedBehindActive").and_then(Value::as_bool).unwrap_or(false) {
"followed_up"
} else {
"queued"
},
"runId": result.get("runId"),
}),
Err(error) => json!({"status":"error","reason": error}),
}
}
async fn computer_tool_status(
state: &AppState,
actor: &Actor,
bot_id: &str,
session_id: &str,
) -> Value {
let holder: Option<String> = sqlx::query_scalar(
"SELECT c.control_holder FROM computers c
JOIN bots b ON b.computer_id=c.id
WHERE b.id=$1 AND b.space_id=$2 AND b.user_id=$3",
)
.bind(bot_id)
.bind(&actor.space_id)
.bind(&actor.user_id)
.fetch_optional(state.pool())
.await
.ok()
.flatten();
if holder.as_deref() == Some("user") {
return json!({"status":"user_control","takeover":false});
}
let row: Option<(String, String, Option<String>)> = sqlx::query_as(
"SELECT id, status, checkpoint->>'step'
FROM runs
WHERE bot_id=$1 AND thread_id=$2
AND status IN ('queued','leased','running','waiting_input','waiting_takeover')
ORDER BY CASE status
WHEN 'running' THEN 0 WHEN 'leased' THEN 1
WHEN 'waiting_takeover' THEN 2 WHEN 'waiting_input' THEN 3
ELSE 4 END,
created_at ASC
LIMIT 1",
)
.bind(bot_id)
.bind(session_id)
.fetch_optional(state.pool())
.await
.ok()
.flatten();
match row {
Some((run_id, status, step)) if status == "waiting_takeover" => json!({
"status":"takeover",
"runId": run_id,
"step": step,
"takeover": true
}),
Some((run_id, status, step)) => json!({
"status": status,
"runId": run_id,
"step": step,
"takeover": false
}),
None => json!({"status":"idle","takeover":false}),
}
}
async fn watch_computer(
state: &AppState,
actor: &Actor,
session_id: &str,
bot_id: &str,
voice_provider: lazyboy_contracts::VoiceProvider,
provider: &mut Box<dyn VoiceSocket>,
client_write: &Arc<Mutex<futures_util::stream::SplitSink<WebSocket, AxumMessage>>>,
last_progress: &mut String,
last_spoken_at: &mut std::time::Instant,
) -> Result<(), ()> {
let snapshot = computer_tool_status(state, actor, bot_id, session_id).await;
let key = snapshot.to_string();
if key == *last_progress {
return Ok(());
}
let previous = last_progress.clone();
*last_progress = key;
send_json(
client_write,
json!({
"type":"computer",
"status": snapshot.get("status"),
"step": snapshot.get("step"),
"takeover": snapshot.get("takeover")
}),
)
.await?;
if let Some(line) = speakable_progress(&previous, &snapshot) {
let urgent = snapshot.get("status").and_then(Value::as_str)
== Some("takeover")
|| snapshot.get("status").and_then(Value::as_str) == Some("failed");
if urgent || last_spoken_at.elapsed() > Duration::from_secs(6) {
let _ = provider.send(VoiceEvent::SpeakNow { text: line }).await;
if voice_provider == lazyboy_contracts::VoiceProvider::Openai {
let _ = provider.send(VoiceEvent::ResponseCreate).await;
}
*last_spoken_at = std::time::Instant::now();
}
}
Ok(())
}
pub fn speakable_progress(previous: &str, snapshot: &Value) -> Option<String> {
let status = snapshot.get("status")?.as_str()?;
if previous.contains(&format!("\"status\":\"{status}\"")) {
return None;
}
match status {
"takeover" => Some("需要你接手畫面登入或驗證。".into()),
"user_control" => Some("畫面現在在你手上,完成後再叫我繼續。".into()),
"failed" => Some("這次電腦操作沒做成。".into()),
"idle" if previous.contains("running") || previous.contains("queued") => {
Some("電腦上的工作做完了。".into())
}
_ => None,
}
}
async fn persist_transcript(
state: &AppState,
session_id: &str,
role: &str,
body: &str,
call_id: &str,
) -> Result<(), String> {
let mut tx = state
.pool()
.begin()
.await
.map_err(|error| error.to_string())?;
let seq: i32 = sqlx::query_scalar(
"UPDATE threads SET next_message_seq=next_message_seq+1, updated_at=now()
WHERE id=$1 RETURNING next_message_seq-1",
)
.bind(session_id)
.fetch_one(&mut *tx)
.await
.map_err(|error| error.to_string())?;
let message_id = Uuid::new_v4().to_string();
sqlx::query(
"INSERT INTO messages (id,thread_id,seq,role,body,blocks)
VALUES ($1,$2,$3,$4,$5,$6)",
)
.bind(&message_id)
.bind(session_id)
.bind(seq)
.bind(role)
.bind(body)
.bind(json!([{"kind":"voice","callId": call_id}]))
.execute(&mut *tx)
.await
.map_err(|error| error.to_string())?;
tx.commit().await.map_err(|error| error.to_string())?;
let _ = crate::sessions::append_event(
state,
session_id,
"message.created",
json!({"id":message_id,"seq":seq,"role":role,"body":body,"voice":true}),
)
.await;
Ok(())
}
async fn send_json(
client_write: &Arc<Mutex<futures_util::stream::SplitSink<WebSocket, AxumMessage>>>,
value: Value,
) -> Result<(), ()> {
client_write
.lock()
.await
.send(AxumMessage::Text(value.to_string().into()))
.await
.map_err(|_| ())
}
#[cfg(test)]
mod tests {
use super::{is_interrupt_request, speakable_progress};
use serde_json::json;
#[test]
fn only_a_typed_interrupt_frame_cancels_the_reply() {
assert!(is_interrupt_request(r#"{"type":"interrupt"}"#));
assert!(!is_interrupt_request(r#"{"type":"hangup"}"#));
assert!(!is_interrupt_request("interrupt"));
assert!(!is_interrupt_request(""));
}
#[test]
fn takeover_and_idle_after_work_are_spoken_once() {
let takeover = json!({"status":"takeover","takeover":true});
assert_eq!(
speakable_progress("{\"status\":\"running\"}", &takeover).as_deref(),
Some("需要你接手畫面登入或驗證。")
);
assert!(speakable_progress(&takeover.to_string(), &takeover).is_none());
let idle = json!({"status":"idle"});
assert!(speakable_progress("{\"status\":\"running\"}", &idle).unwrap().contains("做完"));
}
}

View File

@ -0,0 +1,111 @@
use std::path::PathBuf;
use axum::Router;
use tower_http::services::{ServeDir, ServeFile};
/// Vite puts built files in `dist/` and source assets in `public/`.
/// `LAZYBOY_WEB_DIR` is sometimes the source tree (`apps/web`) and sometimes
/// the build output (`apps/web/dist` or Docker `/web`).
pub fn resolve_web_roots(configured: &str) -> (PathBuf, Option<PathBuf>) {
let configured = PathBuf::from(configured);
let dist = if configured.file_name().is_some_and(|name| name == "dist") {
configured.clone()
} else {
configured.join("dist")
};
let primary = if dist.join("index.html").is_file() {
dist
} else {
configured.clone()
};
let public = [
configured.join("public"),
configured
.parent()
.map(|parent| parent.join("public"))
.unwrap_or_default(),
]
.into_iter()
.find(|path| path.is_dir());
(primary, public)
}
pub fn static_router(configured: &str) -> Router {
let (primary, public) = resolve_web_roots(configured);
let fallback = public.clone().unwrap_or_else(|| primary.clone());
let files = ServeDir::new(&primary).fallback(ServeDir::new(fallback));
let mut router = Router::new().fallback_service(files);
if let Some(icon) = first_existing(&[
primary.join("favicon.ico"),
primary.join("favicon.png"),
public
.as_ref()
.map(|path| path.join("favicon.ico"))
.unwrap_or_default(),
public
.as_ref()
.map(|path| path.join("favicon.png"))
.unwrap_or_default(),
]) {
router = router.route_service("/favicon.ico", ServeFile::new(icon));
}
router
}
fn first_existing(paths: &[PathBuf]) -> Option<PathBuf> {
paths.iter().find(|path| path.is_file()).cloned()
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
fn scratch(name: &str) -> PathBuf {
let path = std::env::temp_dir().join(format!(
"lazyboy-web-static-{}-{name}",
std::process::id()
));
let _ = fs::remove_dir_all(&path);
fs::create_dir_all(&path).unwrap();
path
}
#[test]
fn source_tree_falls_back_to_public_assets() {
let root = scratch("source");
fs::write(root.join("index.html"), "<!doctype html>").unwrap();
fs::create_dir_all(root.join("public")).unwrap();
fs::write(root.join("public/favicon.svg"), "<svg></svg>").unwrap();
let (primary, public) = resolve_web_roots(root.to_str().unwrap());
assert_eq!(primary, root);
assert_eq!(public.as_deref(), Some(root.join("public").as_path()));
let _ = fs::remove_dir_all(root);
}
#[test]
fn prefers_vite_dist_when_present() {
let root = scratch("built");
fs::write(root.join("index.html"), "source").unwrap();
fs::create_dir_all(root.join("dist")).unwrap();
fs::write(root.join("dist/index.html"), "built").unwrap();
fs::create_dir_all(root.join("public")).unwrap();
let (primary, public) = resolve_web_roots(root.to_str().unwrap());
assert_eq!(primary, root.join("dist"));
assert_eq!(public.as_deref(), Some(root.join("public").as_path()));
let _ = fs::remove_dir_all(root);
}
#[test]
fn dist_dir_uses_sibling_public() {
let root = scratch("dist-only");
let dist = root.join("dist");
fs::create_dir_all(&dist).unwrap();
fs::write(dist.join("index.html"), "built").unwrap();
fs::create_dir_all(root.join("public")).unwrap();
let (primary, public) = resolve_web_roots(dist.to_str().unwrap());
assert_eq!(primary, dist);
assert_eq!(public.as_deref(), Some(root.join("public").as_path()));
let _ = fs::remove_dir_all(root);
}
}

View File

@ -24,6 +24,14 @@ pub enum ScrollDirection {
Down,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum RefVerb {
Click,
Focus,
SetValue,
}
/// Canonical actions the control plane understands.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "lowercase")]
@ -36,6 +44,15 @@ pub enum ComputerAction {
#[serde(default)]
button: Option<PointerButton>,
},
/// Semantic target (DOM selector or AT-SPI path). Execute via CDP/a11y, not xdotool.
Ref {
verb: RefVerb,
target: String,
#[serde(default, rename = "refKind")]
ref_kind: String,
#[serde(default)]
text: Option<String>,
},
Clipboard {
text: String,
},
@ -86,15 +103,24 @@ pub struct UiElement {
pub y: u32,
pub w: u32,
pub h: u32,
/// CSS selector for in-page controls (Chromium CDP). Native windows leave this empty.
/// CSS selector (DOM) or AT-SPI path (a11y). Native windows leave this empty.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// "dom" for page controls, "window" for native windows.
/// "dom" for page controls, "a11y" for AT-SPI widgets, "window" for native windows.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub kind: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
}
impl UiElement {
pub fn has_ref(&self) -> bool {
self.selector
.as_deref()
.is_some_and(|value| !value.is_empty())
&& matches!(self.kind.as_deref(), Some("dom") | Some("a11y"))
}
pub fn center(&self) -> (u32, u32) {
(
self.x.saturating_add(self.w / 2),

View File

@ -189,6 +189,10 @@ pub struct ComputerStatus {
/// What the active run is doing right now ("思考中", "browser: click #12"…).
#[serde(default)]
pub busy_step: Option<String>,
/// True only while the active run holds the desktop screen lease.
/// Chat-only replies set busy_session_id but not this.
#[serde(default)]
pub using_computer: bool,
/// Run paused in `waiting_takeover` (bot asked for, or user forced, control).
#[serde(default)]
pub waiting_run_id: Option<String>,

View File

@ -6,6 +6,7 @@ mod model;
mod room;
mod run;
mod session;
mod voice;
pub use action::*;
pub use bot::*;
@ -15,5 +16,6 @@ pub use model::*;
pub use room::*;
pub use run::*;
pub use session::*;
pub use voice::*;
pub type Id = String;

View File

@ -0,0 +1,200 @@
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use thiserror::Error;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum VoiceProvider {
Xai,
Openai,
Scripted,
}
impl VoiceProvider {
pub fn as_str(self) -> &'static str {
match self {
Self::Xai => "xai",
Self::Openai => "openai",
Self::Scripted => "scripted",
}
}
pub fn env_key_name(self) -> &'static str {
match self {
Self::Xai => "XAI_API_KEY",
Self::Openai => "OPENAI_API_KEY",
Self::Scripted => "",
}
}
pub fn requires_api_key(self) -> bool {
!matches!(self, Self::Scripted)
}
pub fn default_model_id(self) -> &'static str {
match self {
Self::Xai => DEFAULT_XAI_VOICE_MODEL,
Self::Openai => DEFAULT_OPENAI_VOICE_MODEL,
Self::Scripted => "scripted-voice",
}
}
pub fn default_voice_id(self) -> &'static str {
match self {
Self::Xai => "eve",
Self::Openai => "marin",
Self::Scripted => "scripted",
}
}
pub fn realtime_url(self, model_id: &str) -> String {
match self {
Self::Xai => format!("wss://api.x.ai/v1/realtime?model={model_id}"),
Self::Openai => format!("wss://api.openai.com/v1/realtime?model={model_id}"),
Self::Scripted => String::new(),
}
}
pub fn selectable() -> &'static [Self] {
&[Self::Xai, Self::Openai]
}
}
impl std::str::FromStr for VoiceProvider {
type Err = UnknownVoiceProvider;
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"xai" => Ok(Self::Xai),
"openai" => Ok(Self::Openai),
"scripted" => Ok(Self::Scripted),
other => Err(UnknownVoiceProvider(other.to_string())),
}
}
}
#[derive(Debug, Error, Clone, PartialEq, Eq)]
#[error("unknown voice provider: {0}")]
pub struct UnknownVoiceProvider(pub String);
pub const DEFAULT_XAI_VOICE_MODEL: &str = "grok-voice-latest";
pub const DEFAULT_OPENAI_VOICE_MODEL: &str = "gpt-realtime";
pub const VOICE_SAMPLE_RATE: u32 = 24_000;
pub fn catalog_voice_models(provider: VoiceProvider) -> &'static [(&'static str, &'static str)] {
match provider {
VoiceProvider::Xai => &[
("grok-voice-latest", "Grok Voice Latest"),
("grok-voice-think-fast-2.0", "Grok Voice Think Fast 2.0"),
("grok-voice-think-fast-1.0", "Grok Voice Think Fast 1.0"),
],
VoiceProvider::Openai => &[("gpt-realtime", "GPT Realtime")],
VoiceProvider::Scripted => &[("scripted-voice", "Scripted")],
}
}
pub fn catalog_voices(provider: VoiceProvider) -> &'static [(&'static str, &'static str)] {
match provider {
VoiceProvider::Xai => &[
("eve", "Eve"),
("ara", "Ara"),
("leo", "Leo"),
("rex", "Rex"),
("sal", "Sal"),
],
VoiceProvider::Openai => &[
("marin", "Marin"),
("alloy", "Alloy"),
("verse", "Verse"),
("cedar", "Cedar"),
],
VoiceProvider::Scripted => &[("scripted", "Scripted")],
}
}
/// Custom functions the voice model may call. LazyBoy executes them; they return immediately.
pub fn computer_voice_tools() -> Vec<Value> {
vec![
function_tool(
"start_computer_task",
"Start a computer task on the user's Linux desktop. Returns immediately — speak to the user first, do not wait for the work to finish. Use when they want you to operate the computer (open a site, click, download, run a command).",
json!({
"type": "object",
"properties": {
"prompt": {
"type": "string",
"description": "What to do on the computer, in the user's language."
}
},
"required": ["prompt"]
}),
),
function_tool(
"follow_up_computer_task",
"Add or change instructions for the computer task that is already running. Use when the user interrupts with a correction.",
json!({
"type": "object",
"properties": {
"prompt": {
"type": "string",
"description": "The new or extra instruction."
}
},
"required": ["prompt"]
}),
),
function_tool(
"stop_computer_task",
"Cancel the computer task that is running.",
json!({ "type": "object", "properties": {} }),
),
function_tool(
"computer_status",
"Check whether a computer task is running, queued, needs the user to take over the screen, or is idle.",
json!({ "type": "object", "properties": {} }),
),
]
}
fn function_tool(name: &str, description: &str, parameters: Value) -> Value {
json!({
"type": "function",
"name": name,
"description": description,
"parameters": parameters
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn voice_provider_roundtrip() {
assert_eq!("xai".parse::<VoiceProvider>().unwrap(), VoiceProvider::Xai);
assert_eq!(
"openai".parse::<VoiceProvider>().unwrap(),
VoiceProvider::Openai
);
assert!("anthropic".parse::<VoiceProvider>().is_err());
assert_eq!(VoiceProvider::Xai.default_voice_id(), "eve");
}
#[test]
fn computer_tools_are_named() {
let tools = computer_voice_tools();
let names: Vec<_> = tools
.iter()
.filter_map(|tool| tool.get("name")?.as_str())
.collect();
assert_eq!(
names,
[
"start_computer_task",
"follow_up_computer_task",
"stop_computer_task",
"computer_status"
]
);
}
}

353
crates/control/src/a11y.py Normal file
View File

@ -0,0 +1,353 @@
import json, os, sys, time
def fail(msg):
print(json.dumps({"ok": False, "error": msg, "elements": []}))
sys.exit(0)
def load_session(display):
display = display or os.environ.get("DISPLAY") or ":1"
if not display.startswith(":"):
display = ":" + display
os.environ["DISPLAY"] = display
number = display.lstrip(":")
dbus_file = "/tmp/lazyboy/screen-%s.dbus" % number
runtime_file = "/tmp/lazyboy/screen-%s.runtime" % number
try:
addr = open(dbus_file).read().strip()
if addr:
os.environ["DBUS_SESSION_BUS_ADDRESS"] = addr
except Exception:
pass
try:
runtime = open(runtime_file).read().strip()
if runtime:
os.environ["XDG_RUNTIME_DIR"] = runtime
except Exception:
if number == "1":
candidate = "/tmp/xfce-home/runtime"
else:
candidate = "/tmp/xfce-home-%s/runtime" % number
if os.path.isdir(candidate):
os.environ["XDG_RUNTIME_DIR"] = candidate
def atspi():
import gi
gi.require_version("Atspi", "2.0")
from gi.repository import Atspi
try:
Atspi.init()
except Exception:
pass
try:
Atspi.set_timeout(200, 200)
except Exception:
pass
return Atspi
INTERACTIVE = {
"push button", "toggle button", "check box", "radio button",
"combo box", "text", "password text", "menu item", "check menu item",
"radio menu item", "tab", "page tab", "slider", "spin button",
"link", "tree item", "entry", "password", "button", "menu",
"list item", "column header", "toggle",
}
BROWSER_APPS = ("chromium", "chrome", "google-chrome", "chromium-browser")
def is_browser_name(name):
n = (name or "").lower()
return any(token in n for token in BROWSER_APPS)
def child_count(acc):
try:
return int(acc.get_child_count())
except Exception:
return 0
def child_at(acc, i):
try:
return acc.get_child_at_index(i)
except Exception:
return None
def role_name(acc):
try:
return (acc.get_role_name() or "").lower()
except Exception:
return ""
def acc_name(acc):
try:
text = (acc.get_name() or "").strip()
if text:
return text
except Exception:
pass
try:
return (acc.get_description() or "").strip()
except Exception:
return ""
def state_names(Atspi, acc):
out = []
try:
ss = acc.get_state_set()
except Exception:
return out
for name in ("showing", "visible", "enabled", "sensitive", "checked",
"selected", "focused", "editable", "defunct", "expandable",
"expanded"):
try:
st = getattr(Atspi.StateType, name.upper())
if ss.contains(st):
out.append(name)
except Exception:
pass
return out
def extents(Atspi, acc):
try:
ext = acc.get_extents(Atspi.CoordType.SCREEN)
return int(ext.x), int(ext.y), int(ext.width), int(ext.height)
except Exception:
pass
try:
comp = acc.get_component_iface()
if comp is None:
return None
ext = comp.get_extents(Atspi.CoordType.SCREEN)
return int(ext.x), int(ext.y), int(ext.width), int(ext.height)
except Exception:
return None
def action_iface(acc):
for getter in ("get_action_iface", "queryAction", "get_action"):
fn = getattr(acc, getter, None)
if not fn:
continue
try:
iface = fn()
if iface is not None:
return iface
except Exception:
pass
if hasattr(acc, "get_n_actions") and hasattr(acc, "do_action"):
return acc
return None
def text_ifaces(acc):
edit = None
text = None
for getter in ("get_editable_text_iface", "queryEditableText"):
fn = getattr(acc, getter, None)
if not fn:
continue
try:
edit = fn()
if edit is not None:
break
except Exception:
pass
for getter in ("get_text_iface", "queryText"):
fn = getattr(acc, getter, None)
if not fn:
continue
try:
text = fn()
if text is not None:
break
except Exception:
pass
if edit is None and hasattr(acc, "insert_text"):
edit = acc
if text is None and hasattr(acc, "get_character_count"):
text = acc
return edit, text
def resolve(Atspi, path):
desktop = Atspi.get_desktop(0)
node = desktop
for part in str(path).split("/"):
if part == "":
continue
node = child_at(node, int(part))
if node is None:
return None
return node
def grab_focus(acc):
for getter in ("grab_focus",):
fn = getattr(acc, getter, None)
if fn:
try:
fn()
return True
except Exception:
pass
try:
comp = acc.get_component_iface()
if comp is not None:
comp.grab_focus()
return True
except Exception:
pass
return False
def do_click(acc):
action = action_iface(acc)
if action is None:
return grab_focus(acc) and False
try:
n = int(action.get_n_actions())
except Exception:
n = 0
idx = 0
prefer = ("click", "press", "activate", "jump", "open", "toggle", "select")
for i in range(n):
try:
name = (action.get_action_name(i) or "").lower()
except Exception:
name = ""
if name in prefer:
idx = i
break
if n <= 0:
return False
try:
return bool(action.do_action(idx))
except Exception:
return False
def set_text(acc, value):
grab_focus(acc)
edit, text = text_ifaces(acc)
if edit is None:
return False
n = 0
if text is not None:
try:
n = int(text.get_character_count())
except Exception:
n = 0
try:
edit.delete_text(0, n)
except Exception:
pass
try:
edit.insert_text(0, value, len(value))
return True
except Exception:
return False
def snapshot(Atspi, include_browser):
deadline = time.time() + 2.8
desktop = Atspi.get_desktop(0)
found = []
visited = 0
apps = child_count(desktop)
for app_i in range(apps):
if time.time() > deadline or len(found) >= 50:
break
app = child_at(desktop, app_i)
if app is None:
continue
app_label = acc_name(app) or role_name(app)
if not include_browser and is_browser_name(app_label):
continue
stack = [(app, str(app_i), 0)]
while stack:
if time.time() > deadline or len(found) >= 50 or visited > 400:
break
acc, path, depth = stack.pop()
visited += 1
states = state_names(Atspi, acc)
if "defunct" in states:
continue
role = role_name(acc)
name = acc_name(acc)
showing = ("showing" in states) or ("visible" in states) or not states
if role in INTERACTIVE and showing and name:
box = extents(Atspi, acc)
if box and box[2] >= 2 and box[3] >= 2:
x, y, w, h = box
if x + w > 0 and y + h > 0:
title = "%s %s" % (role, name.replace("\n", " ").strip())
if "checked" in states:
title += " (checked)"
if "expanded" in states:
title += " (expanded)"
if "enabled" in states and "sensitive" in states:
pass
elif states and "enabled" not in states:
title += " [disabled]"
title = title[:80]
found.append({
"id": len(found) + 1,
"title": title,
"role": role,
"kind": "a11y",
"selector": path,
"x": max(0, x),
"y": max(0, y),
"w": w,
"h": h,
})
if depth >= 12:
continue
n = child_count(acc)
# Walk children in reverse so index 0 is processed first with pop().
for i in range(n - 1, -1, -1):
child = child_at(acc, i)
if child is None:
continue
stack.append((child, "%s/%d" % (path, i), depth + 1))
return found
def main():
req = json.loads(sys.argv[1]) if len(sys.argv) > 1 else {}
action = req.get("action") or "snapshot"
display = req.get("display") or ":1"
load_session(display)
try:
Atspi = atspi()
except Exception as error:
fail("atspi unavailable: %s" % error)
return
if action == "snapshot":
include_browser = bool(req.get("includeBrowser"))
try:
elements = snapshot(Atspi, include_browser)
except Exception as error:
fail("atspi snapshot failed: %s" % error)
return
print(json.dumps({"ok": True, "elements": elements}))
return
selector = req.get("selector") or ""
if not selector:
fail("a11y action needs selector")
return
try:
acc = resolve(Atspi, selector)
except Exception as error:
fail("a11y resolve failed: %s" % error)
return
if acc is None:
fail("a11y element gone")
return
ok = False
if action == "click":
ok = do_click(acc)
elif action == "type":
ok = set_text(acc, req.get("text") or "")
elif action == "focus":
ok = grab_focus(acc)
else:
fail("unsupported a11y action")
return
print(json.dumps({"ok": bool(ok), "error": None if ok else "a11y %s failed" % action}))
if __name__ == "__main__":
try:
main()
except Exception as error:
fail(str(error))

201
crates/control/src/a11y.rs Normal file
View File

@ -0,0 +1,201 @@
use lazyboy_contracts::UiElement;
use serde_json::{Value, json};
use crate::x11::{is_browser_title, parse_ui_elements};
use crate::{PRIMARY_DISPLAY, normalize_display};
const A11Y_PY: &str = include_str!("a11y.py");
#[derive(Debug, Clone, PartialEq, Default)]
pub struct A11yPage {
pub ok: bool,
pub error: Option<String>,
pub elements: Vec<UiElement>,
}
pub fn a11y_command_on(display: &str, request: &Value) -> Vec<String> {
let mut body = request.clone();
if let Some(object) = body.as_object_mut() {
object
.entry("display")
.or_insert_with(|| json!(normalize_display(display)));
}
vec![
"env".into(),
format!("DISPLAY={}", normalize_display(display)),
"python3".into(),
"-c".into(),
A11Y_PY.into(),
body.to_string(),
]
}
pub fn a11y_command(request: &Value) -> Vec<String> {
a11y_command_on(PRIMARY_DISPLAY, request)
}
pub fn parse_a11y_page(raw: &str) -> A11yPage {
let value: Value = serde_json::from_str(raw.trim()).unwrap_or(Value::Null);
let ok = value.get("ok").and_then(Value::as_bool) == Some(true);
A11yPage {
ok,
error: if ok {
None
} else {
value
.get("error")
.and_then(Value::as_str)
.map(str::to_string)
.or_else(|| Some("atspi unavailable".into()))
},
elements: value
.get("elements")
.map(|items| parse_ui_elements(&items.to_string()))
.unwrap_or_default(),
}
}
/// Merge DOM (keep ids), then a11y, then native windows that are not already
/// covered by a control tree. Chromium's whole window is dropped when the
/// page snapshot returned elements.
pub fn merge_ui_elements(
windows: Vec<UiElement>,
page: &[UiElement],
a11y: &[UiElement],
) -> Vec<UiElement> {
let mut elements = page.to_vec();
let mut next = elements.len() as u32 + 1;
for mut control in a11y.iter().cloned() {
control.id = next;
next += 1;
elements.push(control);
}
for mut window in windows {
if !page.is_empty() && is_browser_title(&window.title) {
continue;
}
if a11y_covers_window(a11y, &window) {
continue;
}
window.id = next;
next += 1;
elements.push(window);
}
elements
}
fn a11y_covers_window(a11y: &[UiElement], window: &UiElement) -> bool {
if a11y.is_empty() || window.w == 0 || window.h == 0 {
return false;
}
a11y.iter().any(|control| center_inside(control, window))
}
fn center_inside(element: &UiElement, window: &UiElement) -> bool {
let (x, y) = element.center();
let area = u64::from(element.w.saturating_mul(element.h.max(1)));
let window_area = u64::from(window.w.saturating_mul(window.h.max(1)));
x >= window.x
&& y >= window.y
&& x < window.x.saturating_add(window.w)
&& y < window.y.saturating_add(window.h)
&& (window_area == 0 || area < window_area)
}
#[cfg(test)]
mod tests {
use super::*;
fn window(id: u32, title: &str, x: u32, y: u32, w: u32, h: u32) -> UiElement {
UiElement {
id,
title: title.into(),
x,
y,
w,
h,
kind: Some("window".into()),
..UiElement::default()
}
}
fn a11y(id: u32, title: &str, x: u32, y: u32, w: u32, h: u32) -> UiElement {
UiElement {
id,
title: title.into(),
x,
y,
w,
h,
kind: Some("a11y".into()),
selector: Some(format!("0/{id}")),
role: Some("push button".into()),
..UiElement::default()
}
}
#[test]
fn command_injects_display_and_script() {
let argv = a11y_command_on(":2", &json!({"action": "snapshot"}));
assert!(argv.contains(&"DISPLAY=:2".into()));
assert!(argv.iter().any(|item| item.contains("python3")));
assert!(argv.last().unwrap().contains("\"display\":\":2\""));
assert!(argv.iter().any(|item| item.contains("Atspi") || item.contains("atspi")));
}
#[test]
fn parses_snapshot_elements() {
let page = parse_a11y_page(
r#"{"ok":true,"elements":[{"id":1,"title":"push button Open","role":"push button","kind":"a11y","selector":"0/2/1","x":10,"y":20,"w":80,"h":24}]}"#,
);
assert!(page.ok);
assert_eq!(page.elements.len(), 1);
assert_eq!(page.elements[0].kind.as_deref(), Some("a11y"));
assert_eq!(page.elements[0].selector.as_deref(), Some("0/2/1"));
assert_eq!(page.elements[0].role.as_deref(), Some("push button"));
}
#[test]
fn merge_keeps_dom_ids_and_replaces_covered_windows() {
let windows = vec![
window(1, "Thunar", 0, 24, 800, 600),
window(2, "Terminal", 800, 24, 480, 600),
window(3, "Chromium", 0, 0, 1280, 800),
];
let page = vec![UiElement {
id: 1,
title: "Login".into(),
selector: Some("[data-lazyboy=\"1\"]".into()),
kind: Some("dom".into()),
x: 40,
y: 80,
w: 60,
h: 20,
..UiElement::default()
}];
let native = vec![a11y(1, "push button Open", 40, 40, 80, 24)];
let merged = merge_ui_elements(windows, &page, &native);
assert_eq!(merged[0].title, "Login");
assert_eq!(merged[0].id, 1);
assert_eq!(merged[1].id, 2);
assert_eq!(merged[1].title, "push button Open");
assert!(merged.iter().any(|element| element.title == "Terminal"));
assert!(!merged.iter().any(|element| element.title == "Thunar"));
assert!(!merged.iter().any(|element| element.title == "Chromium"));
}
#[test]
fn merge_without_trees_keeps_windows() {
let windows = vec![window(1, "Thunar", 0, 24, 800, 600)];
let merged = merge_ui_elements(windows, &[], &[]);
assert_eq!(merged.len(), 1);
assert_eq!(merged[0].title, "Thunar");
}
#[test]
fn unavailable_tree_is_not_ok() {
let page = parse_a11y_page(r#"{"ok":false,"error":"atspi unavailable"}"#);
assert!(!page.ok);
assert_eq!(page.error.as_deref(), Some("atspi unavailable"));
}
}

View File

@ -1,4 +1,7 @@
use lazyboy_contracts::{ComputerAction, PointerButton, PointerType, ScrollDirection, UiElement};
use crate::is_browser_title;
use lazyboy_contracts::{
ComputerAction, PointerButton, PointerType, RefVerb, ScrollDirection, UiElement,
};
use serde_json::{Value, json};
use thiserror::Error;
@ -43,7 +46,8 @@ pub fn element_id(value: Option<&Value>) -> Option<u64> {
}
}
/// Fill x/y from a numbered on-screen element so the model can click by id.
/// Resolve a numbered on-screen element. DOM/a11y refs stay semantic (no
/// pre-filled coordinates). Window-level targets still become center pixels.
pub fn apply_element_targets(value: &mut Value, elements: &[UiElement]) -> Result<(), ActionError> {
let Some(items) = value.as_array_mut() else {
return Ok(());
@ -58,6 +62,21 @@ pub fn apply_element_targets(value: &mut Value, elements: &[UiElement]) -> Resul
let Some(element) = elements.iter().find(|element| u64::from(element.id) == id) else {
return Err(ActionError::UnknownElement(id as u32));
};
let kind = action
.get("kind")
.and_then(Value::as_str)
.or_else(|| action.get("type").and_then(Value::as_str))
.unwrap_or("");
if element.has_ref() && matches!(kind, "click" | "type") {
if let Some(selector) = element.selector.clone() {
action.insert("target".into(), json!(selector));
action.insert(
"refKind".into(),
json!(element.kind.clone().unwrap_or_else(|| "a11y".into())),
);
}
continue;
}
if element.is_offscreen() {
return Err(ActionError::OffscreenElement(id as u32));
}
@ -68,6 +87,97 @@ pub fn apply_element_targets(value: &mut Value, elements: &[UiElement]) -> Resul
Ok(())
}
/// Fingerprint of the first click-like action, used to refuse repeating a miss.
pub fn click_fingerprint(actions: &Value) -> Option<String> {
let items = actions.as_array()?;
for raw in items {
let Some(action) = raw.as_object() else {
continue;
};
let kind = action
.get("kind")
.and_then(Value::as_str)
.or_else(|| action.get("type").and_then(Value::as_str))
.unwrap_or("");
if !matches!(kind, "click" | "down" | "drag") {
continue;
}
if let Some(id) = element_id(action.get("element")) {
return Some(format!("e{id}"));
}
match (
action.get("x").and_then(Value::as_f64),
action.get("y").and_then(Value::as_f64),
) {
(Some(x), Some(y)) if x.is_finite() && y.is_finite() => {
return Some(format!("p{},{}", x.round() as i64, y.round() as i64));
}
_ => {}
}
}
None
}
pub fn should_block_stale_click(miss_streak: u32, last: Option<&str>, next: Option<&str>) -> bool {
miss_streak >= 2 && next.is_some() && next == last
}
/// When a CDP page snapshot is live, refuse pixel-clicking the Chromium window.
pub fn browser_gui_block(actions: &Value, elements: &[UiElement]) -> Option<String> {
if !elements
.iter()
.any(|element| element.kind.as_deref() == Some("dom"))
{
return None;
}
let items = actions.as_array()?;
for raw in items {
let Some(action) = raw.as_object() else {
continue;
};
let kind = action
.get("kind")
.and_then(Value::as_str)
.or_else(|| action.get("type").and_then(Value::as_str))
.unwrap_or("");
if !matches!(kind, "click" | "move" | "down" | "up" | "hover" | "drag") {
continue;
}
if let Some(id) = element_id(action.get("element")) {
match elements.iter().find(|element| u64::from(element.id) == id) {
Some(element) if element.has_ref() => continue,
Some(element)
if element.kind.as_deref() == Some("window")
&& is_browser_title(&element.title) =>
{
return Some(browser_block_message(elements));
}
_ => continue,
}
} else {
return Some(browser_block_message(elements));
}
}
None
}
fn browser_block_message(elements: &[UiElement]) -> String {
let known: Vec<String> = elements
.iter()
.filter(|element| element.kind.as_deref() == Some("dom"))
.take(12)
.map(|element| format!("[{}] {}", element.id, element.title))
.collect();
format!(
"Chromium is in front: use the browser tool (snapshot / click element N) instead of computer_act pixel clicks. Known page elements: {}",
if known.is_empty() {
"call browser snapshot first".to_string()
} else {
known.join(", ")
}
)
}
pub fn parse_computer_actions(value: &Value) -> Result<Vec<ComputerAction>, ActionError> {
let Value::Array(items) = value else {
return Err(ActionError::Empty);
@ -90,6 +200,23 @@ pub fn parse_computer_actions(value: &Value) -> Result<Vec<ComputerAction>, Acti
.unwrap_or_default();
match kind {
"click" | "move" | "down" | "up" => {
if kind == "click" {
if let Some(target) = ref_target(action) {
let pointer = ComputerAction::Ref {
verb: RefVerb::Click,
target,
ref_kind: ref_kind(action),
text: None,
};
let doubled = action.get("double").and_then(Value::as_bool) == Some(true);
actions.push(pointer.clone());
if doubled {
actions.push(ComputerAction::Wait { ms: 70 });
actions.push(pointer);
}
continue;
}
}
let x = coordinate(action.get("x"), "x")?;
let y = coordinate(action.get("y"), "y")?;
let pointer_type = match kind {
@ -177,8 +304,17 @@ pub fn parse_computer_actions(value: &Value) -> Result<Vec<ComputerAction>, Acti
.and_then(Value::as_str)
.unwrap_or("")
.to_string();
if let Some(target) = ref_target(action) {
actions.push(ComputerAction::Ref {
verb: RefVerb::SetValue,
target,
ref_kind: ref_kind(action),
text: Some(text),
});
} else {
actions.push(ComputerAction::Clipboard { text });
}
}
"key" => {
let key = action
.get("key")
@ -247,6 +383,24 @@ pub fn parse_computer_actions(value: &Value) -> Result<Vec<ComputerAction>, Acti
Ok(actions)
}
fn ref_target(action: &serde_json::Map<String, Value>) -> Option<String> {
action
.get("target")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(str::to_string)
}
fn ref_kind(action: &serde_json::Map<String, Value>) -> String {
action
.get("refKind")
.or_else(|| action.get("ref_kind"))
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.unwrap_or("a11y")
.to_string()
}
fn coordinate(value: Option<&Value>, name: &'static str) -> Result<u32, ActionError> {
let number = value.and_then(Value::as_f64).unwrap_or(f64::NAN).round();
if !number.is_finite() || number < 0.0 || number > 100_000.0 {
@ -386,10 +540,145 @@ mod tests {
kind: Some("dom".into()),
..UiElement::default()
};
let mut actions = json!([{"kind": "click", "element": 3}]);
let mut actions = json!([{"kind": "hover", "element": 3}]);
assert_eq!(
apply_element_targets(&mut actions, &[below_fold]).unwrap_err(),
ActionError::OffscreenElement(3)
);
}
fn a11y_button() -> UiElement {
UiElement {
id: 4,
title: "push button Open".into(),
selector: Some("0/2/1".into()),
kind: Some("a11y".into()),
role: Some("push button".into()),
x: 10,
y: 20,
w: 80,
h: 24,
..UiElement::default()
}
}
#[test]
fn a11y_click_stays_semantic() {
let mut actions = json!([{"kind": "click", "element": 4}]);
apply_element_targets(&mut actions, &[a11y_button()]).unwrap();
assert!(actions[0].get("x").is_none());
assert_eq!(actions[0]["target"], "0/2/1");
assert_eq!(actions[0]["refKind"], "a11y");
let parsed = parse_computer_actions(&actions).unwrap();
assert!(matches!(
parsed[0],
ComputerAction::Ref {
verb: RefVerb::Click,
..
}
));
}
#[test]
fn a11y_type_is_set_value() {
let mut actions = json!([{"kind": "type", "element": 4, "text": "report.pdf"}]);
apply_element_targets(&mut actions, &[a11y_button()]).unwrap();
let parsed = parse_computer_actions(&actions).unwrap();
match &parsed[0] {
ComputerAction::Ref {
verb: RefVerb::SetValue,
text,
..
} => assert_eq!(text.as_deref(), Some("report.pdf")),
other => panic!("{other:?}"),
}
}
#[test]
fn dom_click_does_not_fill_coordinates() {
let mut actions = json!([{"kind": "click", "element": 1}]);
let elements = vec![UiElement {
id: 1,
title: "Submit".into(),
selector: Some("[data-lazyboy=\"1\"]".into()),
kind: Some("dom".into()),
x: 10,
y: 20,
w: 80,
h: 24,
..UiElement::default()
}];
apply_element_targets(&mut actions, &elements).unwrap();
assert!(actions[0].get("x").is_none());
let parsed = parse_computer_actions(&actions).unwrap();
match &parsed[0] {
ComputerAction::Ref {
ref_kind, verb, ..
} => {
assert_eq!(ref_kind, "dom");
assert_eq!(*verb, RefVerb::Click);
}
other => panic!("{other:?}"),
}
}
#[test]
fn pixel_clicks_are_blocked_when_dom_is_live() {
let elements = vec![UiElement {
id: 1,
title: "Submit".into(),
selector: Some("[data-lazyboy=\"1\"]".into()),
kind: Some("dom".into()),
x: 10,
y: 20,
w: 80,
h: 24,
..UiElement::default()
}];
let blocked = browser_gui_block(&json!([{"kind":"click","x":40,"y":80}]), &elements);
assert!(blocked.unwrap().contains("browser"));
assert!(
browser_gui_block(&json!([{"kind":"click","element":1}]), &elements).is_none()
);
}
#[test]
fn chromium_window_clicks_are_blocked_when_dom_is_live() {
let elements = vec![
UiElement {
id: 1,
title: "Submit".into(),
selector: Some("[data-lazyboy=\"1\"]".into()),
kind: Some("dom".into()),
..UiElement::default()
},
UiElement {
id: 2,
title: "Chromium".into(),
kind: Some("window".into()),
x: 0,
y: 0,
w: 1280,
h: 800,
..UiElement::default()
},
];
let blocked = browser_gui_block(&json!([{"kind":"click","element":2}]), &elements);
assert!(blocked.unwrap().contains("browser"));
}
#[test]
fn stale_repeat_blocks_the_third_same_click() {
assert!(!should_block_stale_click(1, Some("e3"), Some("e3")));
assert!(should_block_stale_click(2, Some("e3"), Some("e3")));
assert!(!should_block_stale_click(2, Some("e3"), Some("e4")));
assert_eq!(
click_fingerprint(&json!([{"kind":"click","element":3}])).as_deref(),
Some("e3")
);
assert_eq!(
click_fingerprint(&json!([{"kind":"click","x":10,"y":20}])).as_deref(),
Some("p10,20")
);
}
}

View File

@ -12,101 +12,31 @@ def http_json(url, timeout=2):
return None
class Ws:
"""Use a maintained RFC6455 transport (fragmentation, ping/pong, handshake)."""
def __init__(self, url):
rest = url[5:]
hostpath = rest.split("/", 1)
hostport = hostpath[0]
path = "/" + (hostpath[1] if len(hostpath) > 1 else "")
if ":" in hostport:
host, port = hostport.rsplit(":", 1)
port = int(port)
else:
host, port = hostport, 80
self.sock = socket.create_connection((host, port), 5)
key = base64.b64encode(os.urandom(16)).decode()
req = (
"GET %s HTTP/1.1\r\nHost: %s\r\nUpgrade: websocket\r\n"
"Connection: Upgrade\r\nSec-WebSocket-Key: %s\r\n"
"Sec-WebSocket-Version: 13\r\nOrigin: http://%s\r\n\r\n"
% (path, hostport, key, hostport)
)
self.sock.sendall(req.encode())
buf = b""
while b"\r\n\r\n" not in buf:
chunk = self.sock.recv(4096)
if not chunk:
raise RuntimeError("ws handshake closed")
buf += chunk
import websocket
self.sock = websocket.create_connection(url, timeout=5, suppress_origin=True,
http_no_proxy=["127.0.0.1", "localhost"])
self.n = 0
def _frame(self, data):
n = len(data)
hdr = bytearray([0x81])
if n < 126:
hdr.append(0x80 | n)
elif n < 65536:
hdr.append(0x80 | 126)
hdr += struct.pack("!H", n)
else:
hdr.append(0x80 | 127)
hdr += struct.pack("!Q", n)
mask = os.urandom(4)
hdr += mask
masked = bytes(b ^ mask[i % 4] for i, b in enumerate(data))
return bytes(hdr) + masked
def _read(self, n):
buf = b""
while len(buf) < n:
chunk = self.sock.recv(n - len(buf))
if not chunk:
raise RuntimeError("ws eof")
buf += chunk
return buf
def _read_frame(self):
hdr = self._read(2)
opcode = hdr[0] & 0x0F
masked = hdr[1] & 0x80
n = hdr[1] & 0x7F
if n == 126:
n = struct.unpack("!H", self._read(2))[0]
elif n == 127:
n = struct.unpack("!Q", self._read(8))[0]
mask = self._read(4) if masked else b""
payload = self._read(n)
if masked:
payload = bytes(b ^ mask[i % 4] for i, b in enumerate(payload))
return opcode, payload
def recv_json(self):
while True:
opcode, payload = self._read_frame()
if opcode == 0x8:
raise RuntimeError("ws closed")
if opcode == 0x9:
continue
if opcode in (0x1, 0x2):
return json.loads(payload.decode())
return json.loads(self.sock.recv())
def call(self, method, params=None):
self.n += 1
msg = {"id": self.n, "method": method}
if params:
msg["params"] = params
self.sock.sendall(self._frame(json.dumps(msg).encode()))
while True:
self.sock.send(json.dumps({"id": self.n, "method": method, "params": params or {}}))
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
self.sock.settimeout(max(.1, deadline-time.monotonic()))
obj = self.recv_json()
if obj.get("id") == self.n:
if "error" in obj:
raise RuntimeError(str(obj["error"]))
return obj.get("result") or {}
raise TimeoutError("CDP response deadline exceeded")
def close(self):
try:
self.sock.close()
except Exception:
pass
def probe(port):
return http_json("http://127.0.0.1:%s/json/version" % port) is not None
@ -140,7 +70,7 @@ def spawn_browser(display, profile, port):
if profile:
env["LAZYBOY_BROWSER_PROFILE"] = profile
subprocess.Popen(
["lazyboy-browser", "--remote-debugging-port=%s" % port, "--remote-allow-origins=*"],
["lazyboy-browser", "--remote-debugging-port=%s" % port],
env=env,
start_new_session=True,
stdout=subprocess.DEVNULL,
@ -171,6 +101,8 @@ SNAP_JS = r"""
const chromeW = Math.max(0, (window.outerWidth || 0) - (window.innerWidth || 0));
const sx0 = (window.screenX || 0) + Math.floor(chromeW / 2);
const sy0 = (window.screenY || 0) + chromeH;
document.querySelectorAll('[data-lazyboy]').forEach(el => el.removeAttribute('data-lazyboy'));
const generation = crypto.randomUUID();
const seen = new Set();
const inView = [];
const offView = [];
@ -181,7 +113,9 @@ SNAP_JS = r"""
if (st.visibility === "hidden" || st.display === "none" || Number(st.opacity) === 0) continue;
const type = (el.getAttribute("type") || "").toLowerCase();
let text;
if (el.tagName === "INPUT" && (type === "radio" || type === "checkbox")) {
if (type === "password" || /password|secret|token|one-time-code/i.test([el.name, el.id, el.autocomplete].join(" "))) {
text = "[protected input]";
} else if (el.tagName === "INPUT" && (type === "radio" || type === "checkbox")) {
// Quiz answers: the value is usually "on"; the label next to it is what
// the model must read to pick the right option.
const owner = (el.labels && el.labels[0]) || el.closest("label") || el.parentElement;
@ -214,12 +148,12 @@ SNAP_JS = r"""
const out = [];
let n = 1;
for (const item of inView.slice(0, 50).concat(offView.slice(0, 20))) {
item.el.setAttribute("data-lazyboy", String(n));
item.el.setAttribute("data-lazyboy", generation + "-" + n);
out.push({
id: n,
title: item.text,
tag: item.el.tagName.toLowerCase(),
selector: '[data-lazyboy="' + n + '"]',
selector: '[data-lazyboy="' + generation + "-" + n + '"]',
kind: "dom",
x: item.x,
y: item.y,
@ -239,6 +173,11 @@ CLICK_JS = r"""
if (!el) return {ok: false, error: "element gone"};
el.scrollIntoView({block: "center", inline: "nearest"});
const r = el.getBoundingClientRect();
const style = getComputedStyle(el);
const hit = document.elementFromPoint(r.x+r.width/2, r.y+r.height/2);
if (el.disabled || el.getAttribute("aria-disabled") === "true" || r.width <= 0 || r.height <= 0 || style.visibility === "hidden" || style.display === "none" || !hit || !(hit === el || el.contains(hit))) {
return {ok:false,error:"element is disabled, hidden, or covered; observe again"};
}
el.focus();
el.click();
const chromeH = Math.max(0, (window.outerHeight || 0) - (window.innerHeight || 0));
@ -302,7 +241,7 @@ def wait_for_visual_update(ws):
# exits, so wait for two animation frames to keep that screenshot aligned
# with the framebuffer streamed by VNC.
try:
evaluate(ws, "new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))")
evaluate(ws, "new Promise(resolve => {setTimeout(resolve, 250); requestAnimationFrame(() => requestAnimationFrame(resolve));})")
except Exception:
pass
@ -462,7 +401,7 @@ class Recorder:
msg["params"] = params
if session:
msg["sessionId"] = session
self.ws.sock.sendall(self.ws._frame(json.dumps(msg).encode()))
self.ws.sock.send(json.dumps(msg))
while True:
obj = self.ws.recv_json()
if obj.get("id") == self.ws.n:
@ -534,8 +473,47 @@ class Recorder:
self.handle(self.pending.pop(0))
self.handle(self.ws.recv_json())
FILL_LOGIN_JS = r"""
(creds) => {
if (!creds.expectedHost || location.protocol !== "https:" || location.hostname.toLowerCase() !== creds.expectedHost.toLowerCase()) {
return {ok: false, error: "saved login requires the exact configured HTTPS host"};
}
const user = creds.username || "";
const pass = creds.password || "";
const inputs = Array.from(document.querySelectorAll("input"));
const visible = (el) => {
const s = getComputedStyle(el);
const r = el.getBoundingClientRect();
return s.display !== "none" && s.visibility !== "hidden" && el.type !== "hidden" && r.width > 0 && r.height > 0;
};
const password = inputs.find((el) => el.type === "password" && visible(el) && !el.disabled);
if (!password) return {ok: false, error: "no password field on this page"};
const userish = /user|email|login|account|phone|id/i;
const username = inputs.find((el) => {
if (el === password || !visible(el) || el.disabled) return false;
const type = (el.type || "text").toLowerCase();
if (["email", "tel", "url"].includes(type)) return true;
if (type !== "text" && type !== "search") return false;
const blob = [el.name, el.id, el.placeholder, el.autocomplete, el.getAttribute("aria-label")].join(" ");
return userish.test(blob) || el === inputs[0];
});
function setValue(el, value) {
const proto = HTMLInputElement.prototype;
const desc = Object.getOwnPropertyDescriptor(proto, "value");
if (desc && desc.set) desc.set.call(el, value);
else el.value = value;
el.dispatchEvent(new Event("input", {bubbles: true}));
el.dispatchEvent(new Event("change", {bubbles: true}));
}
if (username) setValue(username, user);
setValue(password, pass);
return {ok: true, filledUsername: Boolean(username), submitted: false};
}
"""
def main():
req = json.loads(sys.argv[1])
raw = sys.argv[1] if len(sys.argv) > 1 and sys.argv[1].strip() else sys.stdin.read()
req = json.loads(raw)
action = req.get("action") or "snapshot"
display = req.get("display") or ":1"
profile = req.get("profile") or ""
@ -636,6 +614,23 @@ def main():
out["waitedSeconds"] = round(waited, 1)
print(json.dumps(out))
return
if action == "fill_login":
val = evaluate(ws, FILL_LOGIN_JS, {
"expectedHost": req.get("expectedHost") or "",
"username": req.get("username") or "",
"password": req.get("password") or "",
}) or {}
if not val.get("ok"):
fail(val.get("error") or "could not fill the login form")
wait_for_visual_update(ws)
print(json.dumps({
"ok": True,
"action": "fill_login",
"filledUsername": bool(val.get("filledUsername")),
"submitted": bool(val.get("submitted")),
"restarted": restarted,
}))
return
if action == "type":
sel = req.get("selector") or ""
text = req.get("text") or ""

View File

@ -42,6 +42,24 @@ pub fn cdp_command(request: &Value) -> Vec<String> {
cdp_command_on(PRIMARY_DISPLAY, None, request)
}
/// Same as `cdp_command_on` but the JSON body is meant to arrive on stdin
/// so secrets never appear on the process argv.
pub fn cdp_stdin_command_on(display: &str, profile: Option<&str>) -> Vec<String> {
let mut env = vec![
"env".into(),
format!("DISPLAY={}", normalize_display(display)),
];
if let Some(profile) = profile.filter(|value| !value.is_empty()) {
env.push(format!("LAZYBOY_BROWSER_PROFILE={profile}"));
}
env.extend([
"python3".into(),
"-c".into(),
CDP_PY.into(),
]);
env
}
/// Marker embedded in the recorder's argv so `pkill -f` can find exactly one
/// teaching session without touching other python processes.
pub fn teach_recorder_tag(skill_id: &str) -> String {
@ -132,21 +150,7 @@ pub fn parse_cdp_page(raw: &str) -> CdpPage {
}
pub fn merge_page_elements(windows: Vec<UiElement>, page: &[UiElement]) -> Vec<UiElement> {
if page.is_empty() {
return windows;
}
let mut elements = page.to_vec();
let mut next = elements.len() as u32 + 1;
for mut window in windows {
let title = window.title.to_lowercase();
if title.contains("chromium") || title.contains("chrome") {
continue;
}
window.id = next;
next += 1;
elements.push(window);
}
elements
crate::merge_ui_elements(windows, page, &[])
}
#[cfg(test)]

View File

@ -0,0 +1,51 @@
"""Set X11 clipboard, confirm ownership/content, then paste into the active app."""
import subprocess
import sys
import time
def run(argv, **kwargs):
return subprocess.run(argv, check=True, timeout=2, **kwargs)
def paste(text):
if not text:
return
raw = text.encode('utf-8')
# xclip forks after reading stdin; detached descriptors avoid pipe hangs.
run(['xclip', '-selection', 'clipboard', '-in'], input=raw,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
deadline = time.monotonic() + 2
while True:
actual = run(['xclip', '-selection', 'clipboard', '-out'], stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL).stdout
if actual == raw:
break
if time.monotonic() >= deadline:
raise RuntimeError('clipboard synchronization timed out; nothing pasted')
time.sleep(.02)
key_for_active_app('v')
def key_for_active_app(key):
window = run(['xdotool', 'getactivewindow'], stdout=subprocess.PIPE).stdout.decode().strip()
wmclass = run(['xprop', '-id', window, 'WM_CLASS'], stdout=subprocess.PIPE).stdout.decode().lower()
terminal = any(name in wmclass for name in ('terminal', 'xterm', 'kitty', 'alacritty', 'konsole'))
run(['xdotool', 'key', '--clearmodifiers', ('ctrl+shift+' if terminal else 'ctrl+') + key],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def copy_selection():
key_for_active_app('c')
# Wait for the application to process the shortcut before reading selection.
time.sleep(.1)
return run(['xclip', '-selection', 'clipboard', '-out'], stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL).stdout.decode('utf-8')
if __name__ == '__main__':
if len(sys.argv)>1 and sys.argv[1]=='copy':
sys.stdout.write(copy_selection())
else:
paste(sys.stdin.read())

View File

@ -1,3 +1,4 @@
mod a11y;
mod actions;
mod cdp;
mod lease;
@ -9,6 +10,7 @@ mod screen;
mod takeover;
mod x11;
pub use a11y::*;
pub use actions::*;
pub use cdp::*;
pub use lease::*;

View File

@ -58,6 +58,21 @@ pub struct CommandRequest {
pub argv: Vec<String>,
pub cwd: Option<String>,
pub timeout_ms: Option<u64>,
/// Optional bytes written to the process stdin, then closed.
/// Used so fill-login never puts a password on the argv of `ps`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub stdin: Option<String>,
}
impl Default for CommandRequest {
fn default() -> Self {
Self {
argv: Vec::new(),
cwd: None,
timeout_ms: None,
stdin: None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@ -193,17 +208,13 @@ pub trait SandboxProvider: Send + Sync {
&self,
computer: &ComputerRef,
context: &AdapterContext,
) -> Result<(), SandboxError> {
self.stop(computer, context).await
}
) -> Result<(), SandboxError>;
async fn resume(
&self,
request: ProvisionRequest,
context: &AdapterContext,
) -> Result<ComputerRef, SandboxError> {
self.provision(request, context).await
}
) -> Result<ComputerRef, SandboxError>;
async fn execute(
&self,

View File

@ -1,5 +1,10 @@
use lazyboy_contracts::{ComputerAction, PointerButton, PointerType, ScrollDirection};
pub fn is_browser_title(title: &str) -> bool {
let title = title.to_lowercase();
title.contains("chromium") || title.contains("chrome")
}
use crate::screen::{PRIMARY_DISPLAY, normalize_display};
pub const DISPLAY: &str = PRIMARY_DISPLAY;
@ -129,7 +134,8 @@ pub fn xdotool_argv_on(display: &str, action: &ComputerAction) -> Option<Vec<Str
}
ComputerAction::Wait { .. }
| ComputerAction::Open { .. }
| ComputerAction::Launch { .. } => {
| ComputerAction::Launch { .. }
| ComputerAction::Ref { .. } => {
return None;
}
}
@ -160,6 +166,7 @@ pub fn action_pause_ms(action: &ComputerAction) -> u64 {
ComputerAction::Focus { .. } => 90,
ComputerAction::Open { .. } | ComputerAction::Launch { .. } => 220,
ComputerAction::Wait { .. } => 0,
ComputerAction::Ref { .. } => 55,
}
}
@ -259,6 +266,11 @@ pub fn parse_ui_elements(raw: &str) -> Vec<lazyboy_contracts::UiElement> {
.and_then(serde_json::Value::as_str)
.filter(|value| !value.is_empty())
.map(str::to_string),
role: item
.get("role")
.and_then(serde_json::Value::as_str)
.filter(|value| !value.is_empty())
.map(str::to_string),
})
})
.collect()
@ -454,9 +466,38 @@ mod tests {
);
}
#[test]
fn parses_a11y_role_and_path() {
let elements = parse_ui_elements(
r#"[{"id":2,"title":"push button Open","selector":"0/3/1","kind":"a11y","role":"push button","x":8,"y":9,"w":40,"h":16}]"#,
);
assert_eq!(elements[0].kind.as_deref(), Some("a11y"));
assert_eq!(elements[0].role.as_deref(), Some("push button"));
assert_eq!(elements[0].selector.as_deref(), Some("0/3/1"));
assert!(elements[0].has_ref());
}
#[test]
fn browser_titles_match_chromium() {
assert!(is_browser_title("Example - Chromium"));
assert!(is_browser_title("chrome"));
assert!(!is_browser_title("Thunar"));
}
#[test]
fn empty_or_junk_window_list_is_empty() {
assert!(parse_ui_elements("").is_empty());
assert!(parse_ui_elements("not-json").is_empty());
}
}
/// Native clipboard path; text is supplied on stdin, never process arguments.
pub fn paste_command_on(display: &str) -> Vec<String> {
vec![
"env".into(),
display_env(display),
"python3".into(),
"-c".into(),
include_str!("clipboard.py").into(),
]
}

View File

@ -5,11 +5,12 @@ use axum::extract::State;
use axum::http::{HeaderMap, StatusCode};
use axum::routing::{get, post};
use axum::{Json, Router};
use lazyboy_contracts::ComputerAction;
use lazyboy_contracts::{ComputerAction, RefVerb};
use lazyboy_control::{
ActionRequest, PRIMARY_DISPLAY, action_pause_ms, launch_argv_on, normalize_display,
open_argv_on, parse_pointer_state, parse_ui_elements, pointer_state_command_on,
screenshot_command_on, window_list_command_on, xdotool_argv_on,
ActionRequest, PRIMARY_DISPLAY, a11y_command_on, action_pause_ms, cdp_command_on,
launch_argv_on, normalize_display, open_argv_on, parse_a11y_page, parse_cdp_page,
parse_pointer_state, parse_ui_elements, pointer_state_command_on, screenshot_command_on,
window_list_command_on, xdotool_argv_on,
};
use tokio::io::AsyncReadExt;
use tokio::process::Command;
@ -153,6 +154,12 @@ async fn apply_action(
.ok_or_else(|| "unknown application".to_string())?;
spawn_detached(&argv).await
}
ComputerAction::Ref {
verb,
target,
ref_kind,
text,
} => apply_ref(display, profile, *verb, target, ref_kind, text.as_deref()).await,
other => {
let argv =
xdotool_argv_on(display, other).ok_or_else(|| "unsupported action".to_string())?;
@ -170,6 +177,51 @@ async fn apply_action(
}
}
async fn apply_ref(
display: &str,
profile: Option<&str>,
verb: RefVerb,
target: &str,
kind: &str,
text: Option<&str>,
) -> Result<(), String> {
let action = match verb {
RefVerb::Click => "click",
RefVerb::SetValue => "type",
RefVerb::Focus => "focus",
};
let mut request = serde_json::json!({
"action": action,
"selector": target,
"display": display,
"ensure": false,
});
if let Some(text) = text {
request["text"] = serde_json::json!(text);
}
let argv = if kind == "dom" {
cdp_command_on(display, profile, &request)
} else {
a11y_command_on(display, &request)
};
let output = Command::new(&argv[0])
.args(&argv[1..])
.output()
.await
.map_err(|error| error.to_string())?;
let raw = if output.stdout.is_empty() {
String::from_utf8_lossy(&output.stderr).into_owned()
} else {
String::from_utf8_lossy(&output.stdout).into_owned()
};
let ok = if kind == "dom" {
parse_cdp_page(&raw).ok
} else {
parse_a11y_page(&raw).ok
};
if ok { Ok(()) } else { Err(raw) }
}
async fn spawn_detached(argv: &[String]) -> Result<(), String> {
Command::new(&argv[0])
.args(&argv[1..])
@ -186,14 +238,15 @@ async fn observation_json(display: &str, png: Vec<u8>) -> serde_json::Value {
let mut body = serde_json::json!({
"png_base64": base64::engine::general_purpose::STANDARD.encode(png)
});
let (cursor, window) = run_pointer_state(display).await;
let ((cursor, window), elements) =
tokio::join!(run_pointer_state(display), run_window_list(display));
if let Some(cursor) = cursor {
body["cursor"] = serde_json::json!({ "x": cursor.x, "y": cursor.y });
}
if let Some(window) = window {
body["activeWindow"] = serde_json::json!({ "id": window.id, "title": window.title });
}
let elements = run_window_list(display).await;
if !elements.is_empty() {
body["elements"] = serde_json::to_value(elements).unwrap_or(serde_json::json!([]));
}

View File

@ -8,5 +8,16 @@ publish.workspace = true
[dependencies]
lazyboy-contracts.workspace = true
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
rig-core.workspace = true
async-trait.workspace = true
tokio.workspace = true
tokio-tungstenite.workspace = true
futures-util.workspace = true
base64.workspace = true
http = "1"
reqwest.workspace = true
rustls.workspace = true
rustls-native-certs = "0.8"

View File

@ -0,0 +1,64 @@
//! Execution policy belongs to the harness, independently of task playbooks.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ExecutionMode {
Bounded(u32),
Goal,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum GoalOutcome {
Continue,
Complete,
NeedsInput,
}
pub fn goal_request(prompt: &str) -> Option<&str> {
let rest = prompt.trim().strip_prefix("/goal")?;
(rest.is_empty() || rest.starts_with(char::is_whitespace)).then(|| rest.trim())
}
impl ExecutionMode {
pub fn allows_turn(self, turns: u32) -> bool {
match self {
Self::Goal => true,
Self::Bounded(limit) => turns < limit,
}
}
}
/// Only a standalone terminal marker is an outcome, not a quoted mention.
pub fn goal_outcome(reply: &str) -> GoalOutcome {
match reply.trim().lines().last().map(str::trim) {
Some("[GOAL_COMPLETE]") => GoalOutcome::Complete,
Some("[GOAL_BLOCKED]") => GoalOutcome::NeedsInput,
_ => GoalOutcome::Continue,
}
}
pub const GOAL_INSTRUCTIONS: &str = "Persistent goal execution: plan the requested work, execute it, and verify each requested outcome. Intermediate progress replies do not finish the run. Preserve completed work and incorporate user steering. End your final reply with a standalone [GOAL_COMPLETE] line only when all outcomes are verified; explain the verification. When required information or human action is missing, explain exactly what is needed and end with a standalone [GOAL_BLOCKED] line. For a simple Cloudflare connection-check checkbox, observe the current screen and try connection_check once, then verify the requested content. For other CAPTCHA, failed verification, login or 2FA use request_takeover. Never claim completion merely because you planned the work.";
pub const GOAL_CONTINUE: &str = "The goal remains active. Continue the plan with tools and verify the outcome. Finish only with a standalone [GOAL_COMPLETE] line after verification, or [GOAL_BLOCKED] when required human input is missing.";
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn goals_are_unbounded_while_normal_runs_remain_bounded() {
assert!(ExecutionMode::Goal.allows_turn(40));
assert!(ExecutionMode::Goal.allows_turn(u32::MAX));
assert!(!ExecutionMode::Bounded(40).allows_turn(40));
}
#[test]
fn goal_command_requires_a_token_boundary() {
assert_eq!(goal_request(" /goal\nfinish this"), Some("finish this"));
assert_eq!(goal_request("/goalkeeper"), None);
}
#[test]
fn progress_and_quoted_markers_do_not_complete_a_goal() {
assert_eq!(goal_outcome("Next I will use [GOAL_COMPLETE]."), GoalOutcome::Continue);
assert_eq!(goal_outcome("Verified output.\n[GOAL_COMPLETE]"), GoalOutcome::Complete);
assert_eq!(goal_outcome("Please supply the date.\n[GOAL_BLOCKED]"), GoalOutcome::NeedsInput);
}
}

View File

@ -1,3 +1,6 @@
mod resolve;
pub mod execution;
mod voice;
pub use resolve::*;
pub use voice::*;

View File

@ -106,8 +106,13 @@ fn rewrite_loopback_host(url: &str) -> String {
if !supervisor.contains("://supervisor") {
return url.to_string();
}
url.replace("://127.0.0.1", "://host.docker.internal")
.replace("://localhost", "://host.docker.internal")
let Ok(mut parsed) = reqwest::Url::parse(url) else {
return url.to_string();
};
if matches!(parsed.host_str(), Some("127.0.0.1" | "localhost" | "[::1]")) {
let _ = parsed.set_host(Some("host.docker.internal"));
}
parsed.to_string().trim_end_matches('/').to_string()
}
pub enum DynModel {

741
crates/harness/src/voice.rs Normal file
View File

@ -0,0 +1,741 @@
use std::sync::Arc;
use async_trait::async_trait;
use base64::Engine;
use futures_util::{SinkExt, StreamExt};
use lazyboy_contracts::{
VoiceProvider, catalog_voice_models, catalog_voices, computer_voice_tools,
};
use serde_json::{Value, json};
use thiserror::Error;
use tokio::sync::{Mutex, mpsc};
use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use crate::{CredentialChain, ModelError};
#[derive(Debug, Error, Clone, PartialEq, Eq)]
pub enum VoiceError {
#[error("{0}")]
Message(String),
#[error("missing credential for {provider} ({env_key})")]
MissingCredential { provider: String, env_key: String },
#[error("unknown voice provider: {0}")]
UnknownProvider(String),
}
impl From<ModelError> for VoiceError {
fn from(error: ModelError) -> Self {
match error {
ModelError::MissingCredential { provider, env_key } => {
Self::MissingCredential { provider, env_key }
}
other => Self::Message(other.to_string()),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct VoiceCatalogEntry {
pub id: String,
pub name: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct VoiceCatalog {
pub provider: VoiceProvider,
pub models: Vec<VoiceCatalogEntry>,
pub voices: Vec<VoiceCatalogEntry>,
}
pub fn voice_catalog(provider: VoiceProvider) -> VoiceCatalog {
VoiceCatalog {
provider,
models: catalog_voice_models(provider)
.iter()
.map(|(id, name)| VoiceCatalogEntry {
id: (*id).into(),
name: (*name).into(),
})
.collect(),
voices: catalog_voices(provider)
.iter()
.map(|(id, name)| VoiceCatalogEntry {
id: (*id).into(),
name: (*name).into(),
})
.collect(),
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ResolveVoiceRequest {
pub provider: VoiceProvider,
pub model_id: Option<String>,
pub voice_id: Option<String>,
pub credentials: CredentialChain,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ResolvedVoice {
pub provider: VoiceProvider,
pub model_id: String,
pub voice_id: String,
pub api_key: String,
}
pub fn resolve_voice(request: ResolveVoiceRequest) -> Result<ResolvedVoice, VoiceError> {
let api_key = match request.credentials.resolve() {
Some(key) => key.to_string(),
None if request.provider.requires_api_key() => {
return Err(VoiceError::MissingCredential {
provider: request.provider.as_str().to_string(),
env_key: request.provider.env_key_name().to_string(),
});
}
None => String::new(),
};
let model_id = request
.model_id
.filter(|value| !value.is_empty())
.unwrap_or_else(|| request.provider.default_model_id().to_string());
let voice_id = request
.voice_id
.filter(|value| !value.is_empty())
.unwrap_or_else(|| request.provider.default_voice_id().to_string());
Ok(ResolvedVoice {
provider: request.provider,
model_id,
voice_id,
api_key,
})
}
#[derive(Debug, Clone)]
pub enum VoiceEvent {
AudioPcm(Vec<u8>),
SpeechStarted,
SpeechStopped,
InputTranscript { text: String, final_: bool },
OutputTranscript { text: String, final_: bool },
FunctionCall {
call_id: String,
name: String,
arguments: String,
},
FunctionCallOutput { call_id: String, output: String },
SpeakNow { text: String },
InjectContext { text: String },
ResponseCreate,
ResponseStarted,
ResponseFinished,
CancelResponse,
Error { message: String },
}
#[derive(Debug, Clone)]
pub struct VoiceConnectRequest {
pub api_key: String,
pub model_id: String,
pub voice_id: String,
pub instructions: String,
pub tools: Vec<Value>,
pub history: Vec<(String, String)>,
}
impl VoiceConnectRequest {
pub fn from_resolved(resolved: &ResolvedVoice, instructions: String) -> Self {
Self {
api_key: resolved.api_key.clone(),
model_id: resolved.model_id.clone(),
voice_id: resolved.voice_id.clone(),
instructions,
tools: computer_voice_tools(),
history: Vec::new(),
}
}
}
#[async_trait]
pub trait VoiceRealtime: Send + Sync {
fn provider(&self) -> VoiceProvider;
async fn connect(
&self,
request: VoiceConnectRequest,
) -> Result<Box<dyn VoiceSocket>, VoiceError>;
}
#[async_trait]
pub trait VoiceSocket: Send {
async fn send(&self, event: VoiceEvent) -> Result<(), VoiceError>;
async fn recv(&mut self) -> Result<Option<VoiceEvent>, VoiceError>;
}
pub fn create_voice(provider: VoiceProvider) -> Box<dyn VoiceRealtime> {
match provider {
VoiceProvider::Scripted => Box::new(ScriptedVoice),
other => Box::new(HostedVoice { provider: other }),
}
}
pub fn scripted_voice_enabled() -> bool {
matches!(
std::env::var("LAZYBOY_VOICE_SCRIPTED").as_deref(),
Ok("1" | "true" | "yes")
)
}
fn native_roots() -> Result<rustls::RootCertStore, VoiceError> {
let mut roots = rustls::RootCertStore::empty();
let certs = rustls_native_certs::load_native_certs();
roots.add_parsable_certificates(certs.certs);
if roots.is_empty() {
return Err(VoiceError::Message("No trusted TLS certificates available".into()));
}
Ok(roots)
}
struct HostedVoice {
provider: VoiceProvider,
}
#[async_trait]
impl VoiceRealtime for HostedVoice {
fn provider(&self) -> VoiceProvider {
self.provider
}
async fn connect(
&self,
request: VoiceConnectRequest,
) -> Result<Box<dyn VoiceSocket>, VoiceError> {
let url = self.provider.realtime_url(&request.model_id);
let mut http_request = url
.as_str()
.into_client_request()
.map_err(|error| VoiceError::Message(error.to_string()))?;
let header = format!("Bearer {}", request.api_key);
http_request.headers_mut().insert(
"Authorization",
http::HeaderValue::from_str(&header)
.map_err(|error| VoiceError::Message(error.to_string()))?,
);
// Choose explicitly: the dependency graph enables both ring and aws-lc-rs.
// Rustls's automatic provider selection panics in that configuration.
let tls = rustls::ClientConfig::builder_with_provider(
Arc::new(rustls::crypto::ring::default_provider()),
)
.with_safe_default_protocol_versions()
.map_err(|error| VoiceError::Message(error.to_string()))?
.with_root_certificates(native_roots()?)
.with_no_client_auth();
let (stream, _) = tokio_tungstenite::connect_async_tls_with_config(
http_request, None, false, Some(tokio_tungstenite::Connector::Rustls(Arc::new(tls))),
)
.await
.map_err(|error| VoiceError::Message(error.to_string()))?;
let (write, read) = stream.split();
let socket = HostedSocket {
provider: self.provider,
write: Mutex::new(write),
read,
};
socket
.send_raw(Message::Text(session_update_json(self.provider, &request).into()))
.await?;
for (role, text) in &request.history {
if text.trim().is_empty() {
continue;
}
let item = json!({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": role,
"content": [{
"type": if *role == "assistant" { "output_text" } else { "input_text" },
"text": text
}]
}
});
socket
.send_raw(Message::Text(item.to_string().into()))
.await?;
}
Ok(Box::new(socket))
}
}
type HostedWrite =
futures_util::stream::SplitSink<tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>, Message>;
type HostedRead =
futures_util::stream::SplitStream<tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>>;
struct HostedSocket {
provider: VoiceProvider,
write: Mutex<HostedWrite>,
read: HostedRead,
}
impl HostedSocket {
async fn send_raw(&self, message: Message) -> Result<(), VoiceError> {
self.write
.lock()
.await
.send(message)
.await
.map_err(|error| VoiceError::Message(error.to_string()))
}
}
#[async_trait]
impl VoiceSocket for HostedSocket {
async fn send(&self, event: VoiceEvent) -> Result<(), VoiceError> {
match encode_provider_event(self.provider, &event) {
Some(message) => self.send_raw(message).await,
None => Ok(()),
}
}
async fn recv(&mut self) -> Result<Option<VoiceEvent>, VoiceError> {
loop {
match self.read.next().await {
None => return Ok(None),
Some(Err(error)) => return Err(VoiceError::Message(error.to_string())),
Some(Ok(Message::Close(_))) => return Ok(None),
Some(Ok(Message::Ping(payload))) => {
let _ = self.send_raw(Message::Pong(payload)).await;
}
Some(Ok(Message::Pong(_))) => {}
Some(Ok(Message::Frame(_))) => {}
Some(Ok(Message::Binary(bytes))) => {
return Ok(Some(VoiceEvent::AudioPcm(bytes.to_vec())));
}
Some(Ok(Message::Text(text))) => {
if let Some(event) = parse_provider_event(&text) {
return Ok(Some(event));
}
}
}
}
}
}
fn session_update_json(provider: VoiceProvider, request: &VoiceConnectRequest) -> String {
let mut session = json!({
"voice": request.voice_id,
"instructions": request.instructions,
"turn_detection": { "type": "server_vad" },
"tools": request.tools,
"audio": {
"input": {
"format": { "type": "audio/pcm", "rate": 24000 },
"transport": "binary"
},
"output": {
"format": { "type": "audio/pcm", "rate": 24000 },
"transport": "binary"
}
}
});
if provider == VoiceProvider::Openai {
session.as_object_mut().unwrap().remove("voice");
session.as_object_mut().unwrap().remove("turn_detection");
session["type"] = json!("realtime");
session["audio"] = json!({
"input": {
"format": { "type": "audio/pcm", "rate": 24000 },
"turn_detection": { "type": "server_vad" }
},
"output": {
"format": { "type": "audio/pcm", "rate": 24000 },
"voice": request.voice_id
}
});
}
json!({ "type": "session.update", "session": session }).to_string()
}
pub fn encode_provider_event(provider: VoiceProvider, event: &VoiceEvent) -> Option<Message> {
match event {
VoiceEvent::AudioPcm(bytes) if provider == VoiceProvider::Openai => Some(Message::Text(
json!({ "type": "input_audio_buffer.append", "audio": base64::engine::general_purpose::STANDARD.encode(bytes) }).to_string().into(),
)),
VoiceEvent::AudioPcm(bytes) => Some(Message::Binary(bytes.clone().into())),
VoiceEvent::FunctionCallOutput { call_id, output } => Some(Message::Text(
json!({
"type": "conversation.item.create",
"item": {
"type": "function_call_output",
"call_id": call_id,
"output": output
}
})
.to_string()
.into(),
)),
VoiceEvent::ResponseCreate => Some(Message::Text(
json!({ "type": "response.create" }).to_string().into(),
)),
VoiceEvent::CancelResponse => Some(Message::Text(
json!({ "type": "response.cancel" }).to_string().into(),
)),
VoiceEvent::InjectContext { text } => Some(Message::Text(
json!({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{ "type": "input_text", "text": text }]
}
})
.to_string()
.into(),
)),
VoiceEvent::SpeakNow { text } if provider == VoiceProvider::Xai => Some(Message::Text(
json!({
"type": "conversation.item.create",
"item": {
"type": "force_message",
"role": "assistant",
"interruptible": true,
"content": [{ "type": "output_text", "text": text }]
}
})
.to_string()
.into(),
)),
VoiceEvent::SpeakNow { text } => Some(Message::Text(
json!({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "assistant",
"content": [{ "type": "output_text", "text": text }]
}
})
.to_string()
.into(),
)),
_ => None,
}
}
pub fn parse_provider_event(text: &str) -> Option<VoiceEvent> {
let event: Value = serde_json::from_str(text).ok()?;
let kind = event.get("type")?.as_str()?;
match kind {
"input_audio_buffer.speech_started" => Some(VoiceEvent::SpeechStarted),
"input_audio_buffer.speech_stopped" => Some(VoiceEvent::SpeechStopped),
"response.created" => Some(VoiceEvent::ResponseStarted),
"response.done" | "response.completed" | "response.cancelled" => {
Some(VoiceEvent::ResponseFinished)
}
"conversation.item.input_audio_transcription.delta"
| "response.input_audio_transcription.delta" => {
let text = event.get("delta")?.as_str()?.to_string();
Some(VoiceEvent::InputTranscript {
text,
final_: false,
})
}
"conversation.item.input_audio_transcription.completed"
| "conversation.item.input_audio_transcription.done"
| "response.input_audio_transcription.completed" => {
let text = event
.get("transcript")
.and_then(Value::as_str)
.or_else(|| event.get("text").and_then(Value::as_str))?
.to_string();
Some(VoiceEvent::InputTranscript {
text,
final_: true,
})
}
"response.output_audio.delta" | "response.audio.delta" => {
let encoded = event.get("delta").and_then(Value::as_str)?;
let bytes = base64::engine::general_purpose::STANDARD
.decode(encoded)
.ok()?;
Some(VoiceEvent::AudioPcm(bytes))
}
"response.output_audio_transcript.delta" | "response.audio_transcript.delta" => {
let text = event.get("delta")?.as_str()?.to_string();
Some(VoiceEvent::OutputTranscript {
text,
final_: false,
})
}
"response.output_audio_transcript.done" | "response.audio_transcript.done" => {
let text = event
.get("transcript")
.and_then(Value::as_str)
.or_else(|| event.get("text").and_then(Value::as_str))
.unwrap_or("")
.to_string();
Some(VoiceEvent::OutputTranscript {
text,
final_: true,
})
}
"response.function_call_arguments.done" => {
let call_id = event.get("call_id")?.as_str()?.to_string();
let name = event.get("name")?.as_str()?.to_string();
let arguments = event
.get("arguments")
.and_then(Value::as_str)
.unwrap_or("{}")
.to_string();
Some(VoiceEvent::FunctionCall {
call_id,
name,
arguments,
})
}
"error" => {
let message = event
.pointer("/error/message")
.and_then(Value::as_str)
.or_else(|| event.get("message").and_then(Value::as_str))
.unwrap_or("voice error")
.to_string();
Some(VoiceEvent::Error { message })
}
_ => None,
}
}
struct ScriptedVoice;
#[async_trait]
impl VoiceRealtime for ScriptedVoice {
fn provider(&self) -> VoiceProvider {
VoiceProvider::Scripted
}
async fn connect(
&self,
_request: VoiceConnectRequest,
) -> Result<Box<dyn VoiceSocket>, VoiceError> {
Ok(Box::new(ScriptedSocket::new()))
}
}
struct ScriptedSocket {
incoming: Mutex<mpsc::UnboundedSender<VoiceEvent>>,
outgoing: mpsc::UnboundedReceiver<VoiceEvent>,
heard: Arc<Mutex<bool>>,
}
impl ScriptedSocket {
fn new() -> Self {
let (tx, rx) = mpsc::unbounded_channel();
Self {
incoming: Mutex::new(tx),
outgoing: rx,
heard: Arc::new(Mutex::new(false)),
}
}
fn beep() -> Vec<u8> {
// 80 ms of 24 kHz PCM16 silence so tests have a non-empty clip.
vec![0; 24000 / 12 * 2]
}
}
#[async_trait]
impl VoiceSocket for ScriptedSocket {
async fn send(&self, event: VoiceEvent) -> Result<(), VoiceError> {
match event {
VoiceEvent::AudioPcm(bytes) if !bytes.is_empty() => {
let mut heard = self.heard.lock().await;
if !*heard {
*heard = true;
let tx = self.incoming.lock().await;
let _ = tx.send(VoiceEvent::SpeechStarted);
let _ = tx.send(VoiceEvent::InputTranscript {
text: "hello from the test microphone".into(),
final_: true,
});
let _ = tx.send(VoiceEvent::FunctionCall {
call_id: "call_scripted".into(),
name: "start_computer_task".into(),
arguments: json!({ "prompt": "open the browser" }).to_string(),
});
}
}
VoiceEvent::FunctionCallOutput { .. } => {
let tx = self.incoming.lock().await;
let _ = tx.send(VoiceEvent::OutputTranscript {
text: "好,我去電腦上開。".into(),
final_: true,
});
let _ = tx.send(VoiceEvent::AudioPcm(Self::beep()));
}
VoiceEvent::SpeakNow { text } | VoiceEvent::InjectContext { text } => {
let tx = self.incoming.lock().await;
let _ = tx.send(VoiceEvent::OutputTranscript {
text,
final_: true,
});
let _ = tx.send(VoiceEvent::AudioPcm(Self::beep()));
}
VoiceEvent::ResponseCreate => {}
_ => {}
}
Ok(())
}
async fn recv(&mut self) -> Result<Option<VoiceEvent>, VoiceError> {
Ok(self.outgoing.recv().await)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn voice_tls_config_uses_an_explicit_provider() {
let config = rustls::ClientConfig::builder_with_provider(
Arc::new(rustls::crypto::ring::default_provider()),
).with_safe_default_protocol_versions().unwrap()
.with_root_certificates(native_roots().unwrap()).with_no_client_auth();
assert!(!config.crypto_provider().cipher_suites.is_empty());
}
#[test]
fn openai_audio_uses_json_and_nested_session_settings() {
let request = VoiceConnectRequest {
api_key: String::new(), model_id: "gpt-realtime".into(),
voice_id: "marin".into(), instructions: "test".into(),
tools: vec![], history: vec![],
};
let session: Value = serde_json::from_str(&session_update_json(VoiceProvider::Openai, &request)).unwrap();
assert!(session["session"].get("voice").is_none());
assert!(session["session"].get("turn_detection").is_none());
assert_eq!(session.pointer("/session/audio/input/format/rate"), Some(&json!(24000)));
assert_eq!(session.pointer("/session/audio/input/turn_detection/type"), Some(&json!("server_vad")));
let event = VoiceEvent::AudioPcm(vec![0, 1, 2, 3]);
let Some(Message::Text(text)) = encode_provider_event(VoiceProvider::Openai, &event) else { panic!("expected JSON audio") };
let encoded: Value = serde_json::from_str(&text).unwrap();
assert_eq!(encoded["type"], "input_audio_buffer.append");
assert_eq!(encoded["audio"], "AAECAw==");
assert!(matches!(encode_provider_event(VoiceProvider::Xai, &event), Some(Message::Binary(_))));
}
#[test]
fn resolve_voice_defaults_and_requires_key() {
let missing = resolve_voice(ResolveVoiceRequest {
provider: VoiceProvider::Xai,
model_id: None,
voice_id: None,
credentials: CredentialChain {
bot: None,
space: None,
env: None,
},
});
assert!(matches!(missing, Err(VoiceError::MissingCredential { .. })));
let ok = resolve_voice(ResolveVoiceRequest {
provider: VoiceProvider::Xai,
model_id: None,
voice_id: None,
credentials: CredentialChain {
bot: None,
space: Some("sk".into()),
env: None,
},
})
.unwrap();
assert_eq!(ok.model_id, "grok-voice-latest");
assert_eq!(ok.voice_id, "eve");
assert_eq!(ok.api_key, "sk");
}
#[test]
fn parses_xai_and_openai_event_aliases() {
let started = parse_provider_event(
r#"{"type":"input_audio_buffer.speech_started"}"#,
);
assert!(matches!(started, Some(VoiceEvent::SpeechStarted)));
let transcript = parse_provider_event(
r#"{"type":"conversation.item.input_audio_transcription.completed","transcript":"hello"}"#,
);
match transcript {
Some(VoiceEvent::InputTranscript { text, final_ }) => {
assert_eq!(text, "hello");
assert!(final_);
}
other => panic!("{other:?}"),
}
let old_audio = parse_provider_event(
r#"{"type":"response.audio.delta","delta":"AQID"}"#,
);
assert!(matches!(old_audio, Some(VoiceEvent::AudioPcm(_))));
let tool = parse_provider_event(
r#"{"type":"response.function_call_arguments.done","call_id":"1","name":"start_computer_task","arguments":"{}"}"#,
);
match tool {
Some(VoiceEvent::FunctionCall { name, .. }) => {
assert_eq!(name, "start_computer_task");
}
other => panic!("{other:?}"),
}
}
#[tokio::test]
async fn scripted_voice_emits_transcript_and_tool() {
let provider = create_voice(VoiceProvider::Scripted);
let mut socket = provider
.connect(VoiceConnectRequest {
api_key: String::new(),
model_id: "scripted-voice".into(),
voice_id: "scripted".into(),
instructions: "test".into(),
tools: computer_voice_tools(),
history: Vec::new(),
})
.await
.unwrap();
socket
.send(VoiceEvent::AudioPcm(vec![0, 1, 2, 3]))
.await
.unwrap();
let mut kinds = Vec::new();
for _ in 0..3 {
match socket.recv().await.unwrap() {
Some(VoiceEvent::SpeechStarted) => kinds.push("start"),
Some(VoiceEvent::InputTranscript { final_: true, .. }) => kinds.push("in"),
Some(VoiceEvent::FunctionCall { name, .. }) => {
assert_eq!(name, "start_computer_task");
kinds.push("tool");
}
other => panic!("{other:?}"),
}
}
assert_eq!(kinds, ["start", "in", "tool"]);
socket
.send(VoiceEvent::FunctionCallOutput {
call_id: "call_scripted".into(),
output: json!({"status":"queued"}).to_string(),
})
.await
.unwrap();
match socket.recv().await.unwrap() {
Some(VoiceEvent::OutputTranscript { text, final_ }) => {
assert!(final_);
assert!(text.contains("電腦"));
}
other => panic!("{other:?}"),
}
match socket.recv().await.unwrap() {
Some(VoiceEvent::AudioPcm(bytes)) => assert!(!bytes.is_empty()),
other => panic!("{other:?}"),
}
}
}

View File

@ -57,6 +57,29 @@ impl DockerSandbox {
fn url(&self, path: &str) -> String {
format!("{}{path}", self.base_url)
}
async fn post_lifecycle(
&self,
computer: &ComputerRef,
context: &AdapterContext,
action: &str,
) -> Result<(), SandboxError> {
let response = self
.client
.post(self.url(&format!("/computers/{}/{action}", computer.id)))
.headers(self.headers(context))
.send()
.await
.map_err(|error| SandboxError::message(error.to_string()))?;
if response.status().is_success() {
Ok(())
} else {
Err(SandboxError::message(format!(
"{action} failed: {}",
response.status()
)))
}
}
}
#[async_trait]
@ -345,18 +368,44 @@ impl SandboxProvider for DockerSandbox {
}
}
async fn suspend(
&self,
computer: &ComputerRef,
context: &AdapterContext,
) -> Result<(), SandboxError> {
self.post_lifecycle(computer, context, "pause").await
}
async fn resume(
&self,
request: ProvisionRequest,
context: &AdapterContext,
) -> Result<ComputerRef, SandboxError> {
if let Some(provider_ref) = request.provider_ref.clone() {
let computer = ComputerRef {
id: provider_ref.clone(),
home_key: request.home_key.clone(),
kind: SandboxKind::Docker,
provider_ref,
fresh: false,
};
if self
.post_lifecycle(&computer, context, "unpause")
.await
.is_ok()
{
return Ok(computer);
}
}
self.provision(request, context).await
}
async fn stop(
&self,
computer: &ComputerRef,
context: &AdapterContext,
) -> Result<(), SandboxError> {
self.client
.post(self.url(&format!("/computers/{}/stop", computer.id)))
.headers(self.headers(context))
.send()
.await
.map_err(|error| SandboxError::message(error.to_string()))?;
Ok(())
self.post_lifecycle(computer, context, "stop").await
}
async fn destroy(

View File

@ -166,6 +166,24 @@ impl SandboxProvider for FakeSandbox {
Ok(())
}
async fn suspend(
&self,
_computer: &ComputerRef,
_context: &AdapterContext,
) -> Result<(), SandboxError> {
Ok(())
}
async fn resume(
&self,
request: ProvisionRequest,
context: &AdapterContext,
) -> Result<ComputerRef, SandboxError> {
let mut computer = self.provision(request, context).await?;
computer.fresh = false;
Ok(computer)
}
async fn stop(
&self,
_computer: &ComputerRef,

View File

@ -21,3 +21,7 @@ base64.workspace = true
reqwest.workspace = true
async-trait = "0.1"
futures-util = "0.3"
hmac.workspace = true
sha2.workspace = true
hex.workspace = true

View File

@ -9,14 +9,13 @@ use bollard::container::{
StartContainerOptions, StopContainerOptions,
};
use bollard::exec::{CreateExecOptions, StartExecOptions, StartExecResults};
use bollard::models::{HostConfig, PortBinding};
use bollard::models::{HostConfig, HostConfigLogConfig, PortBinding};
use bollard::network::CreateNetworkOptions;
use futures_util::StreamExt;
use lazyboy_control::{
ActionRequest, CommandRequest, CommandResult, EnsureScreenRequest, EnsureScreenResult, HOME,
ScreenTarget, TEAM_SCREEN_LIMIT, action_pause_ms, launch_argv_on, normalize_display,
normalize_workspace_path, open_argv_on, pointer_state_command_on, screen_layout,
screenshot_command_on, window_list_command_on, xdotool_argv_on,
ScreenTarget, TEAM_SCREEN_LIMIT, normalize_display, normalize_workspace_path,
pointer_state_command_on, screen_layout, screenshot_command_on, window_list_command_on,
};
use tokio::time::{Duration, sleep};
@ -68,22 +67,40 @@ impl DockerHost {
home_path: &str,
space_id: &str,
) -> Result<Provisioned, String> {
let home_path = host_bind_path(home_path);
tokio::fs::create_dir_all(&home_path)
let data_dir = std::env::var("DATA_DIR").unwrap_or_else(|_| "./data".into());
if home_key.is_empty()
|| !home_key
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b"-_".contains(&b))
{
return Err("invalid home key".into());
}
let expected = PathBuf::from(&data_dir).join("homes").join(home_key);
if PathBuf::from(home_path) != expected {
return Err("home path is outside managed homes".into());
}
// Work on the container-local path, not the host daemon's bind path.
tokio::fs::create_dir_all(&expected)
.await
.map_err(|error| error.to_string())?;
let _ = tokio::process::Command::new("chown")
.args(["-R", "1000:1000", &home_path])
.status()
.await;
.map_err(|e| e.to_string())?;
let canonical = tokio::fs::canonicalize(&expected)
.await
.map_err(|e| e.to_string())?;
let root = tokio::fs::canonicalize(&data_dir)
.await
.map_err(|e| e.to_string())?;
if canonical != root.join("homes").join(home_key) {
return Err("symlinked home is not allowed".into());
}
#[cfg(unix)]
if std::env::var("HOST_DATA_DIR").is_ok() {
std::os::unix::fs::chown(&canonical, Some(1000), Some(1000))
.map_err(|e| e.to_string())?;
}
let home_path = host_bind_path(home_path);
if let Some(existing) = self.find(home_key).await? {
if self.container_reusable(&existing).await.unwrap_or(false) {
self.docker
.start_container(&existing, None::<StartContainerOptions<String>>)
.await
.ok();
self.wait_running(&existing).await?;
self.wait_ready(&existing).await?;
self.wake(&existing).await?;
let screen_url = self.screen_url(&existing, false).await.ok();
return Ok(Provisioned {
id: existing,
@ -100,6 +117,7 @@ impl DockerHost {
let mut labels = HashMap::new();
labels.insert("lazyboy.homeKey".into(), home_key.to_string());
labels.insert("lazyboy.spaceId".into(), space_id.to_string());
labels.insert("lazyboy.controlVersion".into(), "3".into());
let mut port_bindings = HashMap::new();
let mut exposed = HashMap::new();
@ -116,13 +134,37 @@ impl DockerHost {
}
let host_config = HostConfig {
binds: Some(vec![format!("{home_path}:{HOME}")]),
log_config: Some(HostConfigLogConfig {
typ: Some("json-file".into()),
config: Some(HashMap::from([
("max-size".into(), "10m".into()),
("max-file".into(), "3".into()),
])),
}),
binds: Some(
std::iter::once(format!("{home_path}:{HOME}"))
.chain(lxcfs_binds())
.collect(),
),
port_bindings: Some(port_bindings),
memory: Some(computer_memory_bytes()),
nano_cpus: Some(computer_nano_cpus()),
pids_limit: Some(computer_pids_limit()),
cap_drop: Some(vec!["ALL".into()]),
security_opt: Some(vec!["no-new-privileges:true".into()]),
cap_drop: if computer_sudo_enabled() {
None
} else {
Some(vec!["ALL".into()])
},
cap_add: if computer_sudo_enabled() {
None
} else {
Some(vec!["SETUID".into(), "SETGID".into()])
},
security_opt: if computer_sudo_enabled() {
None
} else {
Some(vec!["no-new-privileges:true".into()])
},
privileged: Some(false),
shm_size: Some(512 * 1024 * 1024),
network_mode: Some(network),
@ -131,12 +173,22 @@ impl DockerHost {
let config = Config {
image: Some(self.image.clone()),
user: Some("1000:1000".into()),
// The entrypoint starts as root so it can apply the env-only sudo
// policy, then drops the desktop process to the unprivileged user.
// Exec requests below still run explicitly as 1000:1000.
user: None,
hostname: Some(name.clone()),
env: Some(vec![
"DISPLAY=:1".into(),
format!("HOME={HOME}"),
format!("LAZYBOY_CONTROL_TOKEN={}", self.control_token),
format!(
"LAZYBOY_CONTROL_TOKEN={}",
scoped_control_token(&self.control_token, home_key)
),
format!(
"LAZYBOY_COMPUTER_SUDO={}",
if computer_sudo_enabled() { "true" } else { "false" }
),
]),
labels: Some(labels),
exposed_ports: Some(exposed),
@ -212,12 +264,26 @@ impl DockerHost {
}
None => HOME.to_string(),
};
let timeout_ms = request.timeout_ms.unwrap_or(30_000).clamp(100, 120_000);
let argv = if request.argv.is_empty() {
vec!["/bin/echo".into(), "ready".into()]
} else {
request.argv
};
self.exec_argv(id, &argv, Some(&cwd), &ScreenTarget::default())
let mut bounded = vec![
"timeout".into(),
"--signal=TERM".into(),
"--kill-after=2s".into(),
format!("{}s", timeout_ms as f64 / 1000.0),
];
bounded.extend(argv);
self.exec_raw_cmd(
id,
&bounded,
Some(&cwd),
&ScreenTarget::default(),
request.stdin,
)
.await
}
@ -239,12 +305,21 @@ impl DockerHost {
}
None => HOME.to_string(),
};
let timeout_ms = request.timeout_ms.unwrap_or(30_000).clamp(100, 120_000);
let argv = if request.argv.is_empty() {
vec!["/bin/echo".into(), "ready".into()]
} else {
request.argv
};
self.exec_argv(id, &argv, Some(&cwd), target).await
let mut bounded = vec![
"timeout".into(),
"--signal=TERM".into(),
"--kill-after=2s".into(),
format!("{}s", timeout_ms as f64 / 1000.0),
];
bounded.extend(argv);
self.exec_raw_cmd(id, &bounded, Some(&cwd), target, request.stdin)
.await
}
pub async fn observe(&self, id: &str) -> Result<Vec<u8>, String> {
@ -263,7 +338,13 @@ impl DockerHost {
value
} else {
let (stdout, stderr, code) = self
.exec_raw(id, &screenshot_command_on(&target.display), None, target)
.exec_raw(
id,
&screenshot_command_on(&target.display),
None,
target,
None,
)
.await?;
if code != 0 {
return Err(String::from_utf8_lossy(&stderr).into_owned());
@ -343,63 +424,8 @@ impl DockerHost {
request.profile_path.as_deref(),
None,
);
if let Ok(body) = self.control_act(id, &request, &target).await {
return Ok(body);
}
let mut completed = 0usize;
for action in &request.actions {
match action {
lazyboy_contracts::ComputerAction::Wait { ms } => {
sleep(Duration::from_millis(*ms as u64)).await;
}
lazyboy_contracts::ComputerAction::Open { path } => {
let _ = self
.exec_argv(
id,
&open_argv_on(&target.display, target.profile_path.as_deref(), path),
None,
&target,
)
.await?;
}
lazyboy_contracts::ComputerAction::Launch { application, uri } => {
let argv = launch_argv_on(
&target.display,
target.profile_path.as_deref(),
application,
uri.as_deref(),
)
.ok_or_else(|| "unknown application".to_string())?;
let _ = self.exec_argv(id, &argv, None, &target).await?;
}
other => {
let argv = xdotool_argv_on(&target.display, other)
.ok_or_else(|| "unsupported action".to_string())?;
let result = self.exec_argv(id, &argv, None, &target).await?;
if result.code != 0 {
return Err(result.stderr);
}
}
}
let pause = action_pause_ms(action);
if pause > 0 {
sleep(Duration::from_millis(pause)).await;
}
completed += 1;
}
if request.settle_ms > 0 {
sleep(Duration::from_millis(request.settle_ms as u64)).await;
}
let mut body = serde_json::json!({ "completed": completed });
if request.observe {
let payload = self.observe_payload(id, &target).await?;
if let serde_json::Value::Object(map) = payload.json {
if let Some(object) = body.as_object_mut() {
object.extend(map);
}
}
}
Ok(body)
// A transport failure may follow a successful click. Never replay mutations.
self.control_act(id, &request, &target).await
}
pub async fn screen_url(&self, id: &str, interactive: bool) -> Result<String, String> {
@ -536,6 +562,7 @@ PY"#,
],
None,
&ScreenTarget::default(),
None,
)
.await?;
if code != 0 {
@ -572,6 +599,18 @@ PY"#,
.map_err(|error| error.to_string())
}
pub async fn pause(&self, id: &str) -> Result<(), String> {
match self.docker.pause_container(id).await {
Ok(()) => Ok(()),
Err(error) if docker_already(&error.to_string(), "paused") => Ok(()),
Err(error) => Err(error.to_string()),
}
}
pub async fn unpause(&self, id: &str) -> Result<(), String> {
self.wake(id).await
}
pub async fn destroy(&self, id: &str) -> Result<(), String> {
let info = self.docker.inspect_container(id, None).await.ok();
let home_key = info
@ -610,17 +649,31 @@ PY"#,
.inspect_container(id, None)
.await
.map_err(|error| error.to_string())?;
if info
.config
.as_ref()
.and_then(|c| c.labels.as_ref())
.and_then(|l| l.get("lazyboy.controlVersion"))
.map(String::as_str)
!= Some("3")
{
return Ok(false);
}
let wanted = self.current_image_id().await?;
let have = info.image.unwrap_or_default();
if !image_ids_match(&wanted, &have) {
return Ok(false);
}
let running = info.state.as_ref().and_then(|state| state.running) == Some(true);
let paused = info.state.as_ref().and_then(|state| state.paused) == Some(true);
let exit = info
.state
.as_ref()
.and_then(|state| state.exit_code)
.unwrap_or(0);
if paused {
return Ok(true);
}
if !running && exit != 0 {
return Ok(false);
}
@ -659,6 +712,33 @@ PY"#,
}
}
async fn wake(&self, id: &str) -> Result<(), String> {
let info = self
.docker
.inspect_container(id, None)
.await
.map_err(|error| error.to_string())?;
let running = info.state.as_ref().and_then(|state| state.running) == Some(true);
let paused = info.state.as_ref().and_then(|state| state.paused) == Some(true);
if paused {
match self.docker.unpause_container(id).await {
Ok(()) => {}
Err(error) if docker_already(&error.to_string(), "not paused") => {}
Err(error) => return Err(error.to_string()),
}
return self.wait_ready_fast(id).await;
}
if running {
return self.wait_ready_fast(id).await;
}
self.docker
.start_container(id, None::<StartContainerOptions<String>>)
.await
.map_err(|error| error.to_string())?;
self.wait_running(id).await?;
self.wait_ready(id).await
}
async fn wait_running(&self, id: &str) -> Result<(), String> {
for _ in 0..40 {
let info = self
@ -675,12 +755,23 @@ PY"#,
}
async fn wait_ready(&self, id: &str) -> Result<(), String> {
for _ in 0..160 {
self.wait_ready_attempts(id, 160).await
}
async fn wait_ready_fast(&self, id: &str) -> Result<(), String> {
self.wait_ready_attempts(id, 20).await
}
async fn wait_ready_attempts(&self, id: &str, attempts: u32) -> Result<(), String> {
for _ in 0..attempts {
let info = self
.docker
.inspect_container(id, None)
.await
.map_err(|error| error.to_string())?;
if info.state.as_ref().and_then(|state| state.paused) == Some(true) {
return Err("computer is still paused".into());
}
if info.state.as_ref().and_then(|state| state.running) != Some(true) {
let exit = info.state.and_then(|state| state.exit_code).unwrap_or(1);
return Err(format!("computer exited during startup with code {exit}"));
@ -708,7 +799,20 @@ PY"#,
cwd: Option<&str>,
target: &ScreenTarget,
) -> Result<CommandResult, String> {
let (stdout, stderr, code) = self.exec_raw(id, argv, cwd, target).await?;
self.exec_raw_cmd(id, argv, cwd, target, None).await
}
async fn exec_raw_cmd(
&self,
id: &str,
argv: &[String],
cwd: Option<&str>,
target: &ScreenTarget,
stdin: Option<String>,
) -> Result<CommandResult, String> {
let (stdout, stderr, code) = self
.exec_raw(id, argv, cwd, target, stdin.as_deref())
.await?;
Ok(CommandResult {
stdout: String::from_utf8_lossy(&stdout).into_owned(),
stderr: String::from_utf8_lossy(&stderr).into_owned(),
@ -722,6 +826,7 @@ PY"#,
argv: &[String],
cwd: Option<&str>,
target: &ScreenTarget,
stdin: Option<&str>,
) -> Result<(Vec<u8>, Vec<u8>, i32), String> {
let display = normalize_display(&target.display);
let mut env = vec![
@ -737,6 +842,7 @@ PY"#,
.create_exec(
id,
CreateExecOptions {
attach_stdin: Some(stdin.is_some()),
attach_stdout: Some(true),
attach_stderr: Some(true),
cmd: Some(argv.to_vec()),
@ -750,7 +856,10 @@ PY"#,
.map_err(|error| error.to_string())?;
let mut stdout = Vec::new();
let mut stderr = Vec::new();
if let StartExecResults::Attached { mut output, .. } = self
if let StartExecResults::Attached {
mut output,
mut input,
} = self
.docker
.start_exec(
&exec.id,
@ -761,14 +870,26 @@ PY"#,
.await
.map_err(|error| error.to_string())?
{
if let Some(body) = stdin {
use tokio::io::AsyncWriteExt;
input
.write_all(body.as_bytes())
.await
.map_err(|error| error.to_string())?;
input.shutdown().await.map_err(|error| error.to_string())?;
}
while let Some(chunk) = output.next().await {
match chunk.map_err(|error| error.to_string())? {
bollard::container::LogOutput::StdOut { message } => {
stdout.extend_from_slice(&message)
}
bollard::container::LogOutput::StdErr { message } => {
stderr.extend_from_slice(&message)
}
bollard::container::LogOutput::StdOut { message } => stdout.extend_from_slice(
&message[..message
.len()
.min((16 * 1024 * 1024usize).saturating_sub(stdout.len()))],
),
bollard::container::LogOutput::StdErr { message } => stderr.extend_from_slice(
&message[..message
.len()
.min((1024 * 1024usize).saturating_sub(stderr.len()))],
),
_ => {}
}
}
@ -781,17 +902,45 @@ PY"#,
Ok((stdout, stderr, inspect.exit_code.unwrap_or(1) as i32))
}
pub async fn container_control_token(&self, id: &str) -> Result<String, String> {
let info = self
.docker
.inspect_container(id, None)
.await
.map_err(|e| e.to_string())?;
let labels = info
.config
.and_then(|c| c.labels)
.ok_or("unmanaged container")?;
let home = labels.get("lazyboy.homeKey").ok_or("unmanaged container")?;
Ok(scoped_control_token(&self.control_token, home))
}
async fn control_observe_json(
&self,
id: &str,
target: &ScreenTarget,
) -> Result<serde_json::Value, String> {
let script = format!(
"curl -fsS -H 'Authorization: Bearer {}' -H 'x-lazyboy-display: {}' http://127.0.0.1:7070/observe",
self.control_token, target.display
);
let token = self.container_control_token(id).await?;
let result = self
.exec_argv(id, &["bash".into(), "-lc".into(), script], None, target)
.exec_argv(
id,
&[
"curl".into(),
"-fsS".into(),
"--max-time".into(),
"20".into(),
"-X".into(),
"POST".into(),
"-H".into(),
format!("Authorization: Bearer {token}"),
"-H".into(),
format!("x-lazyboy-display: {}", target.display),
"http://127.0.0.1:7070/observe".into(),
],
None,
target,
)
.await?;
if result.code != 0 {
return Err(result.stderr);
@ -806,19 +955,29 @@ PY"#,
target: &ScreenTarget,
) -> Result<serde_json::Value, String> {
let payload = serde_json::to_string(request).map_err(|error| error.to_string())?;
let profile_header = target
.profile_path
.as_deref()
.map(|profile| format!(" -H 'x-lazyboy-profile: {profile}'"))
.unwrap_or_default();
let script = format!(
"curl -fsS -H 'Authorization: Bearer {}' -H 'x-lazyboy-display: {}'{profile_header} -H 'content-type: application/json' -d {} http://127.0.0.1:7070/act",
self.control_token,
target.display,
shell_single_quote(&payload)
);
let token = self.container_control_token(id).await?;
let mut argv = vec![
"curl".into(),
"-fsS".into(),
"--max-time".into(),
"120".into(),
"-H".into(),
format!("Authorization: Bearer {token}"),
"-H".into(),
format!("x-lazyboy-display: {}", target.display),
"-H".into(),
"content-type: application/json".into(),
];
if let Some(profile) = &target.profile_path {
argv.extend(["-H".into(), format!("x-lazyboy-profile: {profile}")]);
}
argv.extend([
"--data-binary".into(),
"@-".into(),
"http://127.0.0.1:7070/act".into(),
]);
let result = self
.exec_argv(id, &["bash".into(), "-lc".into(), script], None, target)
.exec_raw_cmd(id, &argv, None, target, Some(payload))
.await?;
if result.code != 0 {
return Err(result.stderr);
@ -871,6 +1030,31 @@ fn computer_pids_limit() -> i64 {
.unwrap_or(2048)
}
fn computer_sudo_enabled() -> bool {
matches!(std::env::var("LAZYBOY_COMPUTER_SUDO").as_deref(), Ok("1" | "true" | "yes"))
}
/// LXCFS supplies cgroup-aware /proc views so tools such as htop and free
/// report the Agent container's quota instead of the Docker host. It is
/// optional because Docker Desktop (macOS/Windows) does not ship LXCFS.
fn lxcfs_binds() -> impl Iterator<Item = String> {
let root = std::env::var("LAZYBOY_LXCFS_ROOT").ok();
["cpuinfo", "loadavg", "meminfo", "stat", "swaps", "uptime"]
.into_iter()
.filter_map(move |name| {
let root = root.as_deref()?;
let source = PathBuf::from(root).join("proc").join(name);
source
.is_file()
.then(|| format!("{}:/proc/{name}:ro", source.display()))
})
}
fn docker_already(error: &str, needle: &str) -> bool {
let error = error.to_ascii_lowercase();
error.contains("409") || error.contains(needle)
}
fn container_name(home_key: &str) -> String {
let sanitized: String = home_key
.chars()
@ -890,3 +1074,26 @@ fn network_name(home_key: &str) -> String {
fn shell_single_quote(value: &str) -> String {
format!("'{}'", value.replace('\'', r#"'"'"'"#))
}
fn scoped_control_token(master: &str, home: &str) -> String {
use hmac::{Hmac, Mac};
let mut mac = Hmac::<sha2::Sha256>::new_from_slice(master.as_bytes())
.expect("HMAC accepts any key length");
mac.update(b"lazyboy-computer-control-v2:");
mac.update(home.as_bytes());
hex::encode(mac.finalize().into_bytes())
}
#[cfg(test)]
mod credential_tests {
use super::*;
#[test]
fn computer_credentials_do_not_reveal_or_share_the_master() {
let master = "test-master-key-at-least-32-characters";
let a = scoped_control_token(master, "a");
let b = scoped_control_token(master, "b");
assert_ne!(a, master);
assert_ne!(a, b);
assert_eq!(a, scoped_control_token(master, "a"));
}
}

View File

@ -32,6 +32,15 @@ struct ProvisionBody {
#[tokio::main]
async fn main() {
if std::env::args().any(|arg| arg == "--healthcheck") {
let ok = reqwest::Client::new()
.get("http://127.0.0.1:7091/health")
.timeout(std::time::Duration::from_secs(3))
.send()
.await
.is_ok_and(|r| r.status().is_success());
std::process::exit(if ok { 0 } else { 1 });
}
tracing_subscriber::fmt()
.with_env_filter(EnvFilter::from_default_env().add_directive("info".parse().unwrap()))
.init();
@ -43,6 +52,15 @@ async fn main() {
);
let image =
std::env::var("LAZYBOY_COMPUTER_IMAGE").unwrap_or_else(|_| "lazyboy/computer:local".into());
let data_dir = std::env::var("DATA_DIR").unwrap_or_else(|_| "./data".into());
tokio::fs::create_dir_all(&data_dir)
.await
.expect("create data directory");
#[cfg(unix)]
if std::env::var("HOST_DATA_DIR").is_ok() {
std::os::unix::fs::chown(&data_dir, Some(1000), Some(1000))
.expect("set data directory owner");
}
let docker = DockerHost::connect(image, token.clone())
.await
.expect("docker");
@ -64,7 +82,13 @@ async fn main() {
.route("/computers/{id}/files", get(list_files).post(write_file))
.route("/computers/{id}/read", post(read_file))
.route("/computers/{id}/stop", post(stop))
.route("/computers/{id}/pause", post(pause))
.route("/computers/{id}/unpause", post(unpause))
.route("/computers/{id}", delete(destroy))
.route_layer(axum::middleware::from_fn_with_state(
app.clone(),
managed_boundary,
))
.with_state(app);
let bind = std::env::var("SUPERVISOR_BIND").unwrap_or_else(|_| "127.0.0.1:7091".into());
let listener = tokio::net::TcpListener::bind(&bind).await.expect("bind");
@ -301,6 +325,32 @@ async fn stop(
Ok(StatusCode::NO_CONTENT)
}
async fn pause(
State(app): State<App>,
headers: HeaderMap,
Path(id): Path<String>,
) -> Result<StatusCode, StatusCode> {
require_token(&headers, &app.token)?;
app.docker.pause(&id).await.map_err(|error| {
tracing::error!("pause: {error}");
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(StatusCode::NO_CONTENT)
}
async fn unpause(
State(app): State<App>,
headers: HeaderMap,
Path(id): Path<String>,
) -> Result<StatusCode, StatusCode> {
require_token(&headers, &app.token)?;
app.docker.unpause(&id).await.map_err(|error| {
tracing::error!("unpause: {error}");
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(StatusCode::NO_CONTENT)
}
async fn destroy(
State(app): State<App>,
headers: HeaderMap,
@ -321,3 +371,27 @@ struct _Home(&'static str);
fn _assert_home() {
let _ = HOME;
}
async fn managed_boundary(
State(app): State<App>,
req: axum::extract::Request,
next: axum::middleware::Next,
) -> axum::response::Response {
use axum::response::IntoResponse;
if req.uri().path() != "/health" {
if require_token(req.headers(), &app.token).is_err() {
return StatusCode::UNAUTHORIZED.into_response();
}
if let Some(id) = req
.uri()
.path()
.strip_prefix("/computers/")
.and_then(|p| p.split('/').next())
{
if app.docker.container_control_token(id).await.is_err() {
return StatusCode::NOT_FOUND.into_response();
}
}
}
next.run(req).await
}

5
docker-compose.dev.yml Normal file
View File

@ -0,0 +1,5 @@
# Opt-in host access for local Rust development only.
services:
postgres:
ports:
- "127.0.0.1:5434:5432"

View File

@ -1,12 +1,15 @@
services:
postgres:
restart: unless-stopped
networks: [database]
logging: &bounded-logs
driver: json-file
options: {max-size: "10m", max-file: "3"}
image: pgvector/pgvector:pg16
environment:
POSTGRES_USER: lazyboy
POSTGRES_PASSWORD: lazyboy
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-lazyboy}
POSTGRES_DB: lazyboy
ports:
- "127.0.0.1:5434:5432"
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
@ -25,18 +28,37 @@ services:
network_mode: none
supervisor:
restart: unless-stopped
init: true
networks: [control]
logging: *bounded-logs
security_opt: ["no-new-privileges:true"]
cap_drop: [ALL]
cap_add: [CHOWN, DAC_OVERRIDE]
read_only: true
healthcheck:
test: ["CMD", "/usr/local/bin/lazyboy-supervisor", "--healthcheck"]
interval: 5s
timeout: 4s
retries: 12
build:
context: .
dockerfile: image/supervisor/Dockerfile
environment:
SANDBOX_SUPERVISOR_TOKEN: ${SANDBOX_SUPERVISOR_TOKEN:?Set SANDBOX_SUPERVISOR_TOKEN in .env}
LAZYBOY_COMPUTER_IMAGE: lazyboy/computer:local
LAZYBOY_COMPUTER_CPUS: ${LAZYBOY_COMPUTER_CPUS:-2}
LAZYBOY_COMPUTER_MEMORY_MB: ${LAZYBOY_COMPUTER_MEMORY_MB:-2048}
LAZYBOY_COMPUTER_PIDS: ${LAZYBOY_COMPUTER_PIDS:-2048}
LAZYBOY_COMPUTER_SUDO: ${LAZYBOY_COMPUTER_SUDO:-false}
LAZYBOY_LXCFS_ROOT: /var/lib/lxcfs
SUPERVISOR_BIND: 0.0.0.0:7091
DATA_DIR: /data
HOST_DATA_DIR: ${PWD}/data
HOST_DATA_DIR: ${LAZYBOY_HOST_DATA_DIR:-${PWD}/data}
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ./data:/data
- ${LAZYBOY_LXCFS_ROOT:-./data/lxcfs}:/var/lib/lxcfs:ro
extra_hosts:
- "host.docker.internal:host-gateway"
depends_on:
@ -44,18 +66,26 @@ services:
condition: service_completed_successfully
api:
restart: unless-stopped
init: true
networks: [database, control, egress]
logging: *bounded-logs
security_opt: ["no-new-privileges:true"]
cap_drop: [ALL]
pids_limit: 256
build:
context: .
dockerfile: image/api/Dockerfile
environment:
DATABASE_URL: postgres://lazyboy:lazyboy@postgres:5432/lazyboy
DATABASE_URL: postgres://lazyboy:${POSTGRES_PASSWORD:-lazyboy}@postgres:5432/lazyboy
SANDBOX_SUPERVISOR_URL: http://supervisor:7091
SANDBOX_SUPERVISOR_TOKEN: ${SANDBOX_SUPERVISOR_TOKEN:?Set SANDBOX_SUPERVISOR_TOKEN in .env}
LAZYBOY_APP_TOKEN: ${LAZYBOY_APP_TOKEN:-}
LAZYBOY_APP_TOKEN: ${LAZYBOY_APP_TOKEN:?Set LAZYBOY_APP_TOKEN in .env}
LAZYBOY_VAULT_KEY: ${LAZYBOY_VAULT_KEY:-}
LAZYBOY_SECURE_COOKIE: ${LAZYBOY_SECURE_COOKIE:-false}
SANDBOX_PROVIDER: docker
DATA_DIR: /data
HOST_DATA_DIR: ${PWD}/data
HOST_DATA_DIR: ${LAZYBOY_HOST_DATA_DIR:-${PWD}/data}
API_BIND: 0.0.0.0:3100
XAI_API_KEY: ${XAI_API_KEY:-}
OPENCODE_GO_API_KEY: ${OPENCODE_GO_API_KEY:-}
@ -64,10 +94,17 @@ services:
LAZYBOY_MEMORY_MODEL_CACHE: /data/fastembed
LAZYBOY_MEMORY_TOP_K: ${LAZYBOY_MEMORY_TOP_K:-8}
LAZYBOY_MEMORY_BYTE_BUDGET: ${LAZYBOY_MEMORY_BYTE_BUDGET:-6000}
LAZYBOY_EVENT_RETENTION_DAYS: ${LAZYBOY_EVENT_RETENTION_DAYS:-30}
LAZYBOY_CHECKPOINT_RETENTION_DAYS: ${LAZYBOY_CHECKPOINT_RETENTION_DAYS:-7}
LAZYBOY_RUN_RETENTION_DAYS: ${LAZYBOY_RUN_RETENTION_DAYS:-90}
LAZYBOY_RECORDING_RETENTION_DAYS: ${LAZYBOY_RECORDING_RETENTION_DAYS:-30}
LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS: ${LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS:-90}
LAZYBOY_DB_WARN_MB: ${LAZYBOY_DB_WARN_MB:-1024}
LAZYBOY_WEB_DIR: /web
LAZYBOY_SCREEN_UPSTREAM: host.docker.internal
ORT_DYLIB_PATH: /usr/local/lib/libonnxruntime.so
ports:
- "3101:3100"
- "${LAZYBOY_BIND_IP:-127.0.0.1}:3101:3100"
volumes:
- ./data:/data
extra_hosts:
@ -76,7 +113,14 @@ services:
postgres:
condition: service_healthy
supervisor:
condition: service_started
condition: service_healthy
volumes:
pgdata:
networks:
database:
internal: true
control:
internal: true
egress: {}

93
docs/architecture.md Normal file
View File

@ -0,0 +1,93 @@
# 架構與流程
[← 回到 README](../README.zh-TW.md)
## 一個任務如何完成
```mermaid
flowchart LR
U["你<br/>文字・附件・語音"] --> W["React 工作空間<br/>對話・電腦・記憶"]
W -->|HTTP / WebSocket| A["Rust API + Agent<br/>模型・工具・技能"]
A -->|受控工具呼叫| S["Supervisor<br/>資源與生命週期"]
S -->|啟動 / 暫停 / 恢復| C["隔離 Linux 電腦<br/>Chromium・XFCE・檔案"]
C -->|畫面與事件| W
Q["排程"] -. Cron .-> A
A <--> M[("PostgreSQL<br/>Session・記憶・事件")]
V["加密憑證庫"] -. 僅相符 HTTPS 網域 .-> C
classDef ui fill:#151517,stroke:#3ec5a8,color:#f1f1f2
classDef core fill:#101012,stroke:#85858a,color:#f1f1f2
classDef secure fill:#221c0e,stroke:#fcd68a,color:#f1f1f2
class U,W ui
class A,M,Q core
class S,C,V secure
```
<div align="center">
**[開啟可縮放、可搜尋、支援深色模式的互動流程圖 →](./workflow.html)**
</div>
主流程之外還有兩個重要迴圈:
1. **接管迴圈**:使用者接管時,進行中的 run 進入等待;釋放後從目前畫面重新排隊執行。
2. **技能迴圈**:示範期間記錄控制項與頁面情境,模型整理成 playbook往後仍在當下畫面重新尋找元素不重播舊座標。
## 系統架構
LazyBoy 的公開入口只有 API。Supervisor 位於 Compose 內部 control network不直接對主機開埠Agent 桌面也不掛載主機 Docker socket。
```text
Browser
│ HTTP / WebSocket / authenticated screen proxy
lazyboy-api (:3101)
├── React 靜態前端
├── Session / Run / Memory / Schedule / Vault
├── Model provider / MCP client
└── PostgreSQL + pgvector
│ authenticated internal control API
lazyboy-supervisor (:7091, internal only)
├── provision / pause / resume / stop
├── CPU / memory / PID limits
└── isolated computer containers
├── Chromium + CDP
├── XFCE + AT-SPI
├── Xvfb + x11vnc + websockify
└── per-computer persisted home
```
### 主要元件
| 元件 | 職責 |
| --- | --- |
| `apps/web` | Vite + React 19 三欄工作空間、noVNC、語音與雙語 UI |
| `crates/api` | 對外 Axum API、Agent run、Session、排程、記憶、MCP、保險箱 |
| `crates/harness` | 模型供應商、憑證解析與語音契約 |
| `crates/supervisor` | Docker 電腦生命週期、隔離與資源上限 |
| `crates/control` | CDP、AT-SPI、X11 與畫面觀察操作 |
| `crates/controld` | 電腦容器內部的 localhost 控制服務 |
| `crates/contracts` | 跨 crate 的 Bot、Run、Computer、Voice 資料契約 |
| `PostgreSQL` | 對話、run、記憶、排程、憑證與保留政策 |
### 電腦生命週期
下圖保留原本的操作流程視角;`WaitingTakeover` 表示控制權交接期間的等待情境,並非獨立的電腦資料庫狀態。
```mermaid
stateDiagram-v2
[*] --> Stopped
Stopped --> Booting: 第一個需要 GUI 的工具
Booting --> Running: ready
Running --> WaitingTakeover: 使用者接管 / 2FA
WaitingTakeover --> Running: 釋放控制權
Running --> Suspended: 閒置 10 分鐘
Suspended --> Running: 新任務或畫面心跳
Suspended --> Stopped: 休眠約 6 小時
Running --> Stopped: 手動停止
```

80
docs/development.md Normal file
View File

@ -0,0 +1,80 @@
# 開發指南
[← 回到 README](../README.zh-TW.md)
### 本機開發
```bash
make dev # 準備 .env、PostgreSQL、電腦映像
make dev-supervisor # 終端 1Supervisor :7091
make dev-api # 終端 2API :3101
```
前端熱更新:
```bash
cd apps/web
npm install
npm run dev # http://127.0.0.1:5173
```
### 檢查與測試
```bash
make fmt
make clippy
make test
cd apps/web
npm run typecheck
npm run build
cd ../..
node --test tests/frontend.test.mjs
python3 tests/control.test.py
python3 tests/log-rotation.test.py
# 需要已啟動的 PostgreSQL Compose service
docker compose up -d postgres
python3 tests/retention.test.py
```
### 專案結構
```text
LazyBoy/
├── apps/web/ React 前端與 noVNC 頁面
├── crates/
│ ├── api/ 對外 API 與 Agent 執行迴圈
│ ├── contracts/ 跨元件資料契約
│ ├── control/ 瀏覽器與桌面控制
│ ├── controld/ 容器內控制服務
│ ├── harness/ 模型與語音後端
│ ├── sandbox/ API 到 Supervisor 的抽象
│ └── supervisor/ Docker 生命週期管理
├── image/ API、Supervisor、電腦映像
├── migrations/ SQLx PostgreSQL migrations
├── tests/ Rust 以外的契約與回歸測試
├── scripts/ 環境初始化與執行工具
├── docs/hero.html README 首頁視覺原稿
├── docs/workflow.html 可互動產品流程圖
├── docker-compose.yml 正式堆疊
└── Makefile 常用開發與部署指令
```
## 公開發布準備
對外發布前,可依下列清單補齊授權、貢獻流程與自動化檢查。只有在對應結果已公開時才加入品質徽章。
若要取得可公開查驗的安全與品質徽章,建議依序完成:
- [ ] 在公開 GitHub repository 建立正式 mirrorOpenSSF 的公開查驗以 GitHub 為主要整合目標。
- [x] Apache 2.0 `LICENSE`
- [ ] 補 `SECURITY.md`、`CONTRIBUTING.md` 與行為準則。
- [ ] 建立 CIRust format / Clippy / tests、前端 typecheck / build / tests。
- [ ] 啟用 Dependabot 或 Renovate、CodeQL、secret scanning 與 branch protection。
- [ ] 執行並發布 [OpenSSF Scorecard](https://scorecard.dev/) 結果後,再加入 Scorecard 徽章。
- [ ] 在 [OpenSSF Best Practices](https://www.bestpractices.dev/) 登記專案、誠實完成 Passing 問卷後,再加入認證徽章。
- [ ] 若提供容器映像,再加入 SBOM、簽章與可重現版本發布流程。
> 不建議現在顯示 CI passing、coverage、OpenSSF 或 Best Practices 徽章:目前 repository 沒有對應的公開結果,徽章會失真或直接顯示 unknown。

442
docs/hero.html Normal file
View File

@ -0,0 +1,442 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<title>LazyBoy hero</title>
<style>
@font-face {
font-family: Huninn;
font-weight: 400;
src: url("../apps/web/node_modules/@fontsource/huninn/files/huninn-latin-400-normal.woff2") format("woff2");
unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F;
}
@font-face {
font-family: Huninn;
font-weight: 400;
src: url("../apps/web/node_modules/@fontsource/huninn/files/huninn-latin-ext-400-normal.woff2") format("woff2");
unicode-range: U+0100-02AF, U+0304, U+0308, U+0329, U+1E00-1EFF;
}
@font-face {
font-family: Huninn;
font-weight: 400;
src: url("../apps/web/node_modules/@fontsource/huninn/files/huninn-chinese-traditional-400-normal.woff2") format("woff2");
unicode-range: U+4E00-9FFF, U+3400-4DBF, U+F900-FAFF, U+3000-303F, U+FF00-FFEF;
}
* { box-sizing: border-box; }
html, body {
margin: 0;
width: 1280px;
height: 640px;
overflow: hidden;
background: #f3f3f5;
color: #161618;
font-family: Huninn, "jf open 粉圓", "PingFang TC", sans-serif;
}
.page {
width: 1280px;
height: 640px;
padding: 28px 48px 0;
position: relative;
}
.top {
display: flex;
align-items: center;
justify-content: space-between;
height: 36px;
}
.brand {
display: flex;
align-items: center;
gap: 10px;
font-size: 20px;
letter-spacing: .02em;
}
.pill {
display: inline-flex;
align-items: center;
gap: 8px;
height: 28px;
padding: 0 12px;
border: 1px solid #e4e4e8;
border-radius: 999px;
background: #fff;
color: #5c5c64;
font-size: 12px;
}
.pill b {
color: #0f766e;
font-weight: 400;
}
.copy {
text-align: center;
margin: 22px auto 18px;
max-width: 760px;
}
h1 {
margin: 0;
font-size: 42px;
font-weight: 400;
letter-spacing: .01em;
line-height: 1.15;
}
.lead {
margin: 10px 0 0;
color: #6b6b73;
font-size: 15px;
line-height: 1.55;
}
.window {
width: 1184px;
height: 470px;
margin: 0 auto;
border-radius: 16px 16px 0 0;
overflow: hidden;
background: #0d0d0e;
box-shadow: 0 24px 70px rgba(16, 16, 20, .28), 0 0 0 1px rgba(0,0,0,.08);
display: grid;
grid-template-rows: 36px 1fr;
}
.chrome {
display: flex;
align-items: center;
gap: 7px;
padding: 0 14px;
background: #161618;
border-bottom: 1px solid #202023;
}
.dot { width: 10px; height: 10px; border-radius: 50%; }
.dot.r { background: #ff5f57; }
.dot.y { background: #febc2e; }
.dot.g { background: #28c840; }
.app {
display: grid;
grid-template-columns: 228px 1fr 300px;
min-height: 0;
color: #dfdfe2;
background: #050506;
}
.side, .chat, .desk {
min-width: 0;
min-height: 0;
}
.side {
background: #0b0b0c;
border-right: 1px solid #171719;
padding: 12px 10px;
display: flex;
flex-direction: column;
gap: 8px;
}
.ws {
display: flex;
align-items: center;
gap: 8px;
padding: 4px 6px 8px;
font-size: 14px;
}
.ws i {
width: 26px; height: 26px;
display: grid; place-items: center;
border-radius: 50%;
background: #151517;
color: #85858a;
font-size: 9px;
font-style: normal;
letter-spacing: .04em;
}
.search {
height: 34px;
border: 1px solid #202023;
border-radius: 12px;
background: #121214;
color: #626267;
display: flex;
align-items: center;
padding: 0 10px;
font-size: 12px;
gap: 8px;
}
.search svg { width: 13px; height: 13px; stroke: currentColor; fill: none; stroke-width: 1.7; }
.label {
padding: 8px 8px 2px;
color: #626267;
font-size: 10px;
letter-spacing: .06em;
}
.bot {
display: flex;
align-items: center;
gap: 8px;
padding: 8px;
border-radius: 12px;
}
.bot.on { background: #171719; }
.bot strong { display: block; font-size: 13px; font-weight: 400; }
.bot small { display: block; color: #85858a; font-size: 11px; margin-top: 1px; }
.bot .copy { min-width: 0; flex: 1; }
.bot .copy strong, .bot .copy small {
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
}
.time { color: #626267; font-size: 10px; align-self: start; }
.chat {
background: #0d0d0e;
display: grid;
grid-template-rows: 52px 1fr 118px;
}
.topbar {
display: flex;
align-items: center;
gap: 8px;
padding: 0 16px;
border-bottom: 1px solid #171719;
font-size: 15px;
}
.grow { flex: 1; }
.tools {
display: flex; gap: 2px; padding: 3px;
border: 1px solid #29292d; border-radius: 11px; background: rgba(18,18,20,.86);
}
.tools i {
width: 26px; height: 26px; border-radius: 8px;
display: grid; place-items: center;
font-style: normal;
}
.tools i.on { background: #151517; }
.tools svg { width: 13px; height: 13px; stroke: currentColor; fill: none; stroke-width: 1.7; }
.tools i.on svg { stroke: #f1f1f2; }
.tools i:not(.on) svg { stroke: #85858a; }
.msgs {
padding: 18px 28px 8px;
display: flex;
flex-direction: column;
gap: 12px;
}
.bubble {
max-width: 78%;
padding: 10px 14px;
border-radius: 18px;
line-height: 1.5;
font-size: 13px;
white-space: pre-wrap;
}
.me { align-self: flex-end; background: #f1f1ef; color: #1a1a1a; }
.them { align-self: flex-start; background: #19191c; }
.dock {
padding: 8px 22px 16px;
display: flex;
flex-direction: column;
gap: 8px;
}
.think {
display: flex; align-items: center; gap: 8px;
font-size: 12px;
}
.think .lbl {
background: linear-gradient(90deg, #7a8088 0%, #7a8088 28%, #fff 50%, #7a8088 72%, #7a8088 100%);
background-size: 220% 100%;
-webkit-background-clip: text;
background-clip: text;
color: transparent;
}
.think small { color: #85858a; }
.composer {
height: 52px;
display: flex; align-items: center; gap: 8px;
border: 1px solid #29292d;
background: #121214;
border-radius: 22px;
padding: 6px 8px;
}
.plus, .send {
width: 34px; height: 34px; border-radius: 50%;
display: grid; place-items: center;
}
.plus { color: #85858a; font-size: 20px; }
.ph { flex: 1; color: #626267; font-size: 13px; }
.send { background: #f1f1ef; color: #1b1b1c; font-size: 14px; }
.desk {
background: #0a0a0b;
border-left: 1px solid #171719;
padding: 10px 12px 12px;
display: flex;
flex-direction: column;
gap: 8px;
}
.desk-h {
display: flex; align-items: center; gap: 8px;
color: #f1f1f2; font-size: 13px;
}
.desk-h .dot-run {
width: 7px; height: 7px; border-radius: 50%; background: #4ecb71;
}
.desk-h small { color: #85858a; margin-left: auto; font-size: 11px; }
.preview {
position: relative;
flex: 1;
border: 1px solid #29292d;
border-radius: 12px;
background: #101012;
overflow: hidden;
min-height: 168px;
}
.desktop {
position: absolute; inset: 0;
background:
radial-gradient(circle at 20% 0%, #1a2430 0%, transparent 42%),
linear-gradient(180deg, #15202b, #0d1218);
}
.win {
position: absolute;
left: 14px; top: 12px; right: 14px; bottom: 28px;
border-radius: 8px;
background: #fff;
overflow: hidden;
box-shadow: 0 10px 28px rgba(0,0,0,.35);
display: grid;
grid-template-rows: 22px 28px 1fr;
}
.win-bar { background: #e8e8ea; display: flex; align-items: center; gap: 5px; padding: 0 8px; }
.win-bar i { width: 7px; height: 7px; border-radius: 50%; background: #c4c4c8; font-style: normal; }
.addr {
background: #f4f4f6;
display: flex; align-items: center;
padding: 0 10px;
color: #444; font-size: 10px;
border-bottom: 1px solid #ececf0;
}
.page-body { padding: 12px 14px; color: #222; }
.page-body h3 { margin: 0 0 6px; font-size: 13px; font-weight: 400; }
.page-body p { margin: 0; color: #666; font-size: 11px; line-height: 1.45; }
.progress {
margin-top: 10px; height: 6px; border-radius: 99px; background: #ececf0; overflow: hidden;
}
.progress b { display: block; width: 62%; height: 100%; background: #3ec5a8; }
.next {
margin-top: 10px; height: 24px; width: 64px;
border-radius: 6px; background: #1a1a1c; color: #fff;
display: grid; place-items: center; font-size: 11px;
}
.panel {
position: absolute; left: 0; right: 0; bottom: 0; height: 22px;
background: #1b242e; border-top: 1px solid #2a3540;
}
.cap {
display: flex; justify-content: space-between;
color: #85858a; font-size: 11px;
}
.btn {
height: 28px; padding: 0 10px; border-radius: 8px;
border: 1px solid #29292d; color: #dfdfe2; display: grid; place-items: center;
font-size: 12px;
}
.btn.cream { background: #f1f1ef; color: #181819; border-color: #f1f1ef; }
.actions { display: flex; justify-content: flex-end; gap: 6px; }
/* blobatar-ish faces */
.blob { width: 32px; height: 32px; flex: 0 0 32px; }
.blob.sm { width: 22px; height: 22px; flex-basis: 22px; }
.blob.md { width: 28px; height: 28px; flex-basis: 28px; }
.think-ring { position: relative; }
.think-ring:after {
content: "";
position: absolute; inset: -5px;
border-radius: 50%;
background: conic-gradient(from 40deg, transparent 0 8%, #ff4fd8 12%, #7c5cff 18%, transparent 26% 48%, #34d9ff 54%, transparent 62%);
-webkit-mask: radial-gradient(farthest-side, transparent calc(100% - 2px), #000 calc(100% - 1.5px));
mask: radial-gradient(farthest-side, transparent calc(100% - 2px), #000 calc(100% - 1.5px));
}
</style>
</head>
<body>
<div class="page">
<div class="top">
<div class="brand">
<svg class="blob md" viewBox="0 0 64 64" aria-hidden="true">
<path fill="#3ec5a8" d="M33 5c11.5 1 24 11 23 26-1 14-11 27-25 28S6 47 7 31 21 4 33 5z"/>
<ellipse cx="25.5" cy="29" rx="4.2" ry="5" fill="#1b1b22"/>
<ellipse cx="40.5" cy="29" rx="4.2" ry="5" fill="#1b1b22"/>
<circle cx="26.8" cy="27.2" r="1.3" fill="#f3f3f6"/>
<circle cx="41.8" cy="27.2" r="1.3" fill="#f3f3f6"/>
</svg>
LazyBoy
</div>
<div class="pill"><b>Self-hosted</b> a Linux desktop per agent</div>
</div>
<div class="copy">
<h1>Give an agent a real computer</h1>
<p class="lead">Open bots in the browser. They browse, run commands, and learn the flows you demonstrate.<br/>Chats and desktops stay on your host. You choose the model.</p>
</div>
<div class="window">
<div class="chrome"><i class="dot r"></i><i class="dot y"></i><i class="dot g"></i></div>
<div class="app">
<aside class="side">
<div class="ws"><i>LB</i> Local workspace</div>
<div class="search"><svg viewBox="0 0 24 24"><circle cx="11" cy="11" r="7"/><path d="M20 20l-3.5-3.5"/></svg>Search</div>
<div class="label">Pinned</div>
<div class="bot on">
<span class="think-ring">
<svg class="blob" viewBox="0 0 64 64"><path fill="#3ec5a8" d="M33 5c11.5 1 24 11 23 26-1 14-11 27-25 28S6 47 7 31 21 4 33 5z"/><ellipse cx="25.5" cy="29" rx="4.2" ry="5" fill="#1b1b22"/><ellipse cx="40.5" cy="29" rx="4.2" ry="5" fill="#1b1b22"/><circle cx="26.8" cy="27.2" r="1.3" fill="#f3f3f6"/><circle cx="41.8" cy="27.2" r="1.3" fill="#f3f3f6"/></svg>
</span>
<div class="copy"><strong>Engineering</strong><small>Page 12, waiting for Next</small></div>
<span class="time">14:08</span>
</div>
<div class="label">Agent</div>
<div class="bot">
<svg class="blob" viewBox="0 0 64 64"><path fill="#f5a03c" d="M30 7c13-1 26 10 25 24-1 15-12 26-25 27S5 46 7 31 17 8 30 7z"/><ellipse cx="25" cy="30" rx="4" ry="4.8" fill="#1b1b22"/><ellipse cx="40" cy="30" rx="4" ry="4.8" fill="#1b1b22"/><circle cx="26.3" cy="28.3" r="1.2" fill="#f3f3f6"/><circle cx="41.3" cy="28.3" r="1.2" fill="#f3f3f6"/></svg>
<div class="copy"><strong>Morning digest</strong><small>Runs tomorrow at 9:00</small></div>
</div>
<div class="bot">
<svg class="blob" viewBox="0 0 64 64"><path fill="#6a6bf5" d="M34 6c12 2 22 12 21 25s-12 26-25 27S6 46 8 32 22 4 34 6z"/><ellipse cx="26" cy="29" rx="4.1" ry="5" fill="#f3f3f6"/><ellipse cx="41" cy="29" rx="4.1" ry="5" fill="#f3f3f6"/><circle cx="24.8" cy="27.4" r="1.2" fill="#1b1b22"/><circle cx="39.8" cy="27.4" r="1.2" fill="#1b1b22"/></svg>
<div class="copy"><strong>Support</strong><small>Remembered the reply tone</small></div>
</div>
</aside>
<main class="chat">
<header class="topbar">
<svg class="blob sm" viewBox="0 0 64 64"><path fill="#3ec5a8" d="M33 5c11.5 1 24 11 23 26-1 14-11 27-25 28S6 47 7 31 21 4 33 5z"/><ellipse cx="25.5" cy="29" rx="4.2" ry="5" fill="#1b1b22"/><ellipse cx="40.5" cy="29" rx="4.2" ry="5" fill="#1b1b22"/><circle cx="26.8" cy="27.2" r="1.3" fill="#f3f3f6"/><circle cx="41.8" cy="27.2" r="1.3" fill="#f3f3f6"/></svg>
Engineering
<span style="color:#85858a;font-size:12px">STAR training </span>
<span class="grow"></span>
<div class="tools">
<i class="on"><svg viewBox="0 0 24 24"><rect x="3" y="5" width="18" height="12" rx="2"/><path d="M8 21h8M12 17v4"/></svg></i>
<i><svg viewBox="0 0 24 24"><path d="M12 3a7 7 0 0 0-4 12.7V18h8v-2.3A7 7 0 0 0 12 3z"/><path d="M9 21h6"/></svg></i>
<i><svg viewBox="0 0 24 24"><path d="M12 5v2M12 17v2M5 12h2M17 12h2M7 7l1.5 1.5M15.5 15.5L17 17M7 17l1.5-1.5M15.5 8.5L17 7"/><circle cx="12" cy="12" r="3"/></svg></i>
</div>
</header>
<div class="msgs">
<div class="bubble me">Finish the STAR course and submit the quiz.</div>
<div class="bubble them">Computer is up. Page 12 / 24 — Ill hit Next when the video ends. I wont store passwords.</div>
</div>
<div class="dock">
<div class="think">
<span class="think-ring"><svg class="blob sm" viewBox="0 0 64 64"><path fill="#3ec5a8" d="M33 5c11.5 1 24 11 23 26-1 14-11 27-25 28S6 47 7 31 21 4 33 5z"/><ellipse cx="25.5" cy="29" rx="4.2" ry="5" fill="#1b1b22"/><ellipse cx="40.5" cy="29" rx="4.2" ry="5" fill="#1b1b22"/><circle cx="26.8" cy="27.2" r="1.3" fill="#f3f3f6"/><circle cx="41.8" cy="27.2" r="1.3" fill="#f3f3f6"/></svg></span>
<span class="lbl">Working…</span>
<small>click Next</small>
</div>
<div class="composer"><div class="plus"></div><div class="ph">Message Engineering</div><div class="send"></div></div>
</div>
</main>
<aside class="desk">
<div class="desk-h">Engineerings computer <i class="dot-run"></i> <small>Running</small></div>
<div class="preview">
<div class="desktop">
<div class="win">
<div class="win-bar"><i></i><i></i><i></i></div>
<div class="addr">star.example / course / 12</div>
<div class="page-body">
<h3>Workplace Safety · page 12</h3>
<p>Read this, then press Next. It unlocks when the countdown ends.</p>
<div class="progress"><b></b></div>
<div class="next">Next</div>
</div>
</div>
<div class="panel"></div>
</div>
</div>
<div class="cap"><span>This screen</span><span>Enlarge</span></div>
<div class="actions"><div class="btn cream">Take over</div><div class="btn">Stop task</div></div>
</aside>
</div>
</div>
</div>
</body>
</html>

Some files were not shown because too many files have changed in this diff Show More