add readme
This commit is contained in:
parent
158f90770b
commit
11cf2407bc
|
|
@ -13,7 +13,7 @@ members = [
|
|||
[workspace.package]
|
||||
edition = "2024"
|
||||
version = "0.1.0"
|
||||
license = "MIT"
|
||||
license = "Apache-2.0"
|
||||
publish = false
|
||||
|
||||
[profile.release]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,202 @@
|
|||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
345
README.md
345
README.md
|
|
@ -1,334 +1,135 @@
|
|||
<div align="center">
|
||||
|
||||
<img src="./apps/web/public/lazyboy-icon.png" width="96" height="96" alt="LazyBoy" />
|
||||
|
||||
**English** · [繁體中文](./README.zh-TW.md)
|
||||
|
||||
# LazyBoy
|
||||
|
||||
### 給 Agent 一台真的電腦
|
||||
**Give AI a computer so it can do the work.**
|
||||
|
||||
在瀏覽器裡指派工作，讓 AI Agent 在隔離的 Linux 桌面中開網頁、操作應用程式、整理檔案，並在你需要時把控制權交回來。
|
||||
A self-hosted AI agent workspace. Assign tasks in text or voice, watch the desktop live, and take over whenever you need to.
|
||||
|
||||
[](#快速開始)
|
||||
[](./Cargo.toml)
|
||||
[](./apps/web/package.json)
|
||||
[](./docker-compose.yml)
|
||||
|
||||
[快速開始](#快速開始) · [功能](#核心能力) · [運作流程](#一個任務如何完成) · [系統架構](#系統架構) · [二次開發](#二次開發) · [安全](#安全模型)
|
||||
[Quick start](#quick-start) · [Features](#features) · [Architecture](./docs/architecture.md) · [Operations](./docs/operations.md) · [Development](./docs/development.md)
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
LazyBoy 是一套本機優先（local-first）的多 Agent 工作空間。每個 Agent 都能使用瀏覽器、終端、檔案系統與桌面應用程式；執行畫面可即時觀看、可以人工接管，也能把示範整理成技能，之後透過排程重複執行。
|
||||
LazyBoy gives each agent its own Linux desktop in Docker — browser, terminal, and files. You can run several agents, put them in a group, turn a demonstration into a skill, and schedule it to run again.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> LazyBoy 會把對話、記憶、憑證與瀏覽器設定檔持久化在自己的主機，但使用外部模型時，完成任務所需的提示詞、工具結果或畫面仍可能傳送給你設定的模型供應商。請依資料敏感度選擇供應商與部署方式。
|
||||
This is an early `0.1.0` release with desktop and phone browser UIs. You bring your own model API key.
|
||||
|
||||
## 為什麼是 LazyBoy？
|
||||
## Features
|
||||
|
||||
| 一般聊天機器人 | LazyBoy |
|
||||
| --- | --- |
|
||||
| 告訴你怎麼做 | 在隔離桌面中實際操作 |
|
||||
| 看不到執行過程 | 即時 noVNC 畫面與步驟狀態 |
|
||||
| 人與 Agent 搶控制權 | 接管／釋放租約，安全交接 |
|
||||
| 每次重新說明流程 | 示範一次，儲存為可攜技能 |
|
||||
| 一次性對話 | Session、長期記憶、排程與群組 |
|
||||
| 固定模型與工具 | xAI、OpenCode Go、OpenAI 相容端點與 MCP |
|
||||
- **A lasting workspace**: each agent has its own chats, run history, and optional long-term memory.
|
||||
- **A real computer**: open pages, use the terminal, organize files, drive the GUI — and watch it live.
|
||||
- **Take over any time**: sign in, pass a check, or nudge things by hand on the same desktop, then hand it back.
|
||||
- **Several agents and groups**: shared Team computers or private dedicated desktops.
|
||||
- **Teach by demo, then schedule**: turn a walkthrough into a skill; use cron for repeat work.
|
||||
- **Your models and tools**: xAI, OpenCode Go, OpenAI-compatible endpoints, MCP, and file skills.
|
||||
- **Voice calls**: after you enable a voice provider, you can talk to the agent on a call.
|
||||
- **Phone-friendly**: collapse the chat sidebar; the remote desktop has keyboard, trackpad, right-click, drag, and scroll.
|
||||
- **English and Traditional Chinese**: switch the UI language in the app.
|
||||
|
||||
## 快速開始
|
||||
## Quick start
|
||||
|
||||
### 需求
|
||||
You need Docker and Compose, Git, Make, Python 3, and an API key for a supported model. On macOS use Docker Desktop or OrbStack; on Linux use Docker Engine.
|
||||
|
||||
- macOS 或 Linux
|
||||
- Docker Engine / Docker Desktop（含 Compose）
|
||||
- Make、Git
|
||||
- 至少 4 核 CPU、8 GB RAM；建議 8 核、16 GB RAM
|
||||
- 一組支援的模型 API 金鑰
|
||||
|
||||
第一次啟動會建置 Debian、XFCE、Chromium 桌面映像，因此會比後續啟動久。
|
||||
From the repo root:
|
||||
|
||||
```bash
|
||||
git clone <your-repository-url>
|
||||
cd LazyBoy
|
||||
|
||||
make env
|
||||
# 編輯 .env，至少填入 XAI_API_KEY、OPENCODE_GO_API_KEY
|
||||
# 或 OPENAI_API_KEY 其中之一
|
||||
```
|
||||
|
||||
Edit the generated `.env` and set one of `XAI_API_KEY`, `OPENCODE_GO_API_KEY`, or `OPENAI_API_KEY`. The init tool creates the login and service secrets; running it again keeps existing values.
|
||||
|
||||
```bash
|
||||
make up
|
||||
make health
|
||||
```
|
||||
|
||||
開啟 [http://127.0.0.1:3101](http://127.0.0.1:3101)，使用 `.env` 中的 `LAZYBOY_APP_TOKEN` 登入，建立第一個 Agent，然後直接描述目標。
|
||||
Open **[http://127.0.0.1:3101](http://127.0.0.1:3101)**, sign in with `LAZYBOY_APP_TOKEN` from `.env`, create an agent, and pick a model.
|
||||
|
||||
```text
|
||||
打開指定的訓練網站，完成還沒看完的章節，最後整理進度。
|
||||
```
|
||||
The first run builds the API and Linux desktop images from source and takes a while. The build toolchain lives in the containers, so a full Docker deploy does not need Rust or Node.js on the host.
|
||||
|
||||
常用指令：
|
||||
Try a concrete task:
|
||||
|
||||
| 指令 | 用途 |
|
||||
| --- | --- |
|
||||
| `make up` | 建置並啟動完整堆疊 |
|
||||
| `make health` | 檢查 API 健康狀態 |
|
||||
| `make logs` | 持續查看服務日誌 |
|
||||
| `make ps` | 查看容器狀態 |
|
||||
| `make down` | 停止服務，保留 PostgreSQL 資料 |
|
||||
| `make purge` | 停止服務並刪除 PostgreSQL volume |
|
||||
> Open the site I name, summarize the page, and save the notes as a Markdown file in the workspace.
|
||||
|
||||
沒有 Make 時，可以從 `.env.example` 建立設定後執行：
|
||||
Status and logs:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# 為 LAZYBOY_APP_TOKEN、SANDBOX_SUPERVISOR_TOKEN、
|
||||
# LAZYBOY_VAULT_KEY 各自執行一次 openssl rand -hex 32
|
||||
docker compose up -d --build
|
||||
make ps
|
||||
make logs
|
||||
make down # stop services, keep PostgreSQL data
|
||||
```
|
||||
|
||||
## 核心能力
|
||||
For resource limits, environment variables, HTTPS, and in-container sudo, see [Operations](./docs/operations.md).
|
||||
|
||||
| 能力 | 實際行為 |
|
||||
| --- | --- |
|
||||
| 可觀察的電腦操作 | 右側 noVNC 顯示 Agent 的 1280×800 桌面；模型觀察與人類畫面分流 |
|
||||
| 瀏覽器與原生桌面控制 | 網頁透過 CDP 元素操作；原生 UI 使用 AT-SPI，必要時才退回座標 |
|
||||
| 人工接管 | 登入、2FA、驗證碼或敏感步驟可暫停 Agent，由使用者接管後續跑 |
|
||||
| 示範教學 | 記錄語意操作事件並整理成 playbook，不依賴固定像素重播 |
|
||||
| 長期記憶 | PostgreSQL + pgvector + MiniLM；只保存明確要求記住的內容 |
|
||||
| 安全登入 | 每個 Agent 的 AES-256-GCM 憑證庫；只對相符的 HTTPS 網域填入 |
|
||||
| 排程 | 五欄 cron、時區支援、立即試跑；到期任務進入一般 run 佇列 |
|
||||
| 多 Agent 與群組 | Team 共用電腦可分配不同 DISPLAY；私人模式使用獨立容器 |
|
||||
| MCP 與檔案技能 | 支援 stdio、HTTP、SSE MCP，以及 `data/skills/*/SKILL.md` |
|
||||
| 語音通話 | 前端提供可設定的即時語音工作階段 |
|
||||
| 中英介面 | `zh-TW` 與 English 文案共用完整 key 契約 |
|
||||
## On a phone
|
||||
|
||||
## 一個任務如何完成
|
||||
Desktop and phone share the same web UI. After you deploy on a host the phone can reach, open that URL in the phone browser. `127.0.0.1` is the phone itself — it will not reach another machine.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
U["你<br/>文字・附件・語音"] --> W["React 工作空間<br/>對話・電腦・記憶"]
|
||||
W -->|HTTP / WebSocket| A["Rust API + Agent<br/>模型・工具・技能"]
|
||||
A -->|受控工具呼叫| S["Supervisor<br/>資源與生命週期"]
|
||||
S -->|啟動 / 暫停 / 恢復| C["隔離 Linux 電腦<br/>Chromium・XFCE・檔案"]
|
||||
C -->|畫面與事件| W
|
||||
Tap outside the chat sidebar to collapse it. On the remote desktop you can switch between tap-to-click and trackpad, and use the toolbar for keyboard, right-click, or drag. Put the service behind HTTPS before you expose it; see [Operations](./docs/operations.md#安全模型).
|
||||
|
||||
Q["排程"] -. Cron .-> A
|
||||
A <--> M[("PostgreSQL<br/>Session・記憶・事件")]
|
||||
V["加密憑證庫"] -. 僅相符 HTTPS 網域 .-> C
|
||||
## Stack
|
||||
|
||||
classDef ui fill:#151517,stroke:#3ec5a8,color:#f1f1f2
|
||||
classDef core fill:#101012,stroke:#85858a,color:#f1f1f2
|
||||
classDef secure fill:#221c0e,stroke:#fcd68a,color:#f1f1f2
|
||||
class U,W ui
|
||||
class A,M,Q core
|
||||
class S,C,V secure
|
||||
```
|
||||
- **Frontend**: React 19, TypeScript, Vite, noVNC
|
||||
- **Backend**: Rust 2024, Axum, Tokio
|
||||
- **Data**: PostgreSQL, pgvector, SQLx
|
||||
- **Desktop**: Docker, Debian, XFCE, Chromium, Xvfb
|
||||
- **Computer control**: CDP, AT-SPI, X11
|
||||
- **Extensions**: MCP, file skills, demonstration playbooks
|
||||
|
||||
<div align="center">
|
||||
Flow diagrams, handoff, component roles, and the computer lifecycle live in **[Architecture](./docs/architecture.md)**. The older **[interactive diagram](./docs/workflow.html)** is still there — download it and open it in a browser.
|
||||
|
||||
**[開啟可縮放、可搜尋、支援深色模式的互動流程圖 →](./docs/workflow.html)**
|
||||
## Local development
|
||||
|
||||
</div>
|
||||
|
||||
主流程之外還有兩個重要迴圈：
|
||||
|
||||
1. **接管迴圈**：使用者接管時，進行中的 run 進入等待；釋放後從目前畫面重新排隊執行。
|
||||
2. **技能迴圈**：示範期間記錄控制項與頁面情境，模型整理成 playbook；往後仍在當下畫面重新尋找元素，不重播舊座標。
|
||||
|
||||
## 系統架構
|
||||
|
||||
LazyBoy 的公開入口只有 API。Supervisor 位於 Compose 內部 control network，不直接對主機開埠；Agent 桌面也不掛載主機 Docker socket。
|
||||
|
||||
```text
|
||||
Browser
|
||||
│ HTTP / WebSocket / authenticated screen proxy
|
||||
▼
|
||||
lazyboy-api (:3101)
|
||||
├── React 靜態前端
|
||||
├── Session / Run / Memory / Schedule / Vault
|
||||
├── Model provider / MCP client
|
||||
└── PostgreSQL + pgvector
|
||||
│
|
||||
│ authenticated internal control API
|
||||
▼
|
||||
lazyboy-supervisor (:7091, internal only)
|
||||
│
|
||||
├── provision / pause / resume / stop
|
||||
├── CPU / memory / PID limits
|
||||
└── isolated computer containers
|
||||
├── Chromium + CDP
|
||||
├── XFCE + AT-SPI
|
||||
├── Xvfb + x11vnc + websockify
|
||||
└── per-computer persisted home
|
||||
```
|
||||
|
||||
### 主要元件
|
||||
|
||||
| 元件 | 職責 |
|
||||
| --- | --- |
|
||||
| `apps/web` | Vite + React 19 三欄工作空間、noVNC、語音與雙語 UI |
|
||||
| `crates/api` | 對外 Axum API、Agent run、Session、排程、記憶、MCP、保險箱 |
|
||||
| `crates/harness` | 模型供應商、憑證解析與語音契約 |
|
||||
| `crates/supervisor` | Docker 電腦生命週期、隔離與資源上限 |
|
||||
| `crates/control` | CDP、AT-SPI、X11 與畫面觀察操作 |
|
||||
| `crates/controld` | 電腦容器內部的 localhost 控制服務 |
|
||||
| `crates/contracts` | 跨 crate 的 Bot、Run、Computer、Voice 資料契約 |
|
||||
| `PostgreSQL` | 對話、run、記憶、排程、憑證與保留政策 |
|
||||
|
||||
### 電腦生命週期
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> Stopped
|
||||
Stopped --> Booting: 第一個需要 GUI 的工具
|
||||
Booting --> Running: ready
|
||||
Running --> WaitingTakeover: 使用者接管 / 2FA
|
||||
WaitingTakeover --> Running: 釋放控制權
|
||||
Running --> Suspended: 閒置 10 分鐘
|
||||
Suspended --> Running: 新任務或畫面心跳
|
||||
Suspended --> Stopped: 休眠約 6 小時
|
||||
Running --> Stopped: 手動停止
|
||||
```
|
||||
|
||||
## 安全模型
|
||||
|
||||
目前程式碼包含下列防護：
|
||||
|
||||
- API session token 與 Supervisor token 分離。
|
||||
- Supervisor 只在內部 Compose network，並使用 `no-new-privileges`、唯讀 root filesystem 與 capability drop。
|
||||
- 每台電腦有 CPU、RAM、PID 上限；預設 2 CPU、2 GB、2048 PID。
|
||||
- API 預設只綁定 `127.0.0.1:3101`。
|
||||
- 憑證以獨立 `LAZYBOY_VAULT_KEY` 加密，輪替登入 token 時不應更換此 key。
|
||||
- 已保存登入只接受 HTTPS、精確或合法子網域匹配，不對相似惡意網域填入。
|
||||
- Markdown 連結限制為 HTTP(S)、`mailto:`、`tel:` 與頁內錨點。
|
||||
- 日誌有大小與檔案數上限；診斷資料有可設定的保留週期。
|
||||
|
||||
部署注意事項：
|
||||
|
||||
1. 對區網或網際網路開放前，先放在 HTTPS reverse proxy 後方，並設定 `LAZYBOY_SECURE_COOKIE=true`。
|
||||
2. 不要把 Supervisor `:7091` 對外發布，也不要將 Docker socket 掛進 Agent 電腦。
|
||||
3. `LAZYBOY_APP_TOKEN`、`SANDBOX_SUPERVISOR_TOKEN`、`LAZYBOY_VAULT_KEY` 必須使用不同的高熵值。
|
||||
4. 模型仍可能看見任務所需的網頁內容與截圖；密碼、token 與高敏感資料不要放進提示詞。
|
||||
5. CAPTCHA、2FA 與不確定的敏感操作應由使用者接管。
|
||||
|
||||
## 硬體與資源
|
||||
|
||||
| 項目 | 最低可執行 | 建議 |
|
||||
| --- | --- | --- |
|
||||
| CPU | 4 核 | 8 核以上 |
|
||||
| RAM | 8 GB，單台電腦 | 16 GB 以上 |
|
||||
| 磁碟 | 約 15 GB | Docker 至少保留 30 GB |
|
||||
| GPU | 不需要 | 模型預設走外部 API |
|
||||
| 作業系統 | macOS / Linux | Linux 可選配 LXCFS 顯示容器內 cgroup 配額 |
|
||||
|
||||
每台電腦的預設限制可由 `LAZYBOY_COMPUTER_CPUS`、`LAZYBOY_COMPUTER_MEMORY_MB`、`LAZYBOY_COMPUTER_PIDS` 調整。瀏覽器分頁的心跳會維持熱機；沒有工作且約 10 分鐘無人觀看時暫停，持續休眠約 6 小時後停止。
|
||||
|
||||
## 設定
|
||||
|
||||
`make env` 會以 `.env.example` 為基礎建立 `.env`，並保留既有金鑰。
|
||||
|
||||
| 變數 | 用途 | 預設 |
|
||||
| --- | --- | --- |
|
||||
| `XAI_API_KEY` | xAI 模型 | 空 |
|
||||
| `OPENCODE_GO_API_KEY` | OpenCode Go 模型 | 空 |
|
||||
| `OPENAI_API_KEY` | OpenAI 相容端點 | 空 |
|
||||
| `LAZYBOY_APP_TOKEN` | Web 登入 token | 必填 |
|
||||
| `SANDBOX_SUPERVISOR_TOKEN` | API ↔ Supervisor 驗證 | 必填 |
|
||||
| `LAZYBOY_VAULT_KEY` | 憑證庫加密 key | 必填且必須保持穩定 |
|
||||
| `LAZYBOY_BIND_IP` | 主機監聽位址 | `127.0.0.1` |
|
||||
| `LAZYBOY_SECURE_COOKIE` | HTTPS-only cookie | `false` |
|
||||
| `LAZYBOY_COMPUTER_CPUS` | 每台電腦 CPU | `2` |
|
||||
| `LAZYBOY_COMPUTER_MEMORY_MB` | 每台電腦記憶體 | `2048` |
|
||||
| `LAZYBOY_COMPUTER_PIDS` | 每台電腦 PID 上限 | `2048` |
|
||||
| `LAZYBOY_MEMORY_ENABLED` | 長期記憶 | `true` |
|
||||
|
||||
完整清單與保留政策請見 [`.env.example`](./.env.example)。
|
||||
|
||||
## 二次開發
|
||||
|
||||
### 本機開發
|
||||
Besides Docker, you need a Rust toolchain that supports the 2024 edition, plus Node.js / npm.
|
||||
|
||||
```bash
|
||||
make dev # 準備 .env、PostgreSQL、電腦映像
|
||||
make dev-supervisor # 終端 1：Supervisor :7091
|
||||
make dev-api # 終端 2：API :3101
|
||||
make dev
|
||||
make dev-supervisor # terminal 1
|
||||
make dev-api # terminal 2
|
||||
```
|
||||
|
||||
前端熱更新：
|
||||
Frontend hot reload in another terminal:
|
||||
|
||||
```bash
|
||||
cd apps/web
|
||||
npm install
|
||||
npm run dev # http://127.0.0.1:5173
|
||||
npm run dev
|
||||
```
|
||||
|
||||
### 檢查與測試
|
||||
Open [http://127.0.0.1:5173](http://127.0.0.1:5173). Tests and the tree layout are in the [development guide](./docs/development.md).
|
||||
|
||||
```bash
|
||||
make fmt
|
||||
make clippy
|
||||
make test
|
||||
## Docs
|
||||
|
||||
cd apps/web
|
||||
npm run typecheck
|
||||
npm run build
|
||||
The guides below are currently in Traditional Chinese.
|
||||
|
||||
cd ../..
|
||||
node --test tests/frontend.test.mjs
|
||||
python3 tests/control.test.py
|
||||
python3 tests/log-rotation.test.py
|
||||
| Doc | Contents |
|
||||
| --- | --- |
|
||||
| [Architecture](./docs/architecture.md) | Task flow, system architecture, computer lifecycle |
|
||||
| [Interactive diagram](./docs/workflow.html) | Zoomable, searchable HTML chart; download and open |
|
||||
| [Operations](./docs/operations.md) | Resources, env vars, security, site checks, sudo |
|
||||
| [Development](./docs/development.md) | Local dev, checks and tests, directory layout |
|
||||
| [Env example](./.env.example) | Environment variables and defaults |
|
||||
|
||||
# 需要已啟動的 PostgreSQL Compose service
|
||||
docker compose up -d postgres
|
||||
python3 tests/retention.test.py
|
||||
```
|
||||
## Data
|
||||
|
||||
### 專案結構
|
||||
|
||||
```text
|
||||
LazyBoy/
|
||||
├── apps/web/ React 前端與 noVNC 頁面
|
||||
├── crates/
|
||||
│ ├── api/ 對外 API 與 Agent 執行迴圈
|
||||
│ ├── contracts/ 跨元件資料契約
|
||||
│ ├── control/ 瀏覽器與桌面控制
|
||||
│ ├── controld/ 容器內控制服務
|
||||
│ ├── harness/ 模型與語音後端
|
||||
│ ├── sandbox/ API 到 Supervisor 的抽象
|
||||
│ └── supervisor/ Docker 生命週期管理
|
||||
├── image/ API、Supervisor、電腦映像
|
||||
├── migrations/ SQLx PostgreSQL migrations
|
||||
├── tests/ Rust 以外的契約與回歸測試
|
||||
├── scripts/ 環境初始化與執行工具
|
||||
├── docs/hero.html README 首頁視覺原稿
|
||||
├── docs/workflow.html 可互動產品流程圖
|
||||
├── docker-compose.yml 正式堆疊
|
||||
└── Makefile 常用開發與部署指令
|
||||
```
|
||||
|
||||
## 徽章與開源認證
|
||||
|
||||
README 頂端目前只顯示可以直接從原始碼驗證的資訊徽章，不把「尚未執行的檢查」包裝成通過。
|
||||
|
||||
若要取得可公開查驗的安全與品質徽章，建議依序完成：
|
||||
|
||||
- [ ] 在公開 GitHub repository 建立正式 mirror；OpenSSF 的公開查驗以 GitHub 為主要整合目標。
|
||||
- [ ] 加入完整 `LICENSE`、`SECURITY.md`、`CONTRIBUTING.md` 與行為準則。
|
||||
- [ ] 建立 CI：Rust format / Clippy / tests、前端 typecheck / build / tests。
|
||||
- [ ] 啟用 Dependabot 或 Renovate、CodeQL、secret scanning 與 branch protection。
|
||||
- [ ] 執行並發布 [OpenSSF Scorecard](https://scorecard.dev/) 結果後，再加入 Scorecard 徽章。
|
||||
- [ ] 在 [OpenSSF Best Practices](https://www.bestpractices.dev/) 登記專案、誠實完成 Passing 問卷後，再加入認證徽章。
|
||||
- [ ] 若提供容器映像，再加入 SBOM、簽章與可重現版本發布流程。
|
||||
|
||||
> 不建議現在顯示 CI passing、coverage、OpenSSF 或 Best Practices 徽章：目前 repository 沒有對應的公開結果，徽章會失真或直接顯示 unknown。
|
||||
|
||||
## 專案狀態
|
||||
|
||||
LazyBoy 目前版本為 `0.1.0`，仍屬早期階段。建議先在本機或受信任網路中使用；對外部署前請完成威脅模型、權限檢查、備份與還原演練。
|
||||
|
||||
Cargo workspace 的授權中繼資料宣告為 MIT。若要正式公開散布，應先在 repository 根目錄補上完整 MIT `LICENSE` 文字，再把上方授權徽章改為連到該檔案。
|
||||
Chats, memory, browser profiles, and encrypted credentials stay on your host. When you use an external model, the prompts, tool results, and screenshots the task needs may still be sent to that provider.
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
|
||||
用自然語言交代工作，保留看得見、接得回來的控制權。
|
||||
**Daniel Wang** <img src="https://flagcdn.com/w20/tw.png" width="20" alt="Taiwan" />
|
||||
|
||||
[igs170911@gmail.com](mailto:igs170911@gmail.com)
|
||||
|
||||
<a href="https://www.buymeacoffee.com/daniel.wang.1993"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me a Coffee" height="50" /></a>
|
||||
|
||||
[Apache License 2.0](./LICENSE)
|
||||
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,133 @@
|
|||
<div align="center">
|
||||
|
||||
<img src="./apps/web/public/lazyboy-icon.png" width="96" height="96" alt="LazyBoy" />
|
||||
|
||||
[English](./README.md) · **繁體中文**
|
||||
|
||||
# LazyBoy
|
||||
|
||||
**給 AI 一台電腦，讓它替你動手做事。**
|
||||
|
||||
自架的 AI Agent 工作空間。用文字或語音交代任務，即時觀看桌面操作，隨時接回控制權。
|
||||
|
||||
[快速開始](#快速開始) · [功能](#功能) · [架構與流程圖](./docs/architecture.md) · [部署指南](./docs/operations.md) · [開發指南](./docs/development.md)
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
LazyBoy 讓 Agent 在 Docker 裡使用自己的 Linux 桌面，操作瀏覽器、終端與檔案。你可以建立不同的 Agent、在群組中協作，把示範整理成技能，再安排定時執行。
|
||||
|
||||
目前為早期版本 `0.1.0`，提供桌面與手機瀏覽器介面；模型金鑰由你自行設定。
|
||||
|
||||
## 功能
|
||||
|
||||
- **持續的工作空間**：每個 Agent 有自己的對話、工作紀錄與可設定的長期記憶。
|
||||
- **真的能操作電腦**：開網頁、使用終端、整理檔案、操作圖形介面，過程可即時觀看。
|
||||
- **隨時人工接管**：在同一個桌面完成登入、驗證或手動調整，再交回 Agent。
|
||||
- **多 Agent 與群組**：支援 Team 共用電腦與 Private 獨立電腦模式。
|
||||
- **示範教學與排程**：把操作示範整理成技能，使用 cron 安排重複工作。
|
||||
- **自選模型與工具**：支援 xAI、OpenCode Go、OpenAI 相容端點，以及 MCP 與檔案技能。
|
||||
- **語音通話**：設定語音服務後，可以透過通話與 Agent 互動。
|
||||
- **手機操作**：可收合聊天側欄，遠端桌面提供鍵盤、觸控板、右鍵、拖曳與捲動控制。
|
||||
- **英文與繁體中文**：可在介面切換語言。
|
||||
|
||||
## 快速開始
|
||||
|
||||
需要 Docker 與 Compose、Git、Make、Python 3，以及一組支援的模型 API 金鑰。macOS 可使用 Docker Desktop 或 OrbStack；Linux 可使用 Docker Engine。
|
||||
|
||||
取得本專案原始碼後，在專案根目錄執行：
|
||||
|
||||
```bash
|
||||
make env
|
||||
```
|
||||
|
||||
編輯產生的 `.env`，填入 `XAI_API_KEY`、`OPENCODE_GO_API_KEY` 或 `OPENAI_API_KEY` 其中之一。初始化工具會產生登入與服務所需的金鑰，重新執行時保留既有設定。
|
||||
|
||||
```bash
|
||||
make up
|
||||
make health
|
||||
```
|
||||
|
||||
開啟 **[http://127.0.0.1:3101](http://127.0.0.1:3101)**，用 `.env` 裡的 `LAZYBOY_APP_TOKEN` 登入，建立 Agent 並選擇模型。
|
||||
|
||||
第一次會從原始碼建置 API 與 Linux 桌面映像，所需時間較長。容器內已包含建置環境，完整 Docker 部署不需要在主機安裝 Rust 或 Node.js。
|
||||
|
||||
可以先試一個具體任務：
|
||||
|
||||
> 開啟我指定的網站，整理頁面重點，將結果存成工作區裡的 Markdown 檔案。
|
||||
|
||||
服務狀態與日誌：
|
||||
|
||||
```bash
|
||||
make ps
|
||||
make logs
|
||||
make down # 停止服務，保留 PostgreSQL 資料
|
||||
```
|
||||
|
||||
資源限制、環境變數、HTTPS 與容器內 sudo 設定，請見 [部署與操作](./docs/operations.md)。
|
||||
|
||||
## 在手機上使用
|
||||
|
||||
手機與桌面使用同一個 Web 介面。將服務部署在可連線的主機後，用手機瀏覽器開啟該服務的網址即可；`127.0.0.1` 只代表手機本身，不能拿來連另一台電腦。
|
||||
|
||||
聊天側欄可點外側空白處收合。操作遠端桌面時，可切換直接點選與觸控板模式，使用工具列叫出鍵盤、按右鍵或拖曳。對外提供服務時請設定 HTTPS，詳見 [部署指南](./docs/operations.md#安全模型)。
|
||||
|
||||
## 技術組成
|
||||
|
||||
- **前端**：React 19、TypeScript、Vite、noVNC
|
||||
- **後端**：Rust 2024、Axum、Tokio
|
||||
- **資料**：PostgreSQL、pgvector、SQLx
|
||||
- **桌面**：Docker、Debian、XFCE、Chromium、Xvfb
|
||||
- **電腦控制**：CDP、AT-SPI、X11
|
||||
- **擴充**：MCP、檔案技能、示範 playbook
|
||||
|
||||
流程圖、控制權交接、元件職責與電腦生命週期狀態機，集中在 **[架構與流程](./docs/architecture.md)**。原有的 **[互動流程圖](./docs/workflow.html)** 也保留；下載後用瀏覽器開啟即可操作。
|
||||
|
||||
## 本機開發
|
||||
|
||||
除了 Docker，需安裝支援 Rust 2024 edition 的 Rust 工具鏈，以及 Node.js／npm。
|
||||
|
||||
```bash
|
||||
make dev
|
||||
make dev-supervisor # 終端 1
|
||||
make dev-api # 終端 2
|
||||
```
|
||||
|
||||
前端熱更新使用另一個終端：
|
||||
|
||||
```bash
|
||||
cd apps/web
|
||||
npm install
|
||||
npm run dev
|
||||
```
|
||||
|
||||
開啟 [http://127.0.0.1:5173](http://127.0.0.1:5173)。測試指令與專案目錄說明請見 [開發指南](./docs/development.md)。
|
||||
|
||||
## 文件
|
||||
|
||||
| 文件 | 內容 |
|
||||
| --- | --- |
|
||||
| [架構與流程](./docs/architecture.md) | 任務流程圖、系統架構、電腦生命週期狀態機 |
|
||||
| [互動流程圖](./docs/workflow.html) | 可縮放、搜尋的 HTML 圖表；下載後開啟 |
|
||||
| [部署與操作](./docs/operations.md) | 資源、環境變數、安全設定、網站驗證、sudo |
|
||||
| [開發指南](./docs/development.md) | 本機開發、檢查與測試、目錄結構 |
|
||||
| [設定範例](./.env.example) | 環境變數與預設值 |
|
||||
|
||||
## 資料
|
||||
|
||||
對話、記憶、瀏覽器設定檔與加密憑證保存在自架主機。使用外部模型時，任務所需的提示詞、工具結果與截圖仍可能傳送給該模型供應商。
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
|
||||
**danielwang** <img src="https://flagcdn.com/w20/tw.png" width="20" alt="Taiwan" />
|
||||
|
||||
[igs170911@gmail.com](mailto:igs170911@gmail.com)
|
||||
|
||||
<a href="https://www.buymeacoffee.com/daniel.wang.1993"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="請我喝杯咖啡" height="50" /></a>
|
||||
|
||||
[Apache License 2.0](./LICENSE)
|
||||
|
||||
</div>
|
||||
|
|
@ -1 +1 @@
|
|||
<!doctype html><html lang="zh-Hant"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#050506"/><title>LazyBoy</title><link rel="icon" href="/favicon.ico" sizes="any"/><link rel="icon" href="/favicon.svg" type="image/svg+xml"/><link rel="icon" href="/favicon.png" type="image/png" sizes="32x32"/><link rel="apple-touch-icon" href="/apple-touch-icon.png"/></head><body><div id="root"></div><script type="module" src="/src/main.tsx"></script></body></html>
|
||||
<!doctype html><html lang="zh-Hant"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#050506"/><title>LazyBoy</title><link rel="icon" href="/favicon.png?v=white-1" type="image/png" sizes="64x64"/><link rel="apple-touch-icon" href="/apple-touch-icon.png?v=white-1"/></head><body><div id="root"></div><script type="module" src="/src/main.tsx"></script></body></html>
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 4.4 KiB After Width: | Height: | Size: 36 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 604 B After Width: | Height: | Size: 6.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 259 KiB |
|
|
@ -115,6 +115,12 @@ export function App(){
|
|||
const [rightCollapsed,setRightCollapsed]=useState(paneStart.collapsed); const [rightPart,setRightPart]=useState<RightPart>(paneStart.part);
|
||||
const [sessionMenuOpen,setSessionMenuOpen]=useState(false); const [clearOpen,setClearOpen]=useState(false); const [sessionToDelete,setSessionToDelete]=useState<string|null>(null); const [remembered,setRemembered]=useState<Record<string,boolean>>({});
|
||||
const [mobileNav,setMobileNav]=useState(false); const [error,setError]=useState<string|null>(null); const [busy,setBusy]=useState(false);
|
||||
useEffect(()=>{
|
||||
if(!mobileNav)return;
|
||||
const dismiss=(event:KeyboardEvent)=>{if(event.key==="Escape")setMobileNav(false)};
|
||||
window.addEventListener("keydown",dismiss);
|
||||
return()=>window.removeEventListener("keydown",dismiss);
|
||||
},[mobileNav]);
|
||||
const [desktopClipboard,setDesktopClipboard]=useState(""); const active=bots.find(b=>b.id===activeId)||null;
|
||||
const activeRoom=rooms.find(room=>room.id===activeRoomId)||null;
|
||||
const [clipboardOpen,setClipboardOpen]=useState(false);
|
||||
|
|
@ -196,7 +202,7 @@ export function App(){
|
|||
useEffect(()=>{if(!plusOpen)setSkillQuery("")},[plusOpen]);
|
||||
useEffect(()=>{messageEndRef.current?.scrollIntoView({block:"end",behavior:"smooth"})},[activeSessionId,lastMessageId,workingMembers.length,pausedForUser]);
|
||||
useEffect(()=>{if(!activeSessionId||(!activeId&&!activeRoomId)){refreshSeqRef.current+=1;setMessages([]);setScreenUrl(null);if(!activeId&&!activeRoomId)setComputer(blankComputer);return}setScreenUrl(null);refresh().catch(e=>setError(localizeError(e.message)));const timer=setInterval(()=>{refresh().catch(()=>{});const beat=roomsRef.current.find(room=>room.id===activeRoomId)?.members[0]?.id||activeId;if(beat)api(`/api/computer/${beat}/heartbeat`,{method:"POST",body:"{}"}).catch(()=>{})},2000);return()=>{clearInterval(timer);refreshSeqRef.current+=1}},[activeId,activeRoomId,activeSessionId,refresh]);
|
||||
useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard?.writeText(text).then(()=>setClipboardStatus(t("clipboardSynced"))).catch(()=>setClipboardStatus(t("clipboardSyncBlocked")))}if(event.data.type==="lazyboy-copy-request"&&computer.controlHolder==="user")void copySelection();if(event.data.type==="lazyboy-paste-text"&&typeof event.data.text==="string")pasteText(event.data.text);if(event.data.type==="lazyboy-paste-request"&&computer.controlHolder==="user")setClipboardOpen(true);if(event.data.type==="lazyboy-desktop-ready")setDesktopReady(true);if(event.data.type==="lazyboy-desktop-lost")setDesktopReady(false)};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)});
|
||||
useEffect(()=>{const listener=(event:MessageEvent)=>{if(event.origin!==location.origin||event.source!==desktopFrameRef.current?.contentWindow||!event.data)return;if(event.data.type==="lazyboy-desktop-clipboard"){const text=String(event.data.text||"");setDesktopClipboard(text);navigator.clipboard?.writeText(text).then(()=>setClipboardStatus(t("clipboardSynced"))).catch(()=>setClipboardStatus(t("clipboardSyncBlocked")))}if(event.data.type==="lazyboy-copy-request"&&computer.controlHolder==="user")void copySelection();if(event.data.type==="lazyboy-paste-text"&&typeof event.data.text==="string")pasteText(event.data.text);if(event.data.type==="lazyboy-mobile-key"&&typeof event.data.key==="string"&&["Return","BackSpace","Tab","Escape","Left","Right","Up","Down"].includes(event.data.key))queueDesktopInput({kind:"key",key:event.data.key});if(event.data.type==="lazyboy-paste-request"&&computer.controlHolder==="user")setClipboardOpen(true);if(event.data.type==="lazyboy-desktop-ready")setDesktopReady(true);if(event.data.type==="lazyboy-desktop-lost")setDesktopReady(false)};window.addEventListener("message",listener);return()=>window.removeEventListener("message",listener)});
|
||||
useEffect(()=>{setDesktopReady(false)},[screenUrl,paneBotId]);
|
||||
useEffect(()=>{if(skipHandoffRef.current){skipHandoffRef.current=false;holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId;return}if((holderRef.current!==computer.controlHolder||paneBotRef.current!==paneBotId)&&computer.state==="running")setHandingOff(true);holderRef.current=computer.controlHolder;paneBotRef.current=paneBotId},[computer.controlHolder,paneBotId,computer.state]);
|
||||
useEffect(()=>{if(!handingOff)return;const timer=setTimeout(()=>setHandingOff(false),1600);return()=>clearTimeout(timer)},[handingOff]);
|
||||
|
|
@ -210,7 +216,7 @@ export function App(){
|
|||
useEffect(()=>{localStorage.setItem(PANE_STORE,JSON.stringify({collapsed:rightCollapsed,part:rightPart}))},[rightCollapsed,rightPart]);
|
||||
useEffect(()=>{if(!paneBotId){setSchedules([]);return}api<ScheduleItem[]>(`/api/bots/${paneBotId}/schedules`).then(setSchedules).catch(()=>setSchedules([]))},[paneBotId]);
|
||||
useEffect(()=>{if(computer.takeoverRequested){setRightPart("computer");setRightCollapsed(false)}},[computer.takeoverRequested]);
|
||||
function openPane(part:RightPart){setRightPart(part);setRightCollapsed(false)}
|
||||
function openPane(part:RightPart){setMobileNav(false);setRightPart(part);setRightCollapsed(false)}
|
||||
async function openLoginScreen(){
|
||||
const id=paneBot?.id||active?.id;if(!id)return;
|
||||
setRightPart("computer");setRightCollapsed(false);setComputerOpen(true);
|
||||
|
|
@ -258,13 +264,14 @@ export function App(){
|
|||
async function clearSession(){if(!activeSessionId)return;setClearOpen(false);setSessionMenuOpen(false);await action(async()=>{await api(`/api/sessions/${activeSessionId}/messages`,{method:"DELETE"});setMessages([]);await loadSessions()})}
|
||||
async function deleteSession(id:string){if(!sessionsPath||!sessionStoreKey)return;setSessionMenuOpen(false);setBusy(true);setError(null);try{await api(`/api/sessions/${id}`,{method:"DELETE"});const next=await api<Session[]>(sessionsPath);setSessions(next);const pick=id===activeSessionId||!next.some(session=>session.id===activeSessionId)?next[0]?.id||null:activeSessionId;setActiveSessionId(pick);if(pick)writeSessionStore(sessionStoreKey,pick)}catch(e){setError(e instanceof Error?localizeError(e.message):t("operationFailed"))}finally{setBusy(false)}}
|
||||
async function rememberMessage(message:Message){const botId=message.speakerBotId||active?.id||activeRoom?.members[0]?.id;if(!botId||!message.body.trim())return;try{await api(`/api/bots/${botId}/memories`,{method:"POST",body:JSON.stringify({content:message.body,sessionId:activeSessionId})});setRemembered(current=>({...current,[message.id]:true}))}catch(e){setError(e instanceof Error?localizeError(e.message):t("rememberFailed"))}}
|
||||
function pasteText(text:string){
|
||||
function pasteText(text:string){queueDesktopInput({kind:"clipboard",text})}
|
||||
function queueDesktopInput(input:{kind:"clipboard";text:string}|{kind:"key";key:string}){
|
||||
const botId=paneBotId;
|
||||
if(!botId||computer.controlHolder!=="user")return;
|
||||
pasteQueueRef.current=pasteQueueRef.current.catch(()=>{}).then(async()=>{
|
||||
if(currentPaneRef.current!==botId)return;
|
||||
await api(`/api/computer/${botId}/input`,{method:"POST",body:JSON.stringify({kind:"clipboard",text})});
|
||||
if(currentPaneRef.current===botId)setClipboardStatus(t("clipboardPasted"));
|
||||
await api(`/api/computer/${botId}/input`,{method:"POST",body:JSON.stringify(input)});
|
||||
if(input.kind==="clipboard"&¤tPaneRef.current===botId)setClipboardStatus(t("clipboardPasted"));
|
||||
}).catch(()=>setClipboardStatus(t("clipboardPasteFailed")));
|
||||
}
|
||||
async function copySelection(){
|
||||
|
|
@ -313,8 +320,9 @@ export function App(){
|
|||
if(authRequired)return <LoginScreen authenticated={async()=>{setAuthRequired(false);setError(null);try{await loadBots()}catch(e){if(e instanceof ApiError&&e.status===401)setAuthRequired(true);else setError(e instanceof Error?localizeError(e.message):t("loginFailed"))}}}/>;
|
||||
|
||||
return <AvatarLookProvider value={looks}><div className={`app-shell ${rightCollapsed?"right-collapsed":"right-open"}`}>
|
||||
<aside className={`sidebar ${mobileNav?"open":""}`}>
|
||||
<div className="brand"><span>LazyBoy</span><div className="create-menu-wrap" onClick={event=>event.stopPropagation()}><button type="button" className="icon-button" onClick={event=>{event.stopPropagation();setCreateMenuOpen(v=>!v)}} aria-label={t("add")} aria-haspopup="menu" aria-expanded={createMenuOpen}><UseAnimations animation={plusToX} size={18} strokeColor="var(--ink)" wrapperStyle={{pointerEvents:"none"}}/></button>{createMenuOpen&&<div className="create-menu" role="menu"><button type="button" role="menuitem" onClick={()=>{setCreateMenuOpen(false);setCreateOpen(true)}}><BotIcon/>{t("addBot")}</button><button type="button" role="menuitem" onClick={()=>{setCreateMenuOpen(false);setGroupOpen(true)}} disabled={bots.length===0}><Users/>{t("addGroup")}</button></div>}</div></div>
|
||||
{mobileNav&&<button type="button" className="mobile-nav-backdrop" aria-label={t("close")} onClick={()=>setMobileNav(false)}/>}
|
||||
<aside id="conversation-sidebar" className={`sidebar ${mobileNav?"open":""}`}>
|
||||
<div className="brand"><img className="brand-icon" src="/lazyboy-icon.png?v=white-1" alt=""/><span>LazyBoy</span><button type="button" className="icon-button mobile-nav-close" aria-label={t("close")} onClick={()=>setMobileNav(false)}>×</button><div className="create-menu-wrap" onClick={event=>event.stopPropagation()}><button type="button" className="icon-button" onClick={event=>{event.stopPropagation();setCreateMenuOpen(v=>!v)}} aria-label={t("add")} aria-haspopup="menu" aria-expanded={createMenuOpen}><UseAnimations animation={plusToX} size={18} strokeColor="var(--ink)" wrapperStyle={{pointerEvents:"none"}}/></button>{createMenuOpen&&<div className="create-menu" role="menu"><button type="button" role="menuitem" onClick={()=>{setCreateMenuOpen(false);setCreateOpen(true)}}><BotIcon/>{t("addBot")}</button><button type="button" role="menuitem" onClick={()=>{setCreateMenuOpen(false);setGroupOpen(true)}} disabled={bots.length===0}><Users/>{t("addGroup")}</button></div>}</div></div>
|
||||
<label className="search"><UseAnimations animation={searchToX} size={16} strokeColor="var(--muted)"/><input value={query} onChange={e=>setQuery(e.target.value)} placeholder={t("search")}/></label>
|
||||
<div className="bot-list">
|
||||
{filteredRooms.length>0&&<section className="bot-group"><div className="group-label">{t("groups")}</div>{filteredRooms.map(room=><button className={`bot-row room-row ${room.id===activeRoomId?"selected":""}`} key={room.id} onClick={()=>openRoom(room)} onContextMenu={e=>{e.preventDefault();setRoomContext({room,x:e.clientX,y:e.clientY})}}><span className="avatar-wrap"><AvatarStack members={room.members} online thinkingIds={room.id===activeRoomId?busyMembers.map(member=>member.id):[]}/>{room.unreadCount>0&&<i className="unread-dot" title={t("unreadMessages",{count:room.unreadCount})}/>}</span><span className="bot-copy"><strong>{room.name}</strong><small>{room.lastPreview||t("members",{count:room.members.length})}</small></span>{room.lastMessageAt&&<time className="row-time">{inboxTime(room.lastMessageAt)}</time>}</button>)}</section>}
|
||||
|
|
@ -347,7 +355,7 @@ export function App(){
|
|||
</aside>
|
||||
|
||||
<main className="chat-panel">
|
||||
<header className="topbar"><button className="icon-button mobile-menu" onClick={()=>setMobileNav(v=>!v)}><UseAnimations animation={menu} size={18} strokeColor="var(--ink)"/></button>{activeRoom?<><AvatarStack members={activeRoom.members} size={32} online thinkingIds={busyMembers.map(member=>member.id)}/><strong>{activeRoom.name}</strong><SessionMenu sessions={sessions} activeSessionId={activeSessionId} open={sessionMenuOpen} setOpen={setSessionMenuOpen} busy={busy} onSelect={selectSession} onCreate={createSession} onDelete={id=>{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/><span className="grow"/>{topTools}</>:active?<><Avatar lookId={active.id} name={active.name} color={active.avatarColor} shape={active.avatarShape} active online/><strong>{active.name}</strong><SessionMenu sessions={sessions} activeSessionId={activeSessionId} open={sessionMenuOpen} setOpen={setSessionMenuOpen} busy={busy} onSelect={selectSession} onCreate={createSession} onDelete={id=>{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/><span className="grow"/>{topTools}</>:<><strong>{t("chooseBot")}</strong><span className="grow"/>{topTools}</>}</header>
|
||||
<header className="topbar"><button type="button" className="icon-button mobile-menu" aria-label={t("chooseBot")} aria-expanded={mobileNav} aria-controls="conversation-sidebar" onClick={()=>setMobileNav(v=>!v)}><UseAnimations animation={menu} size={18} strokeColor="var(--ink)"/></button>{activeRoom?<><AvatarStack members={activeRoom.members} size={32} online thinkingIds={busyMembers.map(member=>member.id)}/><strong>{activeRoom.name}</strong><SessionMenu sessions={sessions} activeSessionId={activeSessionId} open={sessionMenuOpen} setOpen={setSessionMenuOpen} busy={busy} onSelect={selectSession} onCreate={createSession} onDelete={id=>{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/><span className="grow"/>{topTools}</>:active?<><Avatar lookId={active.id} name={active.name} color={active.avatarColor} shape={active.avatarShape} active online/><strong>{active.name}</strong><SessionMenu sessions={sessions} activeSessionId={activeSessionId} open={sessionMenuOpen} setOpen={setSessionMenuOpen} busy={busy} onSelect={selectSession} onCreate={createSession} onDelete={id=>{setSessionMenuOpen(false);setSessionToDelete(id)}} onClear={()=>{setSessionMenuOpen(false);setClearOpen(true)}}/><span className="grow"/>{topTools}</>:<><strong>{t("chooseBot")}</strong><span className="grow"/>{topTools}</>}</header>
|
||||
{callOpen&&voiceSettings?.enabled&&active&&activeSessionId&&!activeRoomId?<CallOverlay bot={active} sessionId={activeSessionId} takeover={computer.takeoverRequested||computer.controlHolder==="user"} onHangUp={()=>setCallOpen(false)} onTakeOver={()=>{setRightPart("computer");setRightCollapsed(false);if(active)void action(()=>api(`/api/computer/${active.id}/takeover`,{method:"POST",body:"{}"}))}}/>:null}
|
||||
<div className="messages">{(activeRoom||active)&&messages.length===0?<div className="welcome">{activeRoom?<AvatarStack members={activeRoom.members} size={56} online/>:<Avatar lookId={active!.id} name={active!.name} color={active!.avatarColor} shape={active!.avatarShape} active online size={64}/>}<h1>{activeRoom?t("startRoomDiscussion",{name:activeRoom.name}):t("startBotWork",{name:active!.name})}</h1><p>{activeRoom?t("roomWillReply",{names:listJoin(activeRoom.members.map(member=>member.name))}):active!.description||t("botWelcome")}</p></div>:messages.map(message=>{const spoken=message.role!=="user"&&Boolean(activeRoom);const speakerName=message.speakerName||(spoken?paneBot?.name:undefined);const speakerShape=(message.speakerShape||paneBot?.avatarShape||"blob") as AvatarShape;const files=messageFiles(message.blocks);const chips=chipBlocks(message.blocks);const hideBody=isAutoAttachCaption(message.body,files)||isAutoScheduleCaption(message.body,chips);return <div key={message.id} className={`message ${message.role} ${spoken?"spoken":""} ${files.length?"with-files":""}`}>{spoken&&<span className="msg-avatar"><Avatar lookId={message.speakerBotId||paneBot?.id||undefined} name={speakerName||"agent"} color={message.speakerColor||undefined} shape={speakerShape} size={22}/></span>}{spoken&&<b className="speaker" style={{color:message.speakerColor||undefined}}>{speakerName}</b>}{files.length>0&&<div className="msg-attachments">{files.map(file=><FileCard key={file.name} file={file}/>)}</div>}{chips.map((chip,index)=>chip.kind==="login"?<div className="login-chip" key={`${message.id}-login-${index}`}><div className="login-label">{t("loginNeedsYou")}</div><div className="login-site">{chip.site||message.body}</div>{chip.why?<div className="login-why">{t("loginWhy",{why:chip.why})}</div>:null}<button type="button" className="primary" onClick={()=>void openLoginScreen()}>{t("loginOpenScreen")}</button></div>:chip.kind==="schedule"?<div className="sched-chip" key={`${message.id}-sched-${index}`}><div className="sched-label">{t("scheduleChip")}</div><strong>{chip.name}</strong><small>{scheduleWhen(chip.cron,chip.human)}</small></div>:<div className="sched-chip" key={`${message.id}-run-${index}`}><div className="sched-label">{t("scheduleRunChip")}</div><strong>{chip.name}</strong><small>{scheduleWhen(chip.cron,chip.human)}</small></div>)}{!hideBody&&(message.role==="assistant"?<div className="message-stack"><div className="message-body md"><ChatMarkdown>{message.body}</ChatMarkdown></div>{message.body.trim()?<CopyMessageButton text={message.body}/>:null}</div>:<span className="message-body">{message.body}</span>)}{!hideBody&&message.body.trim()&&<button type="button" className={`remember-msg ${remembered[message.id]?"saved":""}`} title={remembered[message.id]?t("remembered"):t("remember")} disabled={!!remembered[message.id]} onClick={()=>void rememberMessage(message)}><UseAnimations animation={bookmark} size={14} strokeColor="var(--muted)"/></button>}</div>})}{pausedForUser&&paneBot&&<div className="pause-banner" role="status"><span>{computer.controlHolder==="user"?t("pausedUserControl",{name:paneBot.name}):t("pausedNeedsUser",{name:paneBot.name})}{(computer.queuedRuns||0)>0&&<> {t("queuedMessages",{count:computer.queuedRuns||0})}</>}</span>{computer.controlHolder==="user"?<button type="button" className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${paneBot.id}/release`,{method:"POST",body:"{}"}))}>{t("releaseAndContinue")}</button>:<button type="button" className="primary" disabled={busy} onClick={()=>void action(()=>api(`/api/computer/${paneBot.id}/takeover`,{method:"POST",body:"{}"}))}>{t("takeOverNow")}</button>}</div>}{teaching&&active&&<div className="teach-banner recording" role="status"><span><i className="record-dot live"/>{t("teachingLive",{goal:teaching.goal})}<small>{t("teachingHint")}{teaching.eventCount>0&&<> · {t("teachingCaptured",{count:teaching.eventCount})}</>}</small></span><button type="button" className="primary" disabled={busy} onClick={()=>void stopTeaching()}>{t("finishDemo")}</button><button type="button" className="outline" disabled={busy} onClick={()=>void cancelTeaching()}>{t("cancel")}</button></div>}{drafting&&<div className="teach-banner" role="status"><UseAnimations animation={loading} size={18} wrapperStyle={{display:"inline-block",verticalAlign:"middle"}}/><span>{t("distilling",{goal:drafting.goal})}</span></div>}{skillDraft&&active&&<SkillDraftCard key={skillDraft.id} skill={skillDraft} busy={busy} onSave={(name,playbook)=>void saveSkill(skillDraft,name,playbook)} onTest={(name,playbook)=>void testSkill(skillDraft,name,playbook)} onDiscard={()=>void discardSkill(skillDraft)} onEdit={()=>setEditingSkillId(skillDraft.id)} onExport={(name,playbook)=>downloadSkill(name,skillDraft.goal,playbook)}/>}<div ref={messageEndRef} aria-hidden="true"/></div>
|
||||
{error&&<div className="error-banner"><span>{error}</span><button onClick={()=>setError(null)}><X/></button></div>}
|
||||
|
|
@ -818,7 +826,7 @@ function AboutDialog({close}:{close:()=>void}){
|
|||
const[health,setHealth]=useState<"ok"|"bad"|"…">("…");
|
||||
useEffect(()=>{fetch("/api/health").then(r=>r.ok?setHealth("ok"):setHealth("bad")).catch(()=>setHealth("bad"))},[]);
|
||||
return <div className="modal-backdrop"><div className="dialog compact"><div className="dialog-title"><h2>{t("about")}</h2><button type="button" onClick={close}><X/></button></div>
|
||||
<div className="about-brand"><Avatar name="L" size={52} online/><strong>LazyBoy</strong><small>0.1.0</small></div>
|
||||
<div className="about-brand"><img src="/lazyboy-icon.png?v=white-1" width="52" height="52" alt="" style={{borderRadius:14}}/><strong>LazyBoy</strong><small>0.1.0</small></div>
|
||||
<p>{t("aboutDescription")}</p>
|
||||
<p className="dialog-lead">{t("apiStatus",{status:health==="ok"?t("statusNormal"):health==="bad"?t("statusUnavailable"):t("statusChecking")})}</p>
|
||||
<div className="dialog-actions"><button className="primary" onClick={close}>{t("close")}</button></div>
|
||||
|
|
|
|||
|
|
@ -26,3 +26,32 @@
|
|||
.computer-part .preview{max-height:none}
|
||||
}
|
||||
@media(prefers-reduced-motion:reduce){.computer-signal,.computer-signal::before,.computer-signal-face i,.avatar.blobatar.thinking,.avatar-stack .stack-item .avatar.thinking{animation:none!important}}
|
||||
|
||||
/* Mobile conversation drawer and two-row chat header. */
|
||||
.mobile-nav-backdrop,.mobile-nav-close{display:none}
|
||||
.brand-icon{width:30px;height:30px;flex:none;border-radius:9px;margin-right:9px}
|
||||
@media(max-width:700px){
|
||||
.mobile-nav-backdrop{display:block;position:fixed;inset:0;z-index:39;border:0;border-radius:0;
|
||||
padding:0;background:#0008;backdrop-filter:blur(3px);touch-action:manipulation}
|
||||
.sidebar{visibility:hidden;padding-top:env(safe-area-inset-top);padding-bottom:env(safe-area-inset-bottom)}
|
||||
.sidebar.open{visibility:visible}
|
||||
.mobile-nav-close{display:inline-flex;align-items:center;justify-content:center;min-width:44px;min-height:44px;
|
||||
margin-left:auto;font-size:26px;line-height:1}
|
||||
.brand .create-menu-wrap{margin-left:4px}
|
||||
.topbar{display:grid;grid-template-columns:44px 32px minmax(0,1fr) minmax(0,104px);
|
||||
grid-template-rows:44px auto;gap:8px;height:auto;flex:0 0 auto;
|
||||
padding:calc(8px + env(safe-area-inset-top)) 12px 8px;align-items:center}
|
||||
.topbar>.mobile-menu{grid-column:1;grid-row:1;width:44px;height:44px;justify-content:center}
|
||||
.topbar>.avatar,.topbar>.avatar-stack{grid-column:2;grid-row:1;margin:0}
|
||||
.topbar>strong{grid-column:3;grid-row:1;font-size:15px;line-height:1.3}
|
||||
.topbar>.session-picker{grid-column:4;grid-row:1;margin:0;min-width:0}
|
||||
.topbar .session-current{max-width:100%;min-height:44px;width:100%}
|
||||
.topbar>.grow{display:none}
|
||||
.topbar>.top-tools{grid-column:1 / -1;grid-row:2;display:flex;width:100%;min-width:0;
|
||||
margin:0;padding:0;border:0;border-radius:0;gap:4px;background:transparent;box-shadow:none}
|
||||
.topbar .top-tool-button,.topbar .call-entry{flex:1;min-width:0;width:auto}
|
||||
.topbar .top-tool-button{height:44px;min-height:44px;border-radius:10px;background:var(--surface)}
|
||||
.topbar .call-entry{display:flex}
|
||||
.topbar .call-entry>.top-tool-button{width:100%}
|
||||
.topbar .session-menu{right:0;left:auto;max-width:calc(100vw - 24px)}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { readFileSync } from "node:fs";
|
||||
import { defineConfig } from "vite";
|
||||
import react from "@vitejs/plugin-react";
|
||||
export default defineConfig({plugins:[react()],build:{outDir:"dist",emptyOutDir:true},server:{port:5173,proxy:{"/api":{target:"http://127.0.0.1:3101",ws:true},"/view":{target:"http://127.0.0.1:3101",ws:true}}}});
|
||||
export default defineConfig({plugins:[react(),{name:"desktop-viewer",generateBundle(){this.emitFile({type:"asset",fileName:"vnc.html",source:readFileSync(new URL("./vnc.html",import.meta.url),"utf8")})}}],build:{outDir:"dist",emptyOutDir:true},server:{port:5173,proxy:{"/api":{target:"http://127.0.0.1:3101",ws:true},"/view":{target:"http://127.0.0.1:3101",ws:true}}}});
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>LazyBoy desktop</title>
|
||||
<style>
|
||||
html, body, #screen {
|
||||
|
|
@ -14,7 +15,26 @@
|
|||
#status {
|
||||
display: none;
|
||||
}
|
||||
body { display:flex; flex-direction:column; }
|
||||
#screen { flex:1; min-height:0; height:auto; }
|
||||
#screen canvas { cursor: default; }
|
||||
#mobile-controls { display:none; flex:none; padding:6px 8px max(6px,env(safe-area-inset-bottom));
|
||||
background:#172033; color:#f8fafc; font:12px system-ui; border-top:1px solid #334155; }
|
||||
#mobile-controls .buttons { display:flex; gap:5px; overflow-x:auto; }
|
||||
#mobile-controls button { flex:1 0 auto; min-height:44px; padding:8px 10px;
|
||||
border:1px solid #64748b; border-radius:8px; background:#25334a; color:inherit; font:inherit; touch-action:manipulation; }
|
||||
#mobile-controls button[aria-pressed="true"] { background:#155e75; border-color:#67e8f9; }
|
||||
#mobile-controls button:disabled { opacity:.4; }
|
||||
#pointer-help { margin:5px 0 0; line-height:1.4; }
|
||||
#mobile-cursor { display:none; position:fixed; width:18px; height:18px; border:2px solid white;
|
||||
border-radius:50%; box-shadow:0 0 0 1px #000,0 1px 5px #000; transform:translate(-50%,-50%);
|
||||
pointer-events:none; z-index:5; }
|
||||
#mobile-cursor::after { content:""; position:absolute; width:4px; height:4px; background:#fff;
|
||||
border-radius:50%; left:7px; top:7px; }
|
||||
@media(any-pointer:coarse) { #mobile-controls { display:block; } }
|
||||
/* Keep a real editable element on screen for iOS/Android keyboards. */
|
||||
#mobile-keyboard { position:fixed; bottom:0; left:0; width:1px; height:1px;
|
||||
padding:0; border:0; opacity:.01; font-size:16px; pointer-events:none; }
|
||||
</style>
|
||||
<script type="module">
|
||||
import RFB from "./core/rfb.js";
|
||||
|
|
@ -48,6 +68,221 @@
|
|||
let rfb = null;
|
||||
let reconnectTimer = null;
|
||||
const parentOrigin = window.location.origin;
|
||||
const keyboard = document.getElementById("mobile-keyboard");
|
||||
const sentinel = "\u200b";
|
||||
let composing = false;
|
||||
let touchStart = null;
|
||||
|
||||
const controls = document.getElementById("mobile-controls");
|
||||
const cursor = document.getElementById("mobile-cursor");
|
||||
const modeButton = document.getElementById("pointer-mode");
|
||||
const dragButton = document.getElementById("pointer-drag");
|
||||
const pointerHelp = document.getElementById("pointer-help");
|
||||
let trackpad = false;
|
||||
let dragging = false;
|
||||
let gesture = null;
|
||||
let pointer = { x: .5, y: .5 };
|
||||
|
||||
function pointerGeometry() {
|
||||
const canvas = document.querySelector("#screen canvas");
|
||||
const rect = canvas?.getBoundingClientRect();
|
||||
return rect && rect.width > 0 && rect.height > 0 ? rect : null;
|
||||
}
|
||||
function paintPointer() {
|
||||
const rect = pointerGeometry();
|
||||
if (!cursor?.style) return;
|
||||
cursor.style.display = trackpad && rect && rfb && !rfb.viewOnly ? "block" : "none";
|
||||
if (rect) {
|
||||
cursor.style.left = `${rect.left + pointer.x * rect.width}px`;
|
||||
cursor.style.top = `${rect.top + pointer.y * rect.height}px`;
|
||||
}
|
||||
}
|
||||
// Pointer adapter for the pinned noVNC 1.7.0. These methods take scaled
|
||||
// canvas coordinates and preserve noVNC's throttling/protocol handling.
|
||||
function sendPointer(mask = null) {
|
||||
const rect = pointerGeometry();
|
||||
if (!rect || !rfb || rfb.viewOnly) return;
|
||||
const x = Math.min(rect.width - 1, Math.max(0, pointer.x * rect.width));
|
||||
const y = Math.min(rect.height - 1, Math.max(0, pointer.y * rect.height));
|
||||
if (mask === null) rfb._handleMouseMove(x, y);
|
||||
else rfb._handleMouseButton(x, y, mask);
|
||||
paintPointer();
|
||||
}
|
||||
function releaseDrag() {
|
||||
if (dragging) sendPointer(0);
|
||||
dragging = false;
|
||||
dragButton?.setAttribute?.("aria-pressed", "false");
|
||||
}
|
||||
function pointerClick(mask) {
|
||||
releaseDrag();
|
||||
sendPointer(mask);
|
||||
sendPointer(0);
|
||||
}
|
||||
function scrollPointer(deltaX, deltaY) {
|
||||
const rect = pointerGeometry();
|
||||
if (!rect || !rfb || rfb.viewOnly) return;
|
||||
rfb._handleWheel({ clientX:rect.left + pointer.x * rect.width,
|
||||
clientY:rect.top + pointer.y * rect.height, deltaX, deltaY, deltaMode:0,
|
||||
buttons:0, preventDefault(){}, stopPropagation(){} });
|
||||
}
|
||||
function updatePointerControls() {
|
||||
modeButton?.setAttribute?.("aria-pressed", String(trackpad));
|
||||
if (modeButton) modeButton.textContent = trackpad ? "觸控板" : "直接點選";
|
||||
if (pointerHelp) pointerHelp.textContent = trackpad
|
||||
? "滑動移游標・輕點左鍵・雙指捲動／輕點右鍵・拖曳按完再按一次放開"
|
||||
: "點畫面定位並開鍵盤・長按右鍵・精準操作可切換觸控板";
|
||||
for (const button of controls?.querySelectorAll?.("button[data-action]") || []) {
|
||||
button.disabled = !rfb || rfb.viewOnly;
|
||||
}
|
||||
paintPointer();
|
||||
}
|
||||
function showKeyboard() {
|
||||
if (!rfb || rfb.viewOnly) return;
|
||||
rfb.focusOnClick = false;
|
||||
keyboard.focus({ preventScroll: true });
|
||||
keyboard.setSelectionRange(keyboard.value.length, keyboard.value.length);
|
||||
}
|
||||
window.addEventListener("click", event => {
|
||||
const button = event.target.closest?.("#mobile-controls button");
|
||||
if (!button) return;
|
||||
const action = button.dataset.action;
|
||||
if (action === "mode") {
|
||||
releaseDrag(); gesture = null; touchStart = null;
|
||||
trackpad = !trackpad; keyboard.blur(); updatePointerControls(); return;
|
||||
}
|
||||
if (!rfb || rfb.viewOnly) return;
|
||||
if (action === "keyboard") { releaseDrag(); showKeyboard(); return; }
|
||||
if (action === "left") pointerClick(1);
|
||||
if (action === "right") pointerClick(4);
|
||||
if (action === "up") { releaseDrag(); scrollPointer(0, -100); }
|
||||
if (action === "down") { releaseDrag(); scrollPointer(0, 100); }
|
||||
if (action === "drag") {
|
||||
if (!trackpad) { trackpad = true; keyboard.blur(); }
|
||||
dragging = !dragging; sendPointer(dragging ? 1 : 0);
|
||||
dragButton.setAttribute("aria-pressed", String(dragging));
|
||||
updatePointerControls();
|
||||
}
|
||||
});
|
||||
function stopTouch(event) { event.preventDefault(); event.stopImmediatePropagation(); }
|
||||
function touchCenter(touches) {
|
||||
const points = Array.from(touches);
|
||||
return { x:points.reduce((sum,p)=>sum+p.clientX,0)/points.length,
|
||||
y:points.reduce((sum,p)=>sum+p.clientY,0)/points.length };
|
||||
}
|
||||
function startPointerTouch(event) {
|
||||
if (!trackpad || !event.target.closest?.("#screen")) return false;
|
||||
stopTouch(event);
|
||||
if (!rfb || rfb.viewOnly) {
|
||||
window.parent.postMessage({ type:"lazyboy-request-control" }, parentOrigin);
|
||||
return true;
|
||||
}
|
||||
const center = touchCenter(event.touches);
|
||||
if (event.touches.length > 1) releaseDrag();
|
||||
gesture = { last:center, start:center, fingers:event.touches.length,
|
||||
moved:false, time:Date.now() };
|
||||
return true;
|
||||
}
|
||||
function movePointerTouch(event) {
|
||||
if (!trackpad || !gesture) return false;
|
||||
stopTouch(event);
|
||||
if (!rfb || rfb.viewOnly) { gesture = null; return true; }
|
||||
const center = touchCenter(event.touches);
|
||||
if (event.touches.length !== gesture.fingers) {
|
||||
gesture.last = center; gesture.moved = true; return true;
|
||||
}
|
||||
const dx = center.x - gesture.last.x, dy = center.y - gesture.last.y;
|
||||
gesture.last = center;
|
||||
if (Math.hypot(center.x - gesture.start.x, center.y - gesture.start.y) > 5) gesture.moved = true;
|
||||
if (gesture.fingers === 2) { scrollPointer(-dx, -dy); return true; }
|
||||
const rect = pointerGeometry();
|
||||
if (rect && gesture.fingers === 1 && gesture.moved) {
|
||||
pointer.x = Math.max(0, Math.min(1, pointer.x + dx / rect.width));
|
||||
pointer.y = Math.max(0, Math.min(1, pointer.y + dy / rect.height));
|
||||
sendPointer();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
function endPointerTouch(event) {
|
||||
if (!trackpad || !gesture) return false;
|
||||
stopTouch(event);
|
||||
if (event.touches.length) return true;
|
||||
if (!gesture.moved && Date.now() - gesture.time < 450 && !dragging) {
|
||||
if (gesture.fingers === 1) pointerClick(1);
|
||||
if (gesture.fingers === 2) pointerClick(4);
|
||||
}
|
||||
gesture = null;
|
||||
return true;
|
||||
}
|
||||
window.addEventListener("blur", () => { gesture = null; releaseDrag(); });
|
||||
window.addEventListener("pagehide", () => { gesture = null; releaseDrag(); });
|
||||
|
||||
function resetKeyboard() {
|
||||
keyboard.value = sentinel;
|
||||
composing = false;
|
||||
}
|
||||
function mobileKey(key) {
|
||||
if (!rfb || rfb.viewOnly) return;
|
||||
window.parent.postMessage({ type: "lazyboy-mobile-key", key }, parentOrigin);
|
||||
}
|
||||
function commitKeyboard() {
|
||||
if (composing) return;
|
||||
const value = keyboard.value;
|
||||
if (!rfb || rfb.viewOnly) { resetKeyboard(); return; }
|
||||
if (!value) mobileKey("BackSpace");
|
||||
else {
|
||||
const text = value.startsWith(sentinel) ? value.slice(1) : value;
|
||||
if (text) pasteIntoDesktop(text);
|
||||
}
|
||||
resetKeyboard();
|
||||
}
|
||||
resetKeyboard();
|
||||
window.addEventListener("compositionstart", event => {
|
||||
if (event.target === keyboard) composing = true;
|
||||
});
|
||||
window.addEventListener("compositionend", event => {
|
||||
if (event.target !== keyboard) return;
|
||||
composing = false;
|
||||
commitKeyboard();
|
||||
});
|
||||
window.addEventListener("input", event => {
|
||||
if (event.target === keyboard && !event.isComposing) commitKeyboard();
|
||||
});
|
||||
window.addEventListener("beforeinput", event => {
|
||||
if (event.target !== keyboard || composing || event.isComposing) return;
|
||||
if (event.inputType === "insertLineBreak" || event.inputType === "insertParagraph") {
|
||||
event.preventDefault(); mobileKey("Return"); resetKeyboard();
|
||||
}
|
||||
});
|
||||
// Focus synchronously in the user's tap, before iOS loses activation.
|
||||
// VNC pixels cannot reveal whether the remote target is a text field.
|
||||
window.addEventListener("touchstart", event => {
|
||||
if (event.target === keyboard) return;
|
||||
if (startPointerTouch(event)) { touchStart = null; return; }
|
||||
const point = event.touches[0];
|
||||
const rect = pointerGeometry();
|
||||
if (point && rect && event.target.closest?.("#screen")) {
|
||||
pointer.x = Math.max(0, Math.min(1, (point.clientX - rect.left) / rect.width));
|
||||
pointer.y = Math.max(0, Math.min(1, (point.clientY - rect.top) / rect.height));
|
||||
}
|
||||
touchStart = event.touches.length === 1 && event.target.closest?.("#screen")
|
||||
? { x: point.clientX, y: point.clientY } : null;
|
||||
if (rfb) rfb.focusOnClick = false;
|
||||
}, { passive: false, capture: true });
|
||||
window.addEventListener("touchmove", event => {
|
||||
if (movePointerTouch(event)) return;
|
||||
const point = event.touches[0];
|
||||
if (touchStart && (!point || event.touches.length !== 1 ||
|
||||
Math.hypot(point.clientX - touchStart.x, point.clientY - touchStart.y) > 10)) touchStart = null;
|
||||
}, { passive: false, capture: true });
|
||||
window.addEventListener("touchcancel", () => { touchStart = null; gesture = null; releaseDrag(); }, true);
|
||||
window.addEventListener("touchend", event => {
|
||||
if (endPointerTouch(event)) return;
|
||||
const tapped = touchStart && event.touches.length === 0;
|
||||
touchStart = null;
|
||||
if (!tapped || !rfb || rfb.viewOnly) return;
|
||||
keyboard.focus({ preventScroll: true });
|
||||
keyboard.setSelectionRange(keyboard.value.length, keyboard.value.length);
|
||||
}, { passive: false, capture: true });
|
||||
|
||||
function pasteIntoDesktop(text) {
|
||||
if (!rfb || rfb.viewOnly || !text) return;
|
||||
|
|
@ -70,7 +305,10 @@
|
|||
|
||||
function applyViewOnly(value) {
|
||||
if (!rfb) return;
|
||||
if (value) { releaseDrag(); gesture = null; }
|
||||
rfb.viewOnly = Boolean(value);
|
||||
updatePointerControls();
|
||||
if (rfb.viewOnly) { keyboard.blur(); resetKeyboard(); }
|
||||
}
|
||||
|
||||
function connect() {
|
||||
|
|
@ -82,12 +320,17 @@
|
|||
rfb.clipViewport = false;
|
||||
rfb.background = "#0f172a";
|
||||
pinTaskbar();
|
||||
updatePointerControls();
|
||||
rfb.addEventListener("connect", () => {
|
||||
pinTaskbar();
|
||||
updatePointerControls();
|
||||
try { rfb.focus(); } catch (_) {}
|
||||
window.parent.postMessage({ type: "lazyboy-desktop-ready" }, parentOrigin);
|
||||
});
|
||||
rfb.addEventListener("disconnect", (event) => {
|
||||
releaseDrag(); gesture = null;
|
||||
applyViewOnly(true);
|
||||
keyboard.blur(); resetKeyboard();
|
||||
window.parent.postMessage({ type: "lazyboy-desktop-lost" }, parentOrigin);
|
||||
const clean = event && event.detail && event.detail.clean;
|
||||
setStatus(clean ? "Disconnected — retrying" : "Desktop connection lost — retrying");
|
||||
|
|
@ -105,6 +348,16 @@
|
|||
// Capture before noVNC forwards the shortcut; otherwise stale remote
|
||||
// clipboard contents can be pasted once before the local text arrives.
|
||||
window.addEventListener("keydown", async (event) => {
|
||||
if (event.target === keyboard) {
|
||||
if (event.isComposing || composing || event.keyCode === 229) return;
|
||||
const keys = { Enter: "Return", Backspace: "BackSpace", Tab: "Tab", Escape: "Escape",
|
||||
ArrowLeft: "Left", ArrowRight: "Right", ArrowUp: "Up", ArrowDown: "Down" };
|
||||
if (keys[event.key]) {
|
||||
event.preventDefault(); event.stopImmediatePropagation();
|
||||
mobileKey(keys[event.key]); resetKeyboard();
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (rfb && !rfb.viewOnly && event.metaKey && event.code === "KeyC") {
|
||||
event.preventDefault(); event.stopImmediatePropagation();
|
||||
window.parent.postMessage({type:"lazyboy-copy-request"},parentOrigin); return;
|
||||
|
|
@ -121,9 +374,10 @@
|
|||
}
|
||||
}, true);
|
||||
connect();
|
||||
window.addEventListener("resize", pinTaskbar);
|
||||
window.addEventListener("pointerdown", () => {
|
||||
try { if (rfb) rfb.focus(); } catch (_) {}
|
||||
window.addEventListener("resize", () => { pinTaskbar(); paintPointer(); });
|
||||
window.addEventListener("pointerdown", (event) => {
|
||||
if (event.target === keyboard || event.target.closest?.("#mobile-controls")) return;
|
||||
try { if (rfb && event.pointerType !== "touch") { rfb.focusOnClick = true; rfb.focus(); } } catch (_) {}
|
||||
if (rfb && rfb.viewOnly) {
|
||||
window.parent.postMessage({ type: "lazyboy-request-control" }, parentOrigin);
|
||||
}
|
||||
|
|
@ -134,6 +388,7 @@
|
|||
if (!text) return;
|
||||
event.preventDefault();
|
||||
pasteIntoDesktop(text);
|
||||
if (event.target === keyboard) resetKeyboard();
|
||||
});
|
||||
window.addEventListener("message", (event) => {
|
||||
if (event.origin !== parentOrigin || event.source !== window.parent) return;
|
||||
|
|
@ -150,5 +405,20 @@
|
|||
<body>
|
||||
<div id="status" hidden>Loading desktop…</div>
|
||||
<div id="screen"></div>
|
||||
<div id="mobile-cursor" aria-hidden="true"></div>
|
||||
<nav id="mobile-controls" aria-label="遠端滑鼠">
|
||||
<div class="buttons">
|
||||
<button id="pointer-mode" data-action="mode" aria-pressed="false" title="切換直接點選與觸控板">直接點選</button>
|
||||
<button data-action="left">左鍵</button>
|
||||
<button data-action="right">右鍵</button>
|
||||
<button id="pointer-drag" data-action="drag" aria-pressed="false">拖曳</button>
|
||||
<button data-action="up" aria-label="向上捲動">↑</button>
|
||||
<button data-action="down" aria-label="向下捲動">↓</button>
|
||||
<button data-action="keyboard">鍵盤</button>
|
||||
</div>
|
||||
<p id="pointer-help"></p>
|
||||
</nav>
|
||||
<textarea id="mobile-keyboard" aria-label="Remote desktop keyboard" tabindex="-1"
|
||||
autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"></textarea>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ Waiting is a tool call, never a reply. Ending your turn with \"waiting for X\" s
|
|||
|
||||
Multi-step tasks and taught skills: you are done only when the playbook's check passes (for example the course shows completed, the form shows a confirmation). Do not stop with a status sentence in the middle; keep calling tools until the check passes or you are truly blocked, then say exactly why. Never repeat an earlier reply word for word; describe the current screen.
|
||||
|
||||
Never ask for passwords, codes, or tokens in chat. At a login wall: call list_accounts, then use_saved_login {accountId} when a saved account matches. If none matches, or 2FA/CAPTCHA appears, call request_takeover with site and why so the human signs in on YOUR screen. Recurring work uses create_schedule (five-field cron, Asia/Taipei unless told otherwise).
|
||||
Never ask for passwords, codes, or tokens in chat. At a login wall: call list_accounts, then use_saved_login {accountId} when a saved account matches. For a simple Cloudflare connection-check checkbox, first take computer_observe and use connection_check once with coordinates from that screenshot. Check the returned page content before continuing; disappearance of the checkbox alone is not success. Never reload repeatedly or restart the browser to retry. For other CAPTCHA, 2FA, an unsuccessful connection check, or no matching saved login, call request_takeover with site and why so the human signs in on YOUR screen. Recurring work uses create_schedule (five-field cron, Asia/Taipei unless told otherwise).
|
||||
|
||||
computer_act examples (native windows only):
|
||||
- {\"kind\":\"click\",\"element\":1}
|
||||
|
|
@ -449,6 +449,7 @@ async fn execute_run(
|
|||
last_click_key: std::sync::Mutex::new(None),
|
||||
click_misses: std::sync::Mutex::new(0),
|
||||
takeover_requested: std::sync::Mutex::new(false),
|
||||
connection_check_attempted: std::sync::Mutex::new(false),
|
||||
pool: state.pool().clone(),
|
||||
memory: state.memory.clone(),
|
||||
actor: actor.clone(),
|
||||
|
|
@ -954,6 +955,7 @@ async fn execute_run(
|
|||
| "open_path"
|
||||
| "launch_app"
|
||||
| "browser"
|
||||
| "connection_check"
|
||||
| "wait"
|
||||
| "use_saved_login"
|
||||
| "request_takeover"
|
||||
|
|
@ -2053,6 +2055,7 @@ fn tool_needs_sandbox(name: &str) -> bool {
|
|||
| "computer_observe"
|
||||
| "computer_act"
|
||||
| "browser"
|
||||
| "connection_check"
|
||||
| "open_path"
|
||||
| "launch_app"
|
||||
| "wait"
|
||||
|
|
@ -2067,6 +2070,7 @@ fn tool_needs_gui(name: &str) -> bool {
|
|||
"computer_observe"
|
||||
| "computer_act"
|
||||
| "browser"
|
||||
| "connection_check"
|
||||
| "open_path"
|
||||
| "launch_app"
|
||||
| "wait"
|
||||
|
|
@ -2178,6 +2182,7 @@ fn describe_step(name: &str, args: &Value) -> String {
|
|||
let get = |key: &str| args.get(key).and_then(Value::as_str);
|
||||
let detail = match name {
|
||||
"computer_observe" => "看畫面".to_string(),
|
||||
"connection_check" => "嘗試連線驗證並確認結果".to_string(),
|
||||
"computer_act" => args
|
||||
.get("actions")
|
||||
.and_then(Value::as_array)
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ pub struct ToolCtx {
|
|||
pub last_click_key: Mutex<Option<String>>,
|
||||
pub click_misses: Mutex<u32>,
|
||||
pub takeover_requested: Mutex<bool>,
|
||||
pub connection_check_attempted: Mutex<bool>,
|
||||
pub pool: PgPool,
|
||||
pub memory: MemoryService,
|
||||
pub actor: Actor,
|
||||
|
|
@ -171,6 +172,13 @@ pub fn tool_definitions(memory_enabled: bool) -> Vec<ToolDefinition> {
|
|||
"required":["seconds"]
|
||||
}),
|
||||
},
|
||||
ToolDefinition {
|
||||
name: "connection_check".into(),
|
||||
description: "Try one normal click on a visible Cloudflare connection-check checkbox. First use computer_observe and locate the checkbox in the latest screenshot; pass its screen x/y. Only for a connection-check page, never image/audio puzzles, passwords or 2FA. Waits up to 15 seconds, returns a fresh observation, and requests human takeover if still blocked. Limited to one attempt per run. Never claim success until the requested page content is visible.".into(),
|
||||
parameters: json!({"type":"object","properties":{
|
||||
"x":{"type":"integer","minimum":0},"y":{"type":"integer","minimum":0}
|
||||
},"required":["x","y"]}),
|
||||
},
|
||||
ToolDefinition {
|
||||
name: "request_takeover".into(),
|
||||
description: "Ask the user to take over the live screen for passwords, 2FA, CAPTCHA, or a login wall when no saved account fits. Never ask them to paste secrets in chat. Prefer use_saved_login when list_accounts has a matching site.".into(),
|
||||
|
|
@ -191,7 +199,7 @@ pub fn tool_definitions(memory_enabled: bool) -> Vec<ToolDefinition> {
|
|||
},
|
||||
ToolDefinition {
|
||||
name: "use_saved_login".into(),
|
||||
description: "Fill the current Chromium login form with a saved account. Pass only accountId from list_accounts. The password never appears in chat. If 2FA or CAPTCHA appears afterwards, call request_takeover.".into(),
|
||||
description: "Fill the current Chromium login form with a saved account. Pass only accountId from list_accounts. The password never appears in chat. For a simple Cloudflare checkbox use computer_observe then connection_check once; for other CAPTCHA or 2FA call request_takeover.".into(),
|
||||
parameters: json!({
|
||||
"type":"object",
|
||||
"properties":{"accountId":{"type":"string"}},
|
||||
|
|
@ -285,6 +293,7 @@ pub async fn dispatch(ctx: &ToolCtx, name: &str, args: &Value) -> ToolOutcome {
|
|||
"computer_act" => act(ctx, args).await,
|
||||
"wait" => wait_then_observe(ctx, args).await,
|
||||
"browser" => browser(ctx, args).await,
|
||||
"connection_check" => connection_check(ctx, args).await,
|
||||
"shell" => shell(ctx, args).await,
|
||||
"list_files" => list_files(ctx, args).await,
|
||||
"read_file" => read_file(ctx, args).await,
|
||||
|
|
@ -636,6 +645,72 @@ async fn cdp_call(ctx: &ToolCtx, request: Value) -> CdpPage {
|
|||
}
|
||||
}
|
||||
|
||||
fn is_connection_check(page: &CdpPage) -> bool {
|
||||
let text = format!("{} {}", page.title, page.text).to_lowercase();
|
||||
page.ok && (text.contains("需要確認您的連線是安全") || (text.contains("cloudflare") && [
|
||||
"verify you are human", "verifying you are human", "checking your browser",
|
||||
"checking if the site connection is secure", "needs to review the security",
|
||||
"驗證您是人類", "验证您是人类", "確認您的連線是安全", "確認您的人類身分",
|
||||
].iter().any(|marker| text.contains(marker))))
|
||||
}
|
||||
|
||||
fn connection_takeover(ctx: &ToolCtx, reason: &str) -> ToolOutcome {
|
||||
*ctx.takeover_requested.lock().unwrap() = true;
|
||||
ToolOutcome {
|
||||
text: reason.into(), image: None, pause: true,
|
||||
blocks: login_blocks(&json!({"reason":reason,"site":"網站連線驗證",
|
||||
"why":"完成驗證後，繼續原本的瀏覽任務。"})),
|
||||
}
|
||||
}
|
||||
|
||||
async fn connection_check(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
||||
if let Some(blocked) = vision_guard(ctx) { return blocked; }
|
||||
let (Some(x), Some(y)) = (args.get("x").and_then(Value::as_u64), args.get("y").and_then(Value::as_u64)) else {
|
||||
return text_outcome("Take a fresh computer_observe and provide the visible checkbox's non-negative screen x/y.");
|
||||
};
|
||||
if x > i32::MAX as u64 || y > i32::MAX as u64 {
|
||||
return text_outcome("Checkbox coordinates are out of range.");
|
||||
}
|
||||
let page = cdp_call(ctx, json!({"action":"snapshot","ensure":false})).await;
|
||||
if !is_connection_check(&page) {
|
||||
return text_outcome("No supported Cloudflare connection-check page was confirmed. Re-observe; use request_takeover for other CAPTCHA, login or 2FA. No click was sent.");
|
||||
}
|
||||
let already_attempted = {
|
||||
let mut attempted = ctx.connection_check_attempted.lock().unwrap();
|
||||
let previous = *attempted;
|
||||
*attempted = true;
|
||||
previous
|
||||
};
|
||||
if already_attempted {
|
||||
return connection_takeover(ctx, "這次任務已嘗試過連線驗證，請接管完成驗證。");
|
||||
}
|
||||
let actions = match parse_computer_actions(&json!([{"kind":"click","x":x,"y":y}])) {
|
||||
Ok(actions) => actions,
|
||||
Err(error) => return text_outcome(error.to_string()),
|
||||
};
|
||||
let adapter = ctx.adapter();
|
||||
let result = ctx.sandbox.act(&ctx.computer_ref(), ActionRequest {
|
||||
actions, observe:false, settle_ms:350,
|
||||
display:adapter.display.clone(), profile_path:adapter.profile_path.clone(),
|
||||
}, &adapter).await;
|
||||
if result.is_err() {
|
||||
return connection_takeover(ctx, "無法確認驗證點擊是否完成，請接管檢查。");
|
||||
}
|
||||
for _ in 0..3 {
|
||||
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
|
||||
let after = cdp_call(ctx, json!({"action":"snapshot","ensure":false})).await;
|
||||
// Disappearance alone is not proof of success: loading/error pages
|
||||
// can also remove the checkbox. Return evidence for the task check.
|
||||
if after.ok && !is_connection_check(&after) && !after.text.trim().is_empty() {
|
||||
let mut outcome = observe(ctx).await;
|
||||
outcome.text = format!("Connection-check markers disappeared. This is NOT proof of success. Confirm the requested content is actually visible before continuing; if a challenge/error remains, request_takeover.\n{}\n{}",
|
||||
browser_result_text("snapshot", &after), outcome.text);
|
||||
return outcome;
|
||||
}
|
||||
}
|
||||
connection_takeover(ctx, "已嘗試一次驗證並等待，仍無法確認通過。請接管完成驗證，之後繼續原任務。")
|
||||
}
|
||||
|
||||
async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
||||
if let Some(blocked) = vision_guard(ctx) {
|
||||
return blocked;
|
||||
|
|
@ -706,6 +781,9 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|||
return text_outcome(text);
|
||||
}
|
||||
let mut text = browser_result_text(action, &page);
|
||||
if is_connection_check(&page) {
|
||||
text.push_str("\nConnection check detected. Take computer_observe; if a simple verification checkbox is visible, use connection_check with its current screen coordinates once. For other puzzles use request_takeover. The requested content has NOT been retrieved.");
|
||||
}
|
||||
if let Some(seconds) = page.waited_seconds {
|
||||
text.push_str(&format!(
|
||||
" (the control was disabled; waited {seconds:.0}s for it to enable before clicking)"
|
||||
|
|
@ -1314,7 +1392,7 @@ async fn use_saved_login(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|||
.and_then(|value| value.get("submitted").and_then(Value::as_bool))
|
||||
.unwrap_or(false);
|
||||
text_outcome(format!(
|
||||
"Filled {} as {}. {} If a 2FA/CAPTCHA wall is next, call request_takeover.",
|
||||
"Filled {} as {}. {} For a simple Cloudflare checkbox, observe and try connection_check once; for other CAPTCHA or 2FA, call request_takeover.",
|
||||
account.site,
|
||||
account.username,
|
||||
if submitted {
|
||||
|
|
@ -1435,3 +1513,18 @@ fn schedule_state(ctx: &ToolCtx) -> crate::state::AppState {
|
|||
calls: crate::state::CallRegistry::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod connection_check_tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn recognizes_connection_wall_but_not_cloudflare_footer() {
|
||||
for text in ["Cloudflare 驗證您是人類", "Cloudflare Verify you are human", "Dcard 需要確認您的連線是安全的"] {
|
||||
assert!(is_connection_check(&CdpPage { ok:true, text:text.into(), ..Default::default() }));
|
||||
}
|
||||
for text in ["Article text. Protected by Cloudflare", "Sign in with your password", ""] {
|
||||
assert!(!is_connection_check(&CdpPage { ok:true, text:text.into(), ..Default::default() }));
|
||||
}
|
||||
assert!(!is_connection_check(&CdpPage { ok:false, text:"Cloudflare Verify you are human".into(), ..Default::default() }));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ pub fn goal_outcome(reply: &str) -> GoalOutcome {
|
|||
}
|
||||
}
|
||||
|
||||
pub const GOAL_INSTRUCTIONS: &str = "Persistent goal execution: plan the requested work, execute it, and verify each requested outcome. Intermediate progress replies do not finish the run. Preserve completed work and incorporate user steering. End your final reply with a standalone [GOAL_COMPLETE] line only when all outcomes are verified; explain the verification. When required information or human action is missing, explain exactly what is needed and end with a standalone [GOAL_BLOCKED] line. For login, CAPTCHA or 2FA use request_takeover. Never claim completion merely because you planned the work.";
|
||||
pub const GOAL_INSTRUCTIONS: &str = "Persistent goal execution: plan the requested work, execute it, and verify each requested outcome. Intermediate progress replies do not finish the run. Preserve completed work and incorporate user steering. End your final reply with a standalone [GOAL_COMPLETE] line only when all outcomes are verified; explain the verification. When required information or human action is missing, explain exactly what is needed and end with a standalone [GOAL_BLOCKED] line. For a simple Cloudflare connection-check checkbox, observe the current screen and try connection_check once, then verify the requested content. For other CAPTCHA, failed verification, login or 2FA use request_takeover. Never claim completion merely because you planned the work.";
|
||||
pub const GOAL_CONTINUE: &str = "The goal remains active. Continue the plan with tools and verify the outcome. Finish only with a standalone [GOAL_COMPLETE] line after verification, or [GOAL_BLOCKED] when required human input is missing.";
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,93 @@
|
|||
# 架構與流程
|
||||
|
||||
[← 回到 README](../README.zh-TW.md)
|
||||
|
||||
## 一個任務如何完成
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
U["你<br/>文字・附件・語音"] --> W["React 工作空間<br/>對話・電腦・記憶"]
|
||||
W -->|HTTP / WebSocket| A["Rust API + Agent<br/>模型・工具・技能"]
|
||||
A -->|受控工具呼叫| S["Supervisor<br/>資源與生命週期"]
|
||||
S -->|啟動 / 暫停 / 恢復| C["隔離 Linux 電腦<br/>Chromium・XFCE・檔案"]
|
||||
C -->|畫面與事件| W
|
||||
|
||||
Q["排程"] -. Cron .-> A
|
||||
A <--> M[("PostgreSQL<br/>Session・記憶・事件")]
|
||||
V["加密憑證庫"] -. 僅相符 HTTPS 網域 .-> C
|
||||
|
||||
classDef ui fill:#151517,stroke:#3ec5a8,color:#f1f1f2
|
||||
classDef core fill:#101012,stroke:#85858a,color:#f1f1f2
|
||||
classDef secure fill:#221c0e,stroke:#fcd68a,color:#f1f1f2
|
||||
class U,W ui
|
||||
class A,M,Q core
|
||||
class S,C,V secure
|
||||
```
|
||||
|
||||
<div align="center">
|
||||
|
||||
**[開啟可縮放、可搜尋、支援深色模式的互動流程圖 →](./workflow.html)**
|
||||
|
||||
</div>
|
||||
|
||||
主流程之外還有兩個重要迴圈：
|
||||
|
||||
1. **接管迴圈**：使用者接管時，進行中的 run 進入等待；釋放後從目前畫面重新排隊執行。
|
||||
2. **技能迴圈**：示範期間記錄控制項與頁面情境，模型整理成 playbook；往後仍在當下畫面重新尋找元素，不重播舊座標。
|
||||
|
||||
## 系統架構
|
||||
|
||||
LazyBoy 的公開入口只有 API。Supervisor 位於 Compose 內部 control network，不直接對主機開埠；Agent 桌面也不掛載主機 Docker socket。
|
||||
|
||||
```text
|
||||
Browser
|
||||
│ HTTP / WebSocket / authenticated screen proxy
|
||||
▼
|
||||
lazyboy-api (:3101)
|
||||
├── React 靜態前端
|
||||
├── Session / Run / Memory / Schedule / Vault
|
||||
├── Model provider / MCP client
|
||||
└── PostgreSQL + pgvector
|
||||
│
|
||||
│ authenticated internal control API
|
||||
▼
|
||||
lazyboy-supervisor (:7091, internal only)
|
||||
│
|
||||
├── provision / pause / resume / stop
|
||||
├── CPU / memory / PID limits
|
||||
└── isolated computer containers
|
||||
├── Chromium + CDP
|
||||
├── XFCE + AT-SPI
|
||||
├── Xvfb + x11vnc + websockify
|
||||
└── per-computer persisted home
|
||||
```
|
||||
|
||||
### 主要元件
|
||||
|
||||
| 元件 | 職責 |
|
||||
| --- | --- |
|
||||
| `apps/web` | Vite + React 19 三欄工作空間、noVNC、語音與雙語 UI |
|
||||
| `crates/api` | 對外 Axum API、Agent run、Session、排程、記憶、MCP、保險箱 |
|
||||
| `crates/harness` | 模型供應商、憑證解析與語音契約 |
|
||||
| `crates/supervisor` | Docker 電腦生命週期、隔離與資源上限 |
|
||||
| `crates/control` | CDP、AT-SPI、X11 與畫面觀察操作 |
|
||||
| `crates/controld` | 電腦容器內部的 localhost 控制服務 |
|
||||
| `crates/contracts` | 跨 crate 的 Bot、Run、Computer、Voice 資料契約 |
|
||||
| `PostgreSQL` | 對話、run、記憶、排程、憑證與保留政策 |
|
||||
|
||||
### 電腦生命週期
|
||||
|
||||
下圖保留原本的操作流程視角；`WaitingTakeover` 表示控制權交接期間的等待情境，並非獨立的電腦資料庫狀態。
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> Stopped
|
||||
Stopped --> Booting: 第一個需要 GUI 的工具
|
||||
Booting --> Running: ready
|
||||
Running --> WaitingTakeover: 使用者接管 / 2FA
|
||||
WaitingTakeover --> Running: 釋放控制權
|
||||
Running --> Suspended: 閒置 10 分鐘
|
||||
Suspended --> Running: 新任務或畫面心跳
|
||||
Suspended --> Stopped: 休眠約 6 小時
|
||||
Running --> Stopped: 手動停止
|
||||
```
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
# 開發指南
|
||||
|
||||
[← 回到 README](../README.zh-TW.md)
|
||||
|
||||
### 本機開發
|
||||
|
||||
```bash
|
||||
make dev # 準備 .env、PostgreSQL、電腦映像
|
||||
make dev-supervisor # 終端 1：Supervisor :7091
|
||||
make dev-api # 終端 2：API :3101
|
||||
```
|
||||
|
||||
前端熱更新：
|
||||
|
||||
```bash
|
||||
cd apps/web
|
||||
npm install
|
||||
npm run dev # http://127.0.0.1:5173
|
||||
```
|
||||
|
||||
### 檢查與測試
|
||||
|
||||
```bash
|
||||
make fmt
|
||||
make clippy
|
||||
make test
|
||||
|
||||
cd apps/web
|
||||
npm run typecheck
|
||||
npm run build
|
||||
|
||||
cd ../..
|
||||
node --test tests/frontend.test.mjs
|
||||
python3 tests/control.test.py
|
||||
python3 tests/log-rotation.test.py
|
||||
|
||||
# 需要已啟動的 PostgreSQL Compose service
|
||||
docker compose up -d postgres
|
||||
python3 tests/retention.test.py
|
||||
```
|
||||
|
||||
### 專案結構
|
||||
|
||||
```text
|
||||
LazyBoy/
|
||||
├── apps/web/ React 前端與 noVNC 頁面
|
||||
├── crates/
|
||||
│ ├── api/ 對外 API 與 Agent 執行迴圈
|
||||
│ ├── contracts/ 跨元件資料契約
|
||||
│ ├── control/ 瀏覽器與桌面控制
|
||||
│ ├── controld/ 容器內控制服務
|
||||
│ ├── harness/ 模型與語音後端
|
||||
│ ├── sandbox/ API 到 Supervisor 的抽象
|
||||
│ └── supervisor/ Docker 生命週期管理
|
||||
├── image/ API、Supervisor、電腦映像
|
||||
├── migrations/ SQLx PostgreSQL migrations
|
||||
├── tests/ Rust 以外的契約與回歸測試
|
||||
├── scripts/ 環境初始化與執行工具
|
||||
├── docs/hero.html README 首頁視覺原稿
|
||||
├── docs/workflow.html 可互動產品流程圖
|
||||
├── docker-compose.yml 正式堆疊
|
||||
└── Makefile 常用開發與部署指令
|
||||
```
|
||||
|
||||
## 公開發布準備
|
||||
|
||||
對外發布前，可依下列清單補齊授權、貢獻流程與自動化檢查。只有在對應結果已公開時才加入品質徽章。
|
||||
|
||||
若要取得可公開查驗的安全與品質徽章，建議依序完成：
|
||||
|
||||
- [ ] 在公開 GitHub repository 建立正式 mirror；OpenSSF 的公開查驗以 GitHub 為主要整合目標。
|
||||
- [x] Apache 2.0 `LICENSE`
|
||||
- [ ] 補 `SECURITY.md`、`CONTRIBUTING.md` 與行為準則。
|
||||
- [ ] 建立 CI：Rust format / Clippy / tests、前端 typecheck / build / tests。
|
||||
- [ ] 啟用 Dependabot 或 Renovate、CodeQL、secret scanning 與 branch protection。
|
||||
- [ ] 執行並發布 [OpenSSF Scorecard](https://scorecard.dev/) 結果後，再加入 Scorecard 徽章。
|
||||
- [ ] 在 [OpenSSF Best Practices](https://www.bestpractices.dev/) 登記專案、誠實完成 Passing 問卷後，再加入認證徽章。
|
||||
- [ ] 若提供容器映像，再加入 SBOM、簽章與可重現版本發布流程。
|
||||
|
||||
> 不建議現在顯示 CI passing、coverage、OpenSSF 或 Best Practices 徽章：目前 repository 沒有對應的公開結果，徽章會失真或直接顯示 unknown。
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
<!doctype html>
|
||||
<html lang="zh-Hant">
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<title>LazyBoy hero</title>
|
||||
|
|
@ -360,41 +360,41 @@
|
|||
</svg>
|
||||
LazyBoy
|
||||
</div>
|
||||
<div class="pill"><b>本機自託管</b> 每個 Agent 有自己的 Linux 桌面</div>
|
||||
<div class="pill"><b>Self-hosted</b> a Linux desktop per agent</div>
|
||||
</div>
|
||||
<div class="copy">
|
||||
<h1>給 Agent 一台真的電腦</h1>
|
||||
<p class="lead">在瀏覽器裡開機器人。它會自己開網頁、敲指令、學你示範過的流程。<br/>對話與桌面資料留在你的主機，模型連線由你選擇。</p>
|
||||
<h1>Give an agent a real computer</h1>
|
||||
<p class="lead">Open bots in the browser. They browse, run commands, and learn the flows you demonstrate.<br/>Chats and desktops stay on your host. You choose the model.</p>
|
||||
</div>
|
||||
<div class="window">
|
||||
<div class="chrome"><i class="dot r"></i><i class="dot y"></i><i class="dot g"></i></div>
|
||||
<div class="app">
|
||||
<aside class="side">
|
||||
<div class="ws"><i>LB</i> 本機工作區</div>
|
||||
<div class="search"><svg viewBox="0 0 24 24"><circle cx="11" cy="11" r="7"/><path d="M20 20l-3.5-3.5"/></svg>搜尋</div>
|
||||
<div class="label">已釘選</div>
|
||||
<div class="ws"><i>LB</i> Local workspace</div>
|
||||
<div class="search"><svg viewBox="0 0 24 24"><circle cx="11" cy="11" r="7"/><path d="M20 20l-3.5-3.5"/></svg>Search</div>
|
||||
<div class="label">Pinned</div>
|
||||
<div class="bot on">
|
||||
<span class="think-ring">
|
||||
<svg class="blob" viewBox="0 0 64 64"><path fill="#3ec5a8" d="M33 5c11.5 1 24 11 23 26-1 14-11 27-25 28S6 47 7 31 21 4 33 5z"/><ellipse cx="25.5" cy="29" rx="4.2" ry="5" fill="#1b1b22"/><ellipse cx="40.5" cy="29" rx="4.2" ry="5" fill="#1b1b22"/><circle cx="26.8" cy="27.2" r="1.3" fill="#f3f3f6"/><circle cx="41.8" cy="27.2" r="1.3" fill="#f3f3f6"/></svg>
|
||||
</span>
|
||||
<div class="copy"><strong>工程之神</strong><small>正在第 12 頁，等 Next 亮起</small></div>
|
||||
<div class="copy"><strong>Engineering</strong><small>Page 12, waiting for Next</small></div>
|
||||
<span class="time">14:08</span>
|
||||
</div>
|
||||
<div class="label">Agent</div>
|
||||
<div class="bot">
|
||||
<svg class="blob" viewBox="0 0 64 64"><path fill="#f5a03c" d="M30 7c13-1 26 10 25 24-1 15-12 26-25 27S5 46 7 31 17 8 30 7z"/><ellipse cx="25" cy="30" rx="4" ry="4.8" fill="#1b1b22"/><ellipse cx="40" cy="30" rx="4" ry="4.8" fill="#1b1b22"/><circle cx="26.3" cy="28.3" r="1.2" fill="#f3f3f6"/><circle cx="41.3" cy="28.3" r="1.2" fill="#f3f3f6"/></svg>
|
||||
<div class="copy"><strong>早報摘要</strong><small>明天 09:00 會跑</small></div>
|
||||
<div class="copy"><strong>Morning digest</strong><small>Runs tomorrow at 9:00</small></div>
|
||||
</div>
|
||||
<div class="bot">
|
||||
<svg class="blob" viewBox="0 0 64 64"><path fill="#6a6bf5" d="M34 6c12 2 22 12 21 25s-12 26-25 27S6 46 8 32 22 4 34 6z"/><ellipse cx="26" cy="29" rx="4.1" ry="5" fill="#f3f3f6"/><ellipse cx="41" cy="29" rx="4.1" ry="5" fill="#f3f3f6"/><circle cx="24.8" cy="27.4" r="1.2" fill="#1b1b22"/><circle cx="39.8" cy="27.4" r="1.2" fill="#1b1b22"/></svg>
|
||||
<div class="copy"><strong>客服助手</strong><small>已記住回覆語氣</small></div>
|
||||
<div class="copy"><strong>Support</strong><small>Remembered the reply tone</small></div>
|
||||
</div>
|
||||
</aside>
|
||||
<main class="chat">
|
||||
<header class="topbar">
|
||||
<svg class="blob sm" viewBox="0 0 64 64"><path fill="#3ec5a8" d="M33 5c11.5 1 24 11 23 26-1 14-11 27-25 28S6 47 7 31 21 4 33 5z"/><ellipse cx="25.5" cy="29" rx="4.2" ry="5" fill="#1b1b22"/><ellipse cx="40.5" cy="29" rx="4.2" ry="5" fill="#1b1b22"/><circle cx="26.8" cy="27.2" r="1.3" fill="#f3f3f6"/><circle cx="41.8" cy="27.2" r="1.3" fill="#f3f3f6"/></svg>
|
||||
工程之神
|
||||
<span style="color:#85858a;font-size:12px">STAR 訓練 ▾</span>
|
||||
Engineering
|
||||
<span style="color:#85858a;font-size:12px">STAR training ▾</span>
|
||||
<span class="grow"></span>
|
||||
<div class="tools">
|
||||
<i class="on"><svg viewBox="0 0 24 24"><rect x="3" y="5" width="18" height="12" rx="2"/><path d="M8 21h8M12 17v4"/></svg></i>
|
||||
|
|
@ -403,28 +403,28 @@
|
|||
</div>
|
||||
</header>
|
||||
<div class="msgs">
|
||||
<div class="bubble me">幫我把 STAR 那堂課看完，並把測驗交了。</div>
|
||||
<div class="bubble them">電腦已開。現在在第 12 / 24 頁，影片倒數結束我就按 Next。密碼欄我不會記。</div>
|
||||
<div class="bubble me">Finish the STAR course and submit the quiz.</div>
|
||||
<div class="bubble them">Computer is up. Page 12 / 24 — I’ll hit Next when the video ends. I won’t store passwords.</div>
|
||||
</div>
|
||||
<div class="dock">
|
||||
<div class="think">
|
||||
<span class="think-ring"><svg class="blob sm" viewBox="0 0 64 64"><path fill="#3ec5a8" d="M33 5c11.5 1 24 11 23 26-1 14-11 27-25 28S6 47 7 31 21 4 33 5z"/><ellipse cx="25.5" cy="29" rx="4.2" ry="5" fill="#1b1b22"/><ellipse cx="40.5" cy="29" rx="4.2" ry="5" fill="#1b1b22"/><circle cx="26.8" cy="27.2" r="1.3" fill="#f3f3f6"/><circle cx="41.8" cy="27.2" r="1.3" fill="#f3f3f6"/></svg></span>
|
||||
<span class="lbl">正在工作…</span>
|
||||
<span class="lbl">Working…</span>
|
||||
<small>click Next</small>
|
||||
</div>
|
||||
<div class="composer"><div class="plus">＋</div><div class="ph">傳訊息給 工程之神</div><div class="send">↑</div></div>
|
||||
<div class="composer"><div class="plus">＋</div><div class="ph">Message Engineering</div><div class="send">↑</div></div>
|
||||
</div>
|
||||
</main>
|
||||
<aside class="desk">
|
||||
<div class="desk-h">工程之神 的電腦 <i class="dot-run"></i> <small>執行中</small></div>
|
||||
<div class="desk-h">Engineering’s computer <i class="dot-run"></i> <small>Running</small></div>
|
||||
<div class="preview">
|
||||
<div class="desktop">
|
||||
<div class="win">
|
||||
<div class="win-bar"><i></i><i></i><i></i></div>
|
||||
<div class="addr">star.example / course / 12</div>
|
||||
<div class="page-body">
|
||||
<h3>Workplace Safety · 第 12 頁</h3>
|
||||
<p>看完這段說明後按 Next。頁面倒數結束才會解鎖。</p>
|
||||
<h3>Workplace Safety · page 12</h3>
|
||||
<p>Read this, then press Next. It unlocks when the countdown ends.</p>
|
||||
<div class="progress"><b></b></div>
|
||||
<div class="next">Next</div>
|
||||
</div>
|
||||
|
|
@ -432,8 +432,8 @@
|
|||
<div class="panel"></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cap"><span>獨立螢幕</span><span>放大</span></div>
|
||||
<div class="actions"><div class="btn cream">接手操作</div><div class="btn">停止任務</div></div>
|
||||
<div class="cap"><span>This screen</span><span>Enlarge</span></div>
|
||||
<div class="actions"><div class="btn cream">Take over</div><div class="btn">Stop task</div></div>
|
||||
</aside>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
# LazyBoy icon
|
||||
|
||||
Generated using the built-in image_gen tool, with the user-provided icon as a shape/style reference.
|
||||
|
||||
Prompt:
|
||||
Create a polished square app icon for LazyBoy, using the attached icon as shape/style reference: a soft rounded asymmetrical blob head with two simple tilted dark oval eyes, no mouth, resting lazily and tilted slightly. Recolor head to pale mint green #85dfbb with subtle creamy highlights, eyes deep forest #143429. Background is full-bleed dark charcoal-green #10221d square tile, soft premium 3D clay shading and gentle rim light, rounded head occupies 85% of tile, playful relaxed expression. No text, no letters, no extra elements. App-icon artwork only, front view, square 1024x1024, no outer margin or mockup.
|
||||
|
||||
Assets: apps/web/public/lazyboy-icon.png (512px), favicon.png (64px), apple-touch-icon.png (180px).
|
||||
|
||||
White variant (built-in image_gen edit):
|
||||
Edit this LazyBoy app icon. Change ONLY the mint green blob character material to pearl white with neutral light gray shading and white highlights. Remove green tint from the character. Preserve exactly the silhouette, tilt, two dark oval eyes, layout, size, soft premium 3D shading, and existing dark background. No text or new elements. Square app icon.
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
# 部署與操作
|
||||
|
||||
[← 回到 README](../README.zh-TW.md)
|
||||
|
||||
## 安全模型
|
||||
|
||||
目前程式碼包含下列防護：
|
||||
|
||||
- API session token 與 Supervisor token 分離。
|
||||
- Supervisor 只在內部 Compose network，並使用 `no-new-privileges`、唯讀 root filesystem 與 capability drop。
|
||||
- 每台電腦有 CPU、RAM、PID 上限；預設 2 CPU、2 GB、2048 PID。
|
||||
- API 預設只綁定 `127.0.0.1:3101`。
|
||||
- 憑證以獨立 `LAZYBOY_VAULT_KEY` 加密，輪替登入 token 時不應更換此 key。
|
||||
- 已保存登入只接受 HTTPS、精確或合法子網域匹配，不對相似惡意網域填入。
|
||||
- Markdown 連結限制為 HTTP(S)、`mailto:`、`tel:` 與頁內錨點。
|
||||
- 日誌有大小與檔案數上限；診斷資料有可設定的保留週期。
|
||||
|
||||
部署注意事項：
|
||||
|
||||
1. 對區網或網際網路開放前，先放在 HTTPS reverse proxy 後方，並設定 `LAZYBOY_SECURE_COOKIE=true`。
|
||||
2. 不要把 Supervisor `:7091` 對外發布，也不要將 Docker socket 掛進 Agent 電腦。
|
||||
3. `LAZYBOY_APP_TOKEN`、`SANDBOX_SUPERVISOR_TOKEN`、`LAZYBOY_VAULT_KEY` 必須使用不同的高熵值。
|
||||
4. 模型仍可能看見任務所需的網頁內容與截圖；密碼、token 與高敏感資料不要放進提示詞。
|
||||
5. 簡單的 Cloudflare 連線驗證可嘗試一次正常點擊；未通過、其他 CAPTCHA 與 2FA 交由使用者接管。
|
||||
|
||||
## 硬體與資源
|
||||
|
||||
以下為容量規劃起點，並非效能測試結果；實際需求取決於同時啟動的電腦與瀏覽器工作量。
|
||||
|
||||
| 項目 | 最低可執行 | 建議 |
|
||||
| --- | --- | --- |
|
||||
| CPU | 4 核 | 8 核以上 |
|
||||
| RAM | 8 GB，單台電腦 | 16 GB 以上 |
|
||||
| 磁碟 | 約 15 GB | Docker 至少保留 30 GB |
|
||||
| GPU | 不需要 | 模型預設走外部 API |
|
||||
| 作業系統 | macOS / Linux | Linux 可選配 LXCFS 顯示容器內 cgroup 配額 |
|
||||
|
||||
每台電腦的預設限制可由 `LAZYBOY_COMPUTER_CPUS`、`LAZYBOY_COMPUTER_MEMORY_MB`、`LAZYBOY_COMPUTER_PIDS` 調整。瀏覽器分頁的心跳會維持熱機；沒有工作且約 10 分鐘無人觀看時暫停，持續休眠約 6 小時後停止。
|
||||
|
||||
## 設定
|
||||
|
||||
`make env` 會以 `.env.example` 為基礎建立 `.env`，並保留既有金鑰。
|
||||
|
||||
| 變數 | 用途 | 預設 |
|
||||
| --- | --- | --- |
|
||||
| `XAI_API_KEY` | xAI 模型 | 空 |
|
||||
| `OPENCODE_GO_API_KEY` | OpenCode Go 模型 | 空 |
|
||||
| `OPENAI_API_KEY` | OpenAI 相容端點 | 空 |
|
||||
| `LAZYBOY_APP_TOKEN` | Web 登入 token | 必填 |
|
||||
| `SANDBOX_SUPERVISOR_TOKEN` | API ↔ Supervisor 驗證 | 必填 |
|
||||
| `LAZYBOY_VAULT_KEY` | 憑證庫加密 key | 必填且必須保持穩定 |
|
||||
| `LAZYBOY_BIND_IP` | 主機監聽位址 | `127.0.0.1` |
|
||||
| `LAZYBOY_SECURE_COOKIE` | HTTPS-only cookie | `false` |
|
||||
| `LAZYBOY_COMPUTER_CPUS` | 每台電腦 CPU | `2` |
|
||||
| `LAZYBOY_COMPUTER_MEMORY_MB` | 每台電腦記憶體 | `2048` |
|
||||
| `LAZYBOY_COMPUTER_PIDS` | 每台電腦 PID 上限 | `2048` |
|
||||
| `LAZYBOY_COMPUTER_SUDO` | 容器內免密碼 sudo；重建桌面容器後生效 | `false` |
|
||||
| `LAZYBOY_MEMORY_ENABLED` | 長期記憶 | `true` |
|
||||
|
||||
完整清單與保留政策請見 [`.env.example`](../.env.example)。
|
||||
|
||||
## 網站連線驗證
|
||||
|
||||
AI 遇到可辨識的 Cloudflare 連線驗證頁時，會先取得最新桌面截圖，再使用
|
||||
`connection_check` 嘗試點擊可見的驗證框一次。工具沿用現有瀏覽器，最多進行三次
|
||||
間隔五秒的結果檢查（不含瀏覽器工具本身的執行時間）。驗證頁消失後仍須確認目標
|
||||
內容已載入；未通過則顯示接管提示。其他圖形／音訊驗證和 2FA 仍交由使用者處理。
|
||||
|
||||
測試：更新 API 服務後，開新任務要求 AI「開啟 Dcard 並讀取指定文章；若有簡單的
|
||||
連線驗證框，使用 connection_check 嘗試一次」。可在任務狀態看到「嘗試連線驗證並
|
||||
確認結果」。此功能不保證網站放行，也不會偽造驗證結果。
|
||||
|
||||
## 容器內使用 sudo
|
||||
|
||||
在 `.env` 設定 `LAZYBOY_COMPUTER_SUDO=true`，執行
|
||||
`docker compose up -d --no-deps supervisor`，再於網頁選擇「重新啟動電腦」。
|
||||
這會重建桌面容器，讓 `lazyboy` 使用者能執行免密碼 `sudo`；可用
|
||||
`sudo -n id -u` 檢查，預期輸出 `0`。既有容器只暫停／恢復不會套用此設定。
|
||||
重新啟動前請儲存工作；家目錄保留，容器系統層自行安裝的套件需重新安裝。
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 268 KiB After Width: | Height: | Size: 242 KiB |
|
|
@ -3,7 +3,7 @@ FROM node:22-bookworm-slim AS web
|
|||
WORKDIR /src/apps/web
|
||||
COPY apps/web/package.json apps/web/package-lock.json ./
|
||||
RUN --mount=type=cache,target=/root/.npm npm ci --no-audit --no-fund
|
||||
COPY apps/web/index.html apps/web/tsconfig.json apps/web/vite.config.ts ./
|
||||
COPY apps/web/index.html apps/web/vnc.html apps/web/tsconfig.json apps/web/vite.config.ts ./
|
||||
COPY apps/web/public public
|
||||
COPY apps/web/src src
|
||||
RUN npm run build \
|
||||
|
|
|
|||
|
|
@ -144,3 +144,100 @@ test('microphone frames are batched instead of sent one websocket message per bl
|
|||
audio.queueCapture(block);
|
||||
assert.deepEqual(sent,[CAPTURE_CHUNK_SAMPLES*2]);
|
||||
});
|
||||
|
||||
function mobileViewer(){
|
||||
const source=fs.readFileSync('apps/web/vnc.html','utf8').match(/<script type="module">([\s\S]*?)<\/script>/)[1].replace(/import RFB[^;]+;/,'');
|
||||
const handlers={},options={},sent=[];
|
||||
const keyboard={value:'',focused:false,focus(){this.focused=true},blur(){this.focused=false},setSelectionRange(){}};
|
||||
const pointerEvents=[];
|
||||
const canvas={style:{},closest:selector=>selector==='#screen'?{}:null,getBoundingClientRect:()=>({left:0,top:0,width:300,height:200})};
|
||||
const elements={};
|
||||
const element=id=>elements[id]||(elements[id]={style:{},attributes:{},setAttribute(k,v){this.attributes[k]=v},querySelectorAll(){return []}});
|
||||
let rfb;
|
||||
class RFB{constructor(){rfb=this}addEventListener(){}focus(){keyboard.focused=false}
|
||||
_handleMouseMove(x,y){pointerEvents.push({type:'move',x,y})}
|
||||
_handleMouseButton(x,y,mask){pointerEvents.push({type:'button',x,y,mask})}
|
||||
_handleWheel(event){pointerEvents.push({type:'wheel',dx:event.deltaX,dy:event.deltaY})}
|
||||
}
|
||||
const window={location:{pathname:'/vnc.html',protocol:'http:',host:'localhost',origin:'http://localhost',hash:''},parent:{postMessage:x=>sent.push(x)},addEventListener:(n,f,o)=>{handlers[n]=f;options[n]=o}};
|
||||
vm.runInNewContext(source,{window,document:{location:{href:'http://localhost/vnc.html?view_only=false'},getElementById:id=>id==='mobile-keyboard'?keyboard:element(id),querySelector:selector=>selector==='#screen canvas'?canvas:null},navigator:{},RFB,setTimeout(){},clearTimeout(){}});
|
||||
return {handlers,options,sent,keyboard,canvas,rfb,window,pointerEvents,elements};
|
||||
}
|
||||
test('mobile tap focuses editable input before noVNC stops touch propagation; drag and multi-touch do not',()=>{
|
||||
const {handlers:h,options,keyboard,canvas,rfb}=mobileViewer();
|
||||
for(const name of ['touchstart','touchmove','touchend'])assert.equal(options[name].capture,true);
|
||||
const point={clientX:20,clientY:30};
|
||||
h.touchstart({target:canvas,touches:[point]});h.touchend({touches:[]});
|
||||
assert.equal(keyboard.focused,true);assert.equal(rfb.focusOnClick,false);
|
||||
keyboard.blur();h.touchstart({target:canvas,touches:[point]});h.touchmove({touches:[{clientX:50,clientY:30}]});h.touchend({touches:[]});assert.equal(keyboard.focused,false);
|
||||
h.touchstart({target:canvas,touches:[point,point]});h.touchend({touches:[]});assert.equal(keyboard.focused,false);
|
||||
rfb.viewOnly=true;h.touchstart({target:canvas,touches:[point]});h.touchend({touches:[]});assert.equal(keyboard.focused,false);
|
||||
});
|
||||
test('mobile IME sends committed Chinese once and supports input-only backspace and Enter',()=>{
|
||||
const {handlers:h,keyboard,sent}=mobileViewer();
|
||||
sent.length=0;
|
||||
h.compositionstart({target:keyboard});keyboard.value+='中文';
|
||||
h.input({target:keyboard,isComposing:true});assert.equal(sent.length,0);
|
||||
h.compositionend({target:keyboard});h.input({target:keyboard,isComposing:false});
|
||||
assert.equal(sent.length,1);assert.equal(sent[0].text,'中文');
|
||||
keyboard.value='';h.input({target:keyboard,inputType:'deleteContentBackward'});
|
||||
let prevented=false;h.beforeinput({target:keyboard,inputType:'insertLineBreak',preventDefault(){prevented=true}});
|
||||
assert.equal(prevented,true);
|
||||
assert.deepEqual(sent.map(x=>x.text||x.key),['中文','BackSpace','Return']);
|
||||
});
|
||||
test('mobile keyboard is cleared and blurred when control is released; foreign messages cannot change control',()=>{
|
||||
const {handlers:h,keyboard,sent,window}=mobileViewer();
|
||||
keyboard.focus();keyboard.value+='secret';
|
||||
h.message({origin:window.location.origin,source:{},data:{type:'lazyboy-view-only',viewOnly:true}});
|
||||
assert.equal(keyboard.focused,true);
|
||||
h.message({origin:window.location.origin,source:window.parent,data:{type:'lazyboy-view-only',viewOnly:true}});
|
||||
assert.equal(keyboard.focused,false);assert.equal(keyboard.value,'\u200b');
|
||||
sent.length=0;keyboard.value+='blocked';h.input({target:keyboard});assert.equal(sent.length,0);
|
||||
});
|
||||
|
||||
function pointerButton(viewer,action){
|
||||
viewer.handlers.click({target:{closest:()=>({dataset:{action}})}});
|
||||
}
|
||||
function pointerTouch(viewer,name,points){
|
||||
let prevented=false,stopped=false;
|
||||
viewer.handlers[name]({target:viewer.canvas,touches:points.map(([clientX,clientY])=>({clientX,clientY})),preventDefault(){prevented=true},stopImmediatePropagation(){stopped=true}});
|
||||
return prevented&&stopped;
|
||||
}
|
||||
test('trackpad moves relative to cursor, clamps edges and taps without opening keyboard',()=>{
|
||||
const v=mobileViewer();pointerButton(v,'mode');
|
||||
assert.equal(pointerTouch(v,'touchstart',[[30,30]]),true);
|
||||
pointerTouch(v,'touchmove',[[60,40]]);pointerTouch(v,'touchend',[]);
|
||||
assert.equal(v.pointerEvents.length,1);assert.equal(v.pointerEvents[0].x,180);assert.ok(Math.abs(v.pointerEvents[0].y-110)<.001);
|
||||
assert.equal(v.keyboard.focused,false);
|
||||
pointerTouch(v,'touchstart',[[10,10]]);pointerTouch(v,'touchend',[]);
|
||||
assert.deepEqual(v.pointerEvents.slice(-2).map(e=>e.mask),[1,0]);
|
||||
pointerTouch(v,'touchstart',[[10,10]]);pointerTouch(v,'touchmove',[[1000,-1000]]);pointerTouch(v,'touchend',[]);
|
||||
assert.equal(v.pointerEvents.at(-1).x,299);assert.equal(v.pointerEvents.at(-1).y,0);
|
||||
pointerButton(v,'keyboard');assert.equal(v.keyboard.focused,true);
|
||||
});
|
||||
test('two-finger trackpad scroll and right tap do not also left click',()=>{
|
||||
const v=mobileViewer();pointerButton(v,'mode');
|
||||
pointerTouch(v,'touchstart',[[20,20],[50,20]]);pointerTouch(v,'touchmove',[[20,80],[50,80]]);pointerTouch(v,'touchend',[]);
|
||||
assert.deepEqual(v.pointerEvents,[{type:'wheel',dx:-0,dy:-60}]);
|
||||
pointerTouch(v,'touchstart',[[20,20],[50,20]]);pointerTouch(v,'touchend',[[20,20]]);pointerTouch(v,'touchend',[]);
|
||||
assert.deepEqual(v.pointerEvents.slice(-2).map(e=>e.mask),[4,0]);
|
||||
});
|
||||
test('drag is released on cancellation, mode switch, blur and control handoff',()=>{
|
||||
for(const release of ['cancel','mode','blur','handoff']){
|
||||
const v=mobileViewer();pointerButton(v,'drag');
|
||||
assert.equal(v.pointerEvents.at(-1).mask,1);
|
||||
if(release==='cancel')v.handlers.touchcancel();
|
||||
if(release==='mode')pointerButton(v,'mode');
|
||||
if(release==='blur')v.handlers.blur();
|
||||
if(release==='handoff')v.handlers.message({origin:v.window.location.origin,source:v.window.parent,data:{type:'lazyboy-view-only',viewOnly:true}});
|
||||
assert.equal(v.pointerEvents.at(-1).mask,0,release);
|
||||
assert.equal(v.elements['pointer-drag'].attributes['aria-pressed'],'false');
|
||||
}
|
||||
});
|
||||
test('view-only pointer controls never send mouse input and trackpad asks for control',()=>{
|
||||
const v=mobileViewer();pointerButton(v,'mode');v.rfb.viewOnly=true;
|
||||
for(const action of ['left','right','drag','up','down'])pointerButton(v,action);
|
||||
pointerTouch(v,'touchstart',[[20,20]]);pointerTouch(v,'touchmove',[[50,50]]);pointerTouch(v,'touchend',[]);
|
||||
assert.equal(v.pointerEvents.length,0);
|
||||
assert.equal(v.sent.at(-1).type,'lazyboy-request-control');
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue