feat/cua-driver-poc #7
|
|
@ -33,6 +33,8 @@ LAZYBOY_MEMORY_BYTE_BUDGET=6000
|
||||||
LAZYBOY_EVENT_RETENTION_DAYS=30
|
LAZYBOY_EVENT_RETENTION_DAYS=30
|
||||||
LAZYBOY_CHECKPOINT_RETENTION_DAYS=7
|
LAZYBOY_CHECKPOINT_RETENTION_DAYS=7
|
||||||
LAZYBOY_RUN_RETENTION_DAYS=90
|
LAZYBOY_RUN_RETENTION_DAYS=90
|
||||||
|
# Per-run trace shown when hovering the thinking avatar (round, tool results, errors).
|
||||||
|
LAZYBOY_RUN_ACTIVITY_RETENTION_DAYS=7
|
||||||
LAZYBOY_RECORDING_RETENTION_DAYS=30
|
LAZYBOY_RECORDING_RETENTION_DAYS=30
|
||||||
LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS=90
|
LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS=90
|
||||||
LAZYBOY_DB_WARN_MB=1024
|
LAZYBOY_DB_WARN_MB=1024
|
||||||
|
|
|
||||||
|
|
@ -1889,7 +1889,6 @@ name = "lazyboy-api"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
"async-trait",
|
|
||||||
"axum",
|
"axum",
|
||||||
"base64 0.22.1",
|
"base64 0.22.1",
|
||||||
"cap-std",
|
"cap-std",
|
||||||
|
|
@ -1900,9 +1899,7 @@ dependencies = [
|
||||||
"fastembed",
|
"fastembed",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"hex",
|
"hex",
|
||||||
"hmac",
|
|
||||||
"http",
|
"http",
|
||||||
"http-body-util",
|
|
||||||
"lazyboy-contracts",
|
"lazyboy-contracts",
|
||||||
"lazyboy-control",
|
"lazyboy-control",
|
||||||
"lazyboy-harness",
|
"lazyboy-harness",
|
||||||
|
|
@ -1915,7 +1912,6 @@ dependencies = [
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha2",
|
"sha2",
|
||||||
"sqlx",
|
"sqlx",
|
||||||
"thiserror",
|
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-tungstenite 0.26.2",
|
"tokio-tungstenite 0.26.2",
|
||||||
"tower-http",
|
"tower-http",
|
||||||
|
|
@ -1958,9 +1954,7 @@ dependencies = [
|
||||||
"base64 0.22.1",
|
"base64 0.22.1",
|
||||||
"lazyboy-contracts",
|
"lazyboy-contracts",
|
||||||
"lazyboy-control",
|
"lazyboy-control",
|
||||||
"serde",
|
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"thiserror",
|
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
|
|
@ -1979,7 +1973,6 @@ dependencies = [
|
||||||
"rig-core",
|
"rig-core",
|
||||||
"rustls",
|
"rustls",
|
||||||
"rustls-native-certs",
|
"rustls-native-certs",
|
||||||
"serde",
|
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
|
@ -1992,39 +1985,30 @@ version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"base64 0.22.1",
|
"base64 0.22.1",
|
||||||
"chrono",
|
|
||||||
"hex",
|
|
||||||
"lazyboy-contracts",
|
"lazyboy-contracts",
|
||||||
"lazyboy-control",
|
"lazyboy-control",
|
||||||
"reqwest 0.12.28",
|
"reqwest 0.12.28",
|
||||||
"serde",
|
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha2",
|
|
||||||
"tokio",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lazyboy-supervisor"
|
name = "lazyboy-supervisor"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
|
||||||
"axum",
|
"axum",
|
||||||
"base64 0.22.1",
|
"base64 0.22.1",
|
||||||
"bollard",
|
"bollard",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"hex",
|
"hex",
|
||||||
"hmac",
|
"hmac",
|
||||||
"lazyboy-contracts",
|
|
||||||
"lazyboy-control",
|
"lazyboy-control",
|
||||||
"reqwest 0.12.28",
|
"reqwest 0.12.28",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha2",
|
"sha2",
|
||||||
"thiserror",
|
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
"uuid",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
|
||||||
18
Cargo.toml
18
Cargo.toml
|
|
@ -12,10 +12,28 @@ members = [
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
rust-version = "1.98"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
publish = false
|
publish = false
|
||||||
|
|
||||||
|
# --- Lint policy -----------------------------------------------------------
|
||||||
|
# A single place decides how the whole workspace is linted; every crate opts in
|
||||||
|
# with `[lints] workspace = true`. `make lint` runs clippy with -D warnings, so a
|
||||||
|
# new warning has to be fixed (or relaxed at the call site with a reason) before
|
||||||
|
# it reaches main. Thresholds live in clippy.toml, which cargo-clippy only reads
|
||||||
|
# from the directory it was started in - keep running it at the workspace root.
|
||||||
|
[workspace.lints.rust]
|
||||||
|
# Safety-relevant code (the vault, the docker socket) has to stay explicit.
|
||||||
|
unsafe_code = "warn"
|
||||||
|
unused_must_use = "deny"
|
||||||
|
unused_crate_dependencies = "warn"
|
||||||
|
|
||||||
|
[workspace.lints.clippy]
|
||||||
|
# Debug leftovers and placeholder implementations must not reach a branch.
|
||||||
|
dbg_macro = "warn"
|
||||||
|
todo = "warn"
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
lto = true
|
lto = true
|
||||||
codegen-units = 1
|
codegen-units = 1
|
||||||
|
|
|
||||||
39
Makefile
39
Makefile
|
|
@ -11,9 +11,9 @@ DATA_DIR ?= ./data
|
||||||
|
|
||||||
.PHONY: help env env-force \
|
.PHONY: help env env-force \
|
||||||
up logs ps health down purge \
|
up logs ps health down purge \
|
||||||
computer postgres postgres-down \
|
computer postgres postgres-down pg-collation \
|
||||||
build build-api build-supervisor build-controld \
|
build build-api build-supervisor build-controld \
|
||||||
fmt clippy test clean \
|
fmt fmt-check clippy lint audit test clean \
|
||||||
web \
|
web \
|
||||||
dev dev-supervisor dev-api
|
dev dev-supervisor dev-api
|
||||||
|
|
||||||
|
|
@ -33,6 +33,7 @@ help: ## Show this help
|
||||||
@echo " make computer Build the heavy Debian desktop image (lazyboy/computer:local)"
|
@echo " make computer Build the heavy Debian desktop image (lazyboy/computer:local)"
|
||||||
@echo " make postgres Start only postgres (127.0.0.1:5434) and wait for ready"
|
@echo " make postgres Start only postgres (127.0.0.1:5434) and wait for ready"
|
||||||
@echo " make postgres-down Stop postgres"
|
@echo " make postgres-down Stop postgres"
|
||||||
|
@echo " make pg-collation Repair a Postgres collation version mismatch (see docs)"
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo " Local dev (postgres in Docker, Rust services on the host):"
|
@echo " Local dev (postgres in Docker, Rust services on the host):"
|
||||||
@echo " make dev Prep .env + postgres + computer image, then print run steps"
|
@echo " make dev Prep .env + postgres + computer image, then print run steps"
|
||||||
|
|
@ -43,8 +44,11 @@ help: ## Show this help
|
||||||
@echo " make build cargo build --release (whole workspace)"
|
@echo " make build cargo build --release (whole workspace)"
|
||||||
@echo " make build-api cargo build --release -p lazyboy-api"
|
@echo " make build-api cargo build --release -p lazyboy-api"
|
||||||
@echo " make fmt cargo fmt --all"
|
@echo " make fmt cargo fmt --all"
|
||||||
@echo " make clippy cargo clippy (deny warnings)"
|
@echo " make fmt-check Report files rustfmt would change (legacy drift exists)"
|
||||||
@echo " make test cargo test --workspace"
|
@echo " make clippy cargo clippy (deny warnings, reads clippy.toml)"
|
||||||
|
@echo " make lint The Rust gate: clippy with -D warnings"
|
||||||
|
@echo " make audit cargo deny: RustSec advisories, licenses, sources"
|
||||||
|
@echo " make test cargo test --workspace (DB tests need: make postgres)"
|
||||||
@echo " make web Build the frontend in $(WEB_DIR) (needs node/npm)"
|
@echo " make web Build the frontend in $(WEB_DIR) (needs node/npm)"
|
||||||
@echo " make clean cargo clean"
|
@echo " make clean cargo clean"
|
||||||
@echo ""
|
@echo ""
|
||||||
|
|
@ -97,6 +101,14 @@ postgres: ## Start only postgres and wait until it is ready
|
||||||
postgres-down: ## Stop postgres
|
postgres-down: ## Stop postgres
|
||||||
$(COMPOSE) down postgres
|
$(COMPOSE) down postgres
|
||||||
|
|
||||||
|
# A pgvector image rebuilt on another glibc leaves every database recording the old
|
||||||
|
# collation version; Postgres then refuses CREATE DATABASE and `cargo test` hangs on
|
||||||
|
# PoolTimedOut. Stop the api first, then reindex + refresh each database in place.
|
||||||
|
pg-collation: ## Repair a Postgres collation version mismatch after an image update
|
||||||
|
$(COMPOSE) exec -T postgres psql -X -v ON_ERROR_STOP=1 -U lazyboy -d template1 -c "REINDEX DATABASE template1;" -c "ALTER DATABASE template1 REFRESH COLLATION VERSION;"
|
||||||
|
$(COMPOSE) exec -T postgres psql -X -v ON_ERROR_STOP=1 -U lazyboy -d postgres -c "REINDEX DATABASE postgres;" -c "ALTER DATABASE postgres REFRESH COLLATION VERSION;"
|
||||||
|
$(COMPOSE) exec -T postgres psql -X -v ON_ERROR_STOP=1 -U lazyboy -d lazyboy -c "REINDEX DATABASE lazyboy;" -c "ALTER DATABASE lazyboy REFRESH COLLATION VERSION;"
|
||||||
|
|
||||||
# --- Rust / web ------------------------------------------------------------
|
# --- Rust / web ------------------------------------------------------------
|
||||||
|
|
||||||
build: ## Release build of the whole workspace
|
build: ## Release build of the whole workspace
|
||||||
|
|
@ -114,8 +126,23 @@ build-controld: ## Release build of the controld binary
|
||||||
fmt: ## Format all Rust code
|
fmt: ## Format all Rust code
|
||||||
cargo fmt --all
|
cargo fmt --all
|
||||||
|
|
||||||
clippy: ## Run clippy, denying warnings
|
fmt-check: ## Check formatting without touching files
|
||||||
cargo clippy --all-targets -- -D warnings
|
cargo fmt --all --check
|
||||||
|
|
||||||
|
# Lint policy lives in [workspace.lints] in the root Cargo.toml; the thresholds
|
||||||
|
# (e.g. too-many-arguments-threshold) live in clippy.toml, which cargo-clippy
|
||||||
|
# only reads from the directory it is started in - keep running this at the root.
|
||||||
|
clippy: ## Run clippy over the workspace, denying warnings
|
||||||
|
cargo clippy --workspace --all-targets -- -D warnings
|
||||||
|
|
||||||
|
lint: clippy ## The Rust quality gate used by CI
|
||||||
|
|
||||||
|
# cargo-deny is a separate CLI: cargo install --locked cargo-deny
|
||||||
|
audit: ## Supply-chain check (RustSec advisories, licenses, dependency sources)
|
||||||
|
@command -v cargo-deny >/dev/null 2>&1 || { \
|
||||||
|
echo "cargo-deny is not installed: cargo install --locked cargo-deny"; exit 1; }
|
||||||
|
@echo "(advisories need the RustSec advisory DB, cloned on first run)"
|
||||||
|
cargo deny check
|
||||||
|
|
||||||
test: ## Run the test suite
|
test: ## Run the test suite
|
||||||
cargo test --workspace
|
cargo test --workspace
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -459,5 +459,53 @@ export const en: { [K in keyof typeof zhTW]: string } = {
|
||||||
resumeProgress: "{turns}/{limit} turns",
|
resumeProgress: "{turns}/{limit} turns",
|
||||||
resumeContinue: "Keep going",
|
resumeContinue: "Keep going",
|
||||||
resumeStop: "Stop here",
|
resumeStop: "Stop here",
|
||||||
resumeSent: "Keep going and finish it.",
|
resumeSent: "Keep going and finish it.", monitorTitle: "Live log",
|
||||||
|
monitorQueued: "Queued",
|
||||||
|
monitorTurnOf: "turn {turn}/{limit}",
|
||||||
|
monitorTurn: "turn {turn}",
|
||||||
|
monitorElapsed: "up {time}",
|
||||||
|
monitorNow: "Now: {step}",
|
||||||
|
monitorEmpty: "Nothing yet — the trail fills in as I work.",
|
||||||
|
monitorLoadFailed: "Couldn’t load the log",
|
||||||
|
monitorCopy: "Copy log",
|
||||||
|
monitorCopied: "Copied",
|
||||||
|
monitorPinHint: "Folds away when the mouse leaves",
|
||||||
|
monitorPinned: "Pinned — press Esc to close",
|
||||||
|
monitorClose: "Close",
|
||||||
|
monitorKindModel: "Model",
|
||||||
|
monitorKindTool: "Action",
|
||||||
|
monitorKindRetry: "Retry",
|
||||||
|
monitorKindNotice: "Note",
|
||||||
|
monitorKindRun: "Status",
|
||||||
|
monitorRunStarted: "Started: {task}",
|
||||||
|
monitorRunCompleted: "Done in {turns} turns",
|
||||||
|
monitorRunFailed: "Failed",
|
||||||
|
monitorRunWaiting: "Waiting for you: {reason}",
|
||||||
|
monitorRunPaused: "Paused: {reason}",
|
||||||
|
monitorRunRetry: "Re-queued",
|
||||||
|
monitorModelTurn: "turn {turn}, thought for {time}",
|
||||||
|
monitorRetryAttempt: "attempt {attempt} failed, retrying",
|
||||||
|
monitorGaveUp: "gave up",
|
||||||
|
monitorStatusOk: "ok",
|
||||||
|
monitorStatusError: "failed",
|
||||||
|
monitorStatusTimedOut: "timed out",
|
||||||
|
monitorStatusPaused: "needs you",
|
||||||
|
errorTitleInterrupted: "Interrupted",
|
||||||
|
errorTitleToolTimeout: "Action timed out",
|
||||||
|
errorTitleModelKey: "Model API key rejected",
|
||||||
|
errorTitleModelQuota: "Rate limited or out of quota",
|
||||||
|
errorTitleModelUnknown: "Model not found",
|
||||||
|
errorTitleModelTimeout: "Model took too long",
|
||||||
|
errorTitleNetwork: "Can’t reach the model",
|
||||||
|
errorTitleComputerGone: "Computer is gone",
|
||||||
|
errorTitleLeaseLost: "Work picked up elsewhere",
|
||||||
|
errorTitleUnknown: "Something broke",
|
||||||
|
errorChip: "Task failed",
|
||||||
|
errorRetry: "Retry",
|
||||||
|
errorRetryHint: "Continue where it stopped",
|
||||||
|
errorOpenSettings: "Model settings",
|
||||||
|
errorDetails: "Show the log",
|
||||||
|
errorRetried: "Back in the queue",
|
||||||
|
errorRetryFailed: "Can’t re-queue right now — another run may still be busy.",
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -154,5 +154,53 @@ export const zhTW = {
|
||||||
resumeProgress: "{turns}/{limit} 輪",
|
resumeProgress: "{turns}/{limit} 輪",
|
||||||
resumeContinue: "繼續",
|
resumeContinue: "繼續",
|
||||||
resumeStop: "就到這裡",
|
resumeStop: "就到這裡",
|
||||||
resumeSent: "繼續，把它做完。",
|
resumeSent: "繼續，把它做完。", monitorTitle: "即時記錄",
|
||||||
|
monitorQueued: "排隊中",
|
||||||
|
monitorTurnOf: "第 {turn}/{limit} 輪",
|
||||||
|
monitorTurn: "第 {turn} 輪",
|
||||||
|
monitorElapsed: "已運行 {time}",
|
||||||
|
monitorNow: "現在：{step}",
|
||||||
|
monitorEmpty: "還沒有記錄。開始動手後會一條條出現。",
|
||||||
|
monitorLoadFailed: "記錄載入失敗",
|
||||||
|
monitorCopy: "複製記錄",
|
||||||
|
monitorCopied: "已複製",
|
||||||
|
monitorPinHint: "滑鼠移開會自動收起來",
|
||||||
|
monitorPinned: "已釘住（按 Esc 關閉）",
|
||||||
|
monitorClose: "關閉",
|
||||||
|
monitorKindModel: "模型",
|
||||||
|
monitorKindTool: "動作",
|
||||||
|
monitorKindRetry: "重試",
|
||||||
|
monitorKindNotice: "提醒",
|
||||||
|
monitorKindRun: "狀態",
|
||||||
|
monitorRunStarted: "開始工作：{task}",
|
||||||
|
monitorRunCompleted: "完成了（{turns} 輪）",
|
||||||
|
monitorRunFailed: "失敗了",
|
||||||
|
monitorRunWaiting: "停下來等你：{reason}",
|
||||||
|
monitorRunPaused: "暫停：{reason}",
|
||||||
|
monitorRunRetry: "重新排進佇列",
|
||||||
|
monitorModelTurn: "第 {turn} 輪，想了 {time}",
|
||||||
|
monitorRetryAttempt: "第 {attempt} 次嘗試失敗，正在重試",
|
||||||
|
monitorGaveUp: "放棄重試",
|
||||||
|
monitorStatusOk: "成功",
|
||||||
|
monitorStatusError: "失敗",
|
||||||
|
monitorStatusTimedOut: "逾時",
|
||||||
|
monitorStatusPaused: "要你接手",
|
||||||
|
errorTitleInterrupted: "被打斷",
|
||||||
|
errorTitleToolTimeout: "動作逾時",
|
||||||
|
errorTitleModelKey: "模型金鑰不被接受",
|
||||||
|
errorTitleModelQuota: "限流或額度不足",
|
||||||
|
errorTitleModelUnknown: "模型名稱找不到",
|
||||||
|
errorTitleModelTimeout: "模型太久沒回話",
|
||||||
|
errorTitleNetwork: "連不到模型",
|
||||||
|
errorTitleComputerGone: "電腦不見了",
|
||||||
|
errorTitleLeaseLost: "工作被另一邊接手",
|
||||||
|
errorTitleUnknown: "出錯了",
|
||||||
|
errorChip: "任務失敗",
|
||||||
|
errorRetry: "重試",
|
||||||
|
errorRetryHint: "從中斷的地方接著做",
|
||||||
|
errorOpenSettings: "開模型設定",
|
||||||
|
errorDetails: "看詳細記錄",
|
||||||
|
errorRetried: "已重新排進佇列",
|
||||||
|
errorRetryFailed: "現在重試排不進去，可能還有別的工作在跑。",
|
||||||
|
|
||||||
} as const;
|
} as const;
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@ import "./avatar.css";
|
||||||
import "./chat.css";
|
import "./chat.css";
|
||||||
import "./computer.css";
|
import "./computer.css";
|
||||||
import "./schedule.css";
|
import "./schedule.css";
|
||||||
|
import "./monitor.css";
|
||||||
import "./call.css";
|
import "./call.css";
|
||||||
import "./responsive.css";
|
import "./responsive.css";
|
||||||
import { App } from "./App";
|
import { App } from "./App";
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,39 @@
|
||||||
|
/* The live run bubble. It floats above whatever opened it, so it never has to
|
||||||
|
fit inside the composer or a message bubble. */
|
||||||
|
.run-probe{position:relative;display:inline-flex;align-items:center;border-radius:10px;outline:none}
|
||||||
|
.run-probe:focus-visible{box-shadow:0 0 0 2px var(--focus)}
|
||||||
|
.run-probe.inactive{cursor:default}
|
||||||
|
.run-monitor{position:fixed;z-index:60;display:flex;flex-direction:column;gap:8px;max-height:min(64vh,460px);padding:12px 13px 10px;border:1px solid var(--border);border-radius:14px;background:var(--menu);box-shadow:0 18px 44px rgba(0,0,0,.55);color:var(--ink);font-size:12px;line-height:1.45;text-align:left;cursor:default}
|
||||||
|
.run-monitor-head{display:flex;flex-wrap:wrap;align-items:baseline;justify-content:space-between;gap:8px}
|
||||||
|
.run-monitor-head b{font-size:13px}
|
||||||
|
.run-monitor-turn{color:var(--accent);font-variant-numeric:tabular-nums}
|
||||||
|
.run-monitor-step{color:var(--muted);overflow-wrap:anywhere}
|
||||||
|
.run-monitor-step.bad{color:var(--error-fg)}
|
||||||
|
.run-monitor-list{display:flex;flex-direction:column;gap:5px;min-height:44px;max-height:min(38vh,300px);overflow-y:auto;padding-right:2px;border-top:1px solid var(--line);border-bottom:1px solid var(--line);padding-top:7px;padding-bottom:7px}
|
||||||
|
.run-monitor-empty{color:var(--faint)}
|
||||||
|
.run-line{display:grid;grid-template-columns:58px 40px 1fr;gap:7px;align-items:baseline}
|
||||||
|
.run-line-clock{color:var(--faint);font-variant-numeric:tabular-nums}
|
||||||
|
.run-line-kind{color:var(--muted)}
|
||||||
|
.run-line-text{overflow-wrap:anywhere}
|
||||||
|
.run-line.is-error .run-line-text,.run-line.is-timed_out .run-line-text{color:var(--error-fg)}
|
||||||
|
.run-line.is-paused .run-line-text{color:var(--warn-fg)}
|
||||||
|
.run-line.kind-run .run-line-text{color:var(--ink)}
|
||||||
|
.run-line.kind-retry .run-line-text{color:var(--warn-fg)}
|
||||||
|
.run-monitor-error{display:flex;flex-direction:column;gap:4px;padding:9px 10px;border:1px solid var(--error-border);border-radius:11px;background:var(--error-bg);color:var(--error-fg)}
|
||||||
|
.run-monitor-error code{display:block;max-height:110px;overflow:auto;white-space:pre-wrap;overflow-wrap:anywhere;color:var(--error-fg);font-size:11px;opacity:.85}
|
||||||
|
.run-monitor-foot{display:flex;align-items:center;gap:9px}
|
||||||
|
.run-monitor-foot button{display:inline-flex;align-items:center;gap:5px;padding:5px 9px;border:1px solid var(--border);border-radius:9px;background:var(--elevated);color:var(--ink);font-size:12px;cursor:pointer}
|
||||||
|
.run-monitor-foot button:hover{background:var(--surface-hover)}
|
||||||
|
.run-monitor-foot small{flex:1;color:var(--faint)}
|
||||||
|
.run-monitor-close{padding:3px 8px !important;color:var(--muted)}
|
||||||
|
|
||||||
|
/* Failure card in the chat: what broke, and the one thing to do next. */
|
||||||
|
.error-chip{border-color:var(--error-border);background:var(--error-bg)}
|
||||||
|
.error-chip .login-label{display:flex;align-items:center;gap:7px;color:var(--error-fg);font-weight:600}
|
||||||
|
.error-chip .error-body{color:var(--ink);overflow-wrap:anywhere}
|
||||||
|
.error-chip .error-actions{display:flex;flex-wrap:wrap;align-items:center;gap:8px;margin-top:2px}
|
||||||
|
.error-chip .error-actions button{padding:6px 12px;border:1px solid var(--border);border-radius:9px;background:var(--elevated);color:var(--ink);font-size:13px;cursor:pointer}
|
||||||
|
.error-chip .error-actions button.primary{border-color:transparent;background:var(--cream);color:var(--on-cream)}
|
||||||
|
.error-chip .error-actions button:disabled{opacity:.55;cursor:default}
|
||||||
|
.error-chip .error-probe{margin-left:auto;color:var(--error-fg);cursor:pointer}
|
||||||
|
.error-chip .error-said{color:var(--error-fg);font-size:12px}
|
||||||
|
|
@ -0,0 +1,372 @@
|
||||||
|
import { useCallback, useEffect, useRef, useState, type KeyboardEvent as ReactKeyboardEvent, type ReactNode } from "react";
|
||||||
|
import { api } from "./api";
|
||||||
|
import { t, type MessageKey } from "./i18n";
|
||||||
|
import type { RunActivity, RunActivityEntry } from "./types";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The observability bubble: hover (or tap, or focus) an avatar to watch the run
|
||||||
|
* that avatar is doing, turn by turn. The trail is polled rather than streamed
|
||||||
|
* because it is only interesting while somebody is looking at it.
|
||||||
|
*/
|
||||||
|
const POLL_MS = 1500;
|
||||||
|
const CLOSE_MS = 300;
|
||||||
|
const KEEP = 200;
|
||||||
|
const PANEL_WIDTH = 380;
|
||||||
|
const DONE = ["completed", "failed", "cancelled"];
|
||||||
|
|
||||||
|
export type RunActionId = "retry" | "screen" | "settings";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Which buttons actually help for a failure code, best first. A bad API key is
|
||||||
|
* fixed in settings and a lost computer is checked on screen — retrying alone
|
||||||
|
* would only burn another turn on the same wall.
|
||||||
|
*/
|
||||||
|
const ACTIONS: Record<string, RunActionId[]> = {
|
||||||
|
interrupted: ["screen", "retry"],
|
||||||
|
tool_timeout: ["screen", "retry"],
|
||||||
|
model_key: ["settings", "retry"],
|
||||||
|
model_unknown: ["settings", "retry"],
|
||||||
|
model_quota: ["retry", "settings"],
|
||||||
|
model_timeout: ["retry", "settings"],
|
||||||
|
network: ["retry", "settings"],
|
||||||
|
computer_gone: ["screen", "retry"],
|
||||||
|
lease_lost: ["retry"],
|
||||||
|
unknown: ["retry"],
|
||||||
|
};
|
||||||
|
|
||||||
|
const TITLES: Record<string, MessageKey> = {
|
||||||
|
interrupted: "errorTitleInterrupted",
|
||||||
|
tool_timeout: "errorTitleToolTimeout",
|
||||||
|
model_key: "errorTitleModelKey",
|
||||||
|
model_quota: "errorTitleModelQuota",
|
||||||
|
model_unknown: "errorTitleModelUnknown",
|
||||||
|
model_timeout: "errorTitleModelTimeout",
|
||||||
|
network: "errorTitleNetwork",
|
||||||
|
computer_gone: "errorTitleComputerGone",
|
||||||
|
lease_lost: "errorTitleLeaseLost",
|
||||||
|
unknown: "errorTitleUnknown",
|
||||||
|
};
|
||||||
|
|
||||||
|
export function errorActions(code?: string | null): RunActionId[] {
|
||||||
|
return ACTIONS[code || ""] || ACTIONS.unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Short label for a failure code; the long sentence lives in the message body. */
|
||||||
|
export function errorTitle(code?: string | null): string {
|
||||||
|
return t(TITLES[code || ""] || TITLES.unknown);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 1:05 / 1:02:05 — how long the run has been at it. */
|
||||||
|
export function formatElapsed(ms: number): string {
|
||||||
|
const seconds = Number.isFinite(ms) ? Math.max(0, Math.floor(ms / 1000)) : 0;
|
||||||
|
const pad = (value: number) => String(value).padStart(2, "0");
|
||||||
|
const hours = Math.floor(seconds / 3600);
|
||||||
|
const minutes = Math.floor((seconds % 3600) / 60);
|
||||||
|
return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds % 60)}` : `${minutes}:${pad(seconds % 60)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 340ms / 6.4s / 1:05 — durations inside the trail stay glanceable. */
|
||||||
|
export function shortDuration(ms?: number | null): string {
|
||||||
|
if (typeof ms !== "number" || !Number.isFinite(ms) || ms < 0) return "";
|
||||||
|
if (ms < 1000) return `${Math.round(ms)}ms`;
|
||||||
|
if (ms < 10_000) return `${(ms / 1000).toFixed(1)}s`;
|
||||||
|
return formatElapsed(ms);
|
||||||
|
}
|
||||||
|
|
||||||
|
function kindLabel(kind: string): string {
|
||||||
|
if (kind === "model") return t("monitorKindModel");
|
||||||
|
if (kind === "tool") return t("monitorKindTool");
|
||||||
|
if (kind === "retry") return t("monitorKindRetry");
|
||||||
|
if (kind === "notice") return t("monitorKindNotice");
|
||||||
|
return t("monitorKindRun");
|
||||||
|
}
|
||||||
|
|
||||||
|
function statusLabel(status?: string | null): string {
|
||||||
|
if (status === "ok") return t("monitorStatusOk");
|
||||||
|
if (status === "timed_out") return t("monitorStatusTimedOut");
|
||||||
|
if (status === "paused") return t("monitorStatusPaused");
|
||||||
|
if (status === "error") return t("monitorStatusError");
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One trail line as plain text — what the panel shows and what gets copied. */
|
||||||
|
export function trailText(entry: RunActivityEntry): string {
|
||||||
|
if (entry.kind === "run") {
|
||||||
|
if (entry.event === "started") return t("monitorRunStarted", { task: entry.task || "" });
|
||||||
|
if (entry.event === "completed") return t("monitorRunCompleted", { turns: entry.turns ?? 0 });
|
||||||
|
if (entry.event === "failed") return t("monitorRunFailed");
|
||||||
|
if (entry.event === "waiting_input") return t("monitorRunWaiting", { reason: entry.reason || "" });
|
||||||
|
if (entry.event === "paused") return t("monitorRunPaused", { reason: entry.reason || "" });
|
||||||
|
if (entry.event === "retry") return t("monitorRunRetry");
|
||||||
|
return entry.text || entry.reason || t("monitorKindRun");
|
||||||
|
}
|
||||||
|
if (entry.kind === "model") {
|
||||||
|
const head = t("monitorModelTurn", { turn: entry.turn ?? 0, time: shortDuration(entry.elapsedMs) });
|
||||||
|
return entry.text ? `${head} — ${entry.text}` : head;
|
||||||
|
}
|
||||||
|
if (entry.kind === "tool") {
|
||||||
|
const parts = [entry.step || entry.name || t("monitorKindTool")];
|
||||||
|
const status = statusLabel(entry.status);
|
||||||
|
if (status) parts.push(status);
|
||||||
|
const time = shortDuration(entry.elapsedMs);
|
||||||
|
if (time) parts.push(time);
|
||||||
|
const detail = entry.snippet ? ` — ${entry.snippet}` : "";
|
||||||
|
return `${parts.join(" · ")}${detail}`;
|
||||||
|
}
|
||||||
|
if (entry.kind === "retry") {
|
||||||
|
const head = `${t("monitorRetryAttempt", { attempt: entry.attempt ?? 0 })}${entry.gaveUp ? ` · ${t("monitorGaveUp")}` : ""}`;
|
||||||
|
return entry.error ? `${head} — ${entry.error}` : head;
|
||||||
|
}
|
||||||
|
return entry.text || t("monitorKindNotice");
|
||||||
|
}
|
||||||
|
|
||||||
|
function clockOf(createdAt: string): string {
|
||||||
|
const date = new Date(createdAt);
|
||||||
|
if (Number.isNaN(date.getTime())) return "";
|
||||||
|
const pad = (value: number) => String(value).padStart(2, "0");
|
||||||
|
return `${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function logDump(snapshot: RunActivity | null, entries: RunActivityEntry[]): string {
|
||||||
|
const turn = snapshot?.turnLimit
|
||||||
|
? t("monitorTurnOf", { turn: snapshot.turn ?? 0, limit: snapshot.turnLimit })
|
||||||
|
: t("monitorTurn", { turn: snapshot?.turn ?? 0 });
|
||||||
|
const head = [
|
||||||
|
snapshot?.runId ?? "",
|
||||||
|
snapshot?.status ?? "",
|
||||||
|
snapshot?.turn != null ? turn : "",
|
||||||
|
snapshot?.step ?? "",
|
||||||
|
snapshot?.elapsedMs != null ? t("monitorElapsed", { time: formatElapsed(snapshot.elapsedMs) }) : "",
|
||||||
|
].filter(Boolean).join(" · ");
|
||||||
|
const trail = entries.map(entry => `${clockOf(entry.createdAt)} ${trailText(entry)}`);
|
||||||
|
const error = snapshot?.error ? `\n[${snapshot.error.code}] ${snapshot.error.raw}` : "";
|
||||||
|
return [head, ...trail].join("\n") + error;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function copyLog(text: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
await navigator.clipboard.writeText(text);
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wraps an avatar (or any small trigger) in a live readout of the run behind
|
||||||
|
* it. Hover opens it and it closes ~300ms after the pointer leaves; a click
|
||||||
|
* pins it so the panel survives reaching for the copy button; Esc lets go.
|
||||||
|
*/
|
||||||
|
export function RunProbe({ runId, align = "start", label, children }: { runId?: string | null; align?: "start" | "end"; label?: string; children: ReactNode }) {
|
||||||
|
const [open, setOpen] = useState(false);
|
||||||
|
const [pinned, setPinned] = useState(false);
|
||||||
|
const [snapshot, setSnapshot] = useState<RunActivity | null>(null);
|
||||||
|
const [entries, setEntries] = useState<RunActivityEntry[]>([]);
|
||||||
|
const [stale, setStale] = useState(false);
|
||||||
|
const [copied, setCopied] = useState<boolean | null>(null);
|
||||||
|
const [box, setBox] = useState<{ left: number; bottom: number } | null>(null);
|
||||||
|
const anchorRef = useRef<HTMLSpanElement | null>(null);
|
||||||
|
const listRef = useRef<HTMLSpanElement | null>(null);
|
||||||
|
const lastId = useRef(0);
|
||||||
|
const followTail = useRef(true);
|
||||||
|
const linger = useRef<number | null>(null);
|
||||||
|
const pinnedRef = useRef(false);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
pinnedRef.current = pinned;
|
||||||
|
}, [pinned]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
lastId.current = 0;
|
||||||
|
setSnapshot(null);
|
||||||
|
setEntries([]);
|
||||||
|
setStale(false);
|
||||||
|
}, [runId]);
|
||||||
|
|
||||||
|
const place = useCallback(() => {
|
||||||
|
const node = anchorRef.current;
|
||||||
|
if (!node) return;
|
||||||
|
const rect = node.getBoundingClientRect();
|
||||||
|
const width = Math.min(PANEL_WIDTH, window.innerWidth - 24);
|
||||||
|
const edge = align === "end" ? rect.right - width : rect.left;
|
||||||
|
setBox({
|
||||||
|
left: Math.max(12, Math.min(edge, window.innerWidth - 12 - width)),
|
||||||
|
bottom: Math.max(12, window.innerHeight - rect.top + 8),
|
||||||
|
});
|
||||||
|
}, [align]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!open || !runId) return;
|
||||||
|
let stopped = false;
|
||||||
|
let timer = 0;
|
||||||
|
const load = async () => {
|
||||||
|
try {
|
||||||
|
const after = lastId.current > 0 ? `?after=${lastId.current}` : "";
|
||||||
|
const next = await api<RunActivity>(`/api/runs/${runId}/activity${after}`);
|
||||||
|
if (stopped) return;
|
||||||
|
setStale(false);
|
||||||
|
setSnapshot(next);
|
||||||
|
setEntries(current => {
|
||||||
|
if (lastId.current <= 0) return next.activity.slice(-KEEP);
|
||||||
|
const seen = new Set(current.map(entry => entry.id));
|
||||||
|
return [...current, ...next.activity.filter(entry => !seen.has(entry.id))].slice(-KEEP);
|
||||||
|
});
|
||||||
|
for (const entry of next.activity) lastId.current = Math.max(lastId.current, entry.id);
|
||||||
|
if (DONE.includes(next.status)) window.clearInterval(timer);
|
||||||
|
} catch {
|
||||||
|
if (!stopped) setStale(true);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
timer = window.setInterval(() => void load(), POLL_MS);
|
||||||
|
void load();
|
||||||
|
return () => {
|
||||||
|
stopped = true;
|
||||||
|
window.clearInterval(timer);
|
||||||
|
};
|
||||||
|
}, [open, runId]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!open) return;
|
||||||
|
place();
|
||||||
|
window.addEventListener("resize", place);
|
||||||
|
return () => window.removeEventListener("resize", place);
|
||||||
|
}, [open, place]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const list = listRef.current;
|
||||||
|
if (open && followTail.current && list) list.scrollTop = list.scrollHeight;
|
||||||
|
}, [entries, open]);
|
||||||
|
|
||||||
|
const hoverIn = useCallback(() => {
|
||||||
|
if (linger.current) window.clearTimeout(linger.current);
|
||||||
|
linger.current = null;
|
||||||
|
setOpen(true);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const hoverOut = useCallback(() => {
|
||||||
|
if (pinnedRef.current) return;
|
||||||
|
if (linger.current) window.clearTimeout(linger.current);
|
||||||
|
linger.current = window.setTimeout(() => setOpen(false), CLOSE_MS);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(
|
||||||
|
() => () => {
|
||||||
|
if (linger.current) window.clearTimeout(linger.current);
|
||||||
|
},
|
||||||
|
[],
|
||||||
|
);
|
||||||
|
|
||||||
|
const onKeyDown = (event: ReactKeyboardEvent<HTMLSpanElement>) => {
|
||||||
|
if (event.key === "Escape") {
|
||||||
|
setPinned(false);
|
||||||
|
setOpen(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (event.key === "Enter" || event.key === " ") {
|
||||||
|
event.preventDefault();
|
||||||
|
setPinned(current => !current);
|
||||||
|
setOpen(true);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const turnText =
|
||||||
|
snapshot?.status === "queued"
|
||||||
|
? t("monitorQueued")
|
||||||
|
: snapshot?.turn != null
|
||||||
|
? snapshot.turnLimit
|
||||||
|
? t("monitorTurnOf", { turn: snapshot.turn, limit: snapshot.turnLimit })
|
||||||
|
: t("monitorTurn", { turn: snapshot.turn })
|
||||||
|
: "—";
|
||||||
|
|
||||||
|
return (
|
||||||
|
<span
|
||||||
|
className={`run-probe ${runId ? "" : "inactive"}`}
|
||||||
|
ref={anchorRef}
|
||||||
|
tabIndex={runId ? 0 : -1}
|
||||||
|
aria-expanded={open}
|
||||||
|
aria-label={label || t("monitorTitle")}
|
||||||
|
onMouseEnter={runId ? hoverIn : undefined}
|
||||||
|
onMouseLeave={runId ? hoverOut : undefined}
|
||||||
|
onClick={
|
||||||
|
runId
|
||||||
|
? () => {
|
||||||
|
setOpen(true);
|
||||||
|
setPinned(current => !current);
|
||||||
|
}
|
||||||
|
: undefined
|
||||||
|
}
|
||||||
|
onKeyDown={runId ? onKeyDown : undefined}
|
||||||
|
>
|
||||||
|
{children}
|
||||||
|
{open ? (
|
||||||
|
<span
|
||||||
|
className={`run-monitor ${pinned ? "pinned" : ""}`}
|
||||||
|
style={box ? { left: box.left, bottom: box.bottom, width: Math.min(PANEL_WIDTH, window.innerWidth - 24) } : { display: "none" }}
|
||||||
|
role="dialog"
|
||||||
|
aria-label={t("monitorTitle")}
|
||||||
|
onMouseEnter={hoverIn}
|
||||||
|
onMouseLeave={hoverOut}
|
||||||
|
>
|
||||||
|
<span className="run-monitor-head">
|
||||||
|
<b>{t("monitorTitle")}</b>
|
||||||
|
<span className="run-monitor-turn">
|
||||||
|
{turnText}
|
||||||
|
{snapshot?.elapsedMs != null ? ` · ${t("monitorElapsed", { time: formatElapsed(snapshot.elapsedMs) })}` : ""}
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
<span className={`run-monitor-step ${snapshot?.error ? "bad" : ""}`}>
|
||||||
|
{snapshot?.step ? t("monitorNow", { step: snapshot.step }) : snapshot?.error?.headline || t("monitorEmpty")}
|
||||||
|
</span>
|
||||||
|
<span
|
||||||
|
className="run-monitor-list"
|
||||||
|
ref={listRef}
|
||||||
|
onScroll={event => {
|
||||||
|
const node = event.currentTarget;
|
||||||
|
followTail.current = node.scrollHeight - node.scrollTop - node.clientHeight < 32;
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{entries.length === 0 ? <span className="run-monitor-empty">{stale ? t("monitorLoadFailed") : t("monitorEmpty")}</span> : null}
|
||||||
|
{entries.map(entry => (
|
||||||
|
<span className={`run-line kind-${entry.kind} ${entry.status && entry.status !== "ok" ? `is-${entry.status}` : ""}`} key={entry.id}>
|
||||||
|
<span className="run-line-clock">{clockOf(entry.createdAt)}</span>
|
||||||
|
<span className="run-line-kind">{kindLabel(entry.kind)}</span>
|
||||||
|
<span className="run-line-text">{trailText(entry)}</span>
|
||||||
|
</span>
|
||||||
|
))}
|
||||||
|
</span>
|
||||||
|
{snapshot?.error ? (
|
||||||
|
<span className="run-monitor-error">
|
||||||
|
<b>{`[${snapshot.error.code}] ${errorTitle(snapshot.error.code)}`}</b>
|
||||||
|
<code>{snapshot.error.raw}</code>
|
||||||
|
</span>
|
||||||
|
) : null}
|
||||||
|
<span className="run-monitor-foot">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() =>
|
||||||
|
void copyLog(logDump(snapshot, entries)).then(ok => {
|
||||||
|
setCopied(ok);
|
||||||
|
window.setTimeout(() => setCopied(null), 1800);
|
||||||
|
})
|
||||||
|
}
|
||||||
|
>
|
||||||
|
{copied === null ? t("monitorCopy") : copied ? t("monitorCopied") : t("clipboardWriteBlocked")}
|
||||||
|
</button>
|
||||||
|
<small>{pinned ? t("monitorPinned") : t("monitorPinHint")}</small>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="run-monitor-close"
|
||||||
|
aria-label={t("monitorClose")}
|
||||||
|
onClick={() => {
|
||||||
|
setPinned(false);
|
||||||
|
setOpen(false);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
×
|
||||||
|
</button>
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
) : null}
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -10,6 +10,12 @@ export interface MessageFile { kind:"image"|"file"; name:string; mimeType?:strin
|
||||||
export interface RoomMember { id:string; name:string; avatarColor:string; avatarShape:AvatarShape }
|
export interface RoomMember { id:string; name:string; avatarColor:string; avatarShape:AvatarShape }
|
||||||
export interface Room { id:string; name:string; members:RoomMember[]; lastMessageAt:string|null; lastPreview:string|null; unreadCount:number }
|
export interface Room { id:string; name:string; members:RoomMember[]; lastMessageAt:string|null; lastPreview:string|null; unreadCount:number }
|
||||||
export interface ComputerStatus { botId:string; mode:ComputerMode; state:ComputerState; controlHolder:"none"|"bot"|"user"; takeoverRequested:boolean; busyBotName:string|null; busySessionId:string|null; busyRunId:string|null; busyStep?:string|null; usingComputer?:boolean; waitingRunId?:string|null; waitingSessionId?:string|null; queuedRuns?:number; display:string|null; profileMode:string; screenAvailable:boolean }
|
export interface ComputerStatus { botId:string; mode:ComputerMode; state:ComputerState; controlHolder:"none"|"bot"|"user"; takeoverRequested:boolean; busyBotName:string|null; busySessionId:string|null; busyRunId:string|null; busyStep?:string|null; usingComputer?:boolean; waitingRunId?:string|null; waitingSessionId?:string|null; queuedRuns?:number; display:string|null; profileMode:string; screenAvailable:boolean }
|
||||||
|
/** One line of the live trail a run writes while it works. */
|
||||||
|
export type RunActivityKind = "run"|"model"|"tool"|"retry"|"notice";
|
||||||
|
export interface RunActivityEntry { id:number; kind:RunActivityKind; createdAt:string; turn?:number|null; event?:string|null; task?:string|null; reason?:string|null; turns?:number|null; limit?:number|null; error?:string|null; name?:string|null; step?:string|null; status?:string|null; elapsedMs?:number|null; toolCalls?:number|null; text?:string|null; snippet?:string|null; attempt?:number|null; gaveUp?:boolean|null }
|
||||||
|
export interface RunActivityError { code:string; headline:string; raw:string }
|
||||||
|
export interface RunActivity { runId:string; status:string; turn:number|null; turnLimit:number|null; step:string|null; elapsedMs:number|null; error:RunActivityError|null; activity:RunActivityEntry[] }
|
||||||
|
|
||||||
export interface PlaybookStep { do:string; expect?:string; note?:string }
|
export interface PlaybookStep { do:string; expect?:string; note?:string }
|
||||||
export interface PlaybookInput { name:string; description?:string; example?:string }
|
export interface PlaybookInput { name:string; description?:string; example?:string }
|
||||||
export interface Playbook { name?:string; whenToUse?:string; intent?:string; inputs?:PlaybookInput[]; preconditions?:string[]; steps?:(PlaybookStep|string)[]; howToCheck?:string; whatToReturn?:string; cautions?:string[] }
|
export interface Playbook { name?:string; whenToUse?:string; intent?:string; inputs?:PlaybookInput[]; preconditions?:string[]; steps?:(PlaybookStep|string)[]; howToCheck?:string; whatToReturn?:string; cautions?:string[] }
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
# Clippy configuration for the LazyBoy workspace.
|
||||||
|
#
|
||||||
|
# cargo-clippy looks for this file next to the working directory it was started
|
||||||
|
# in, so always run clippy from the workspace root (`make clippy`).
|
||||||
|
#
|
||||||
|
# Handlers and run/vault/voice helpers legitimately thread 8-10 parameters
|
||||||
|
# (state + actor + ids). The default of 7 is too tight for this codebase, so we
|
||||||
|
# keep the guard for genuinely oversized signatures instead of disabling it.
|
||||||
|
too-many-arguments-threshold = 10
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
name = "lazyboy-api"
|
name = "lazyboy-api"
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
publish.workspace = true
|
publish.workspace = true
|
||||||
|
|
||||||
|
|
@ -14,23 +15,19 @@ axum.workspace = true
|
||||||
tokio.workspace = true
|
tokio.workspace = true
|
||||||
serde.workspace = true
|
serde.workspace = true
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
thiserror.workspace = true
|
|
||||||
tracing.workspace = true
|
tracing.workspace = true
|
||||||
tracing-subscriber.workspace = true
|
tracing-subscriber.workspace = true
|
||||||
sqlx.workspace = true
|
sqlx.workspace = true
|
||||||
uuid.workspace = true
|
uuid.workspace = true
|
||||||
chrono.workspace = true
|
chrono.workspace = true
|
||||||
tower-http.workspace = true
|
tower-http.workspace = true
|
||||||
async-trait = "0.1"
|
|
||||||
rig-core.workspace = true
|
rig-core.workspace = true
|
||||||
base64.workspace = true
|
base64.workspace = true
|
||||||
hmac.workspace = true
|
|
||||||
sha2.workspace = true
|
sha2.workspace = true
|
||||||
hex.workspace = true
|
hex.workspace = true
|
||||||
reqwest.workspace = true
|
reqwest.workspace = true
|
||||||
tokio-tungstenite.workspace = true
|
tokio-tungstenite.workspace = true
|
||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
http-body-util = "0.1"
|
|
||||||
dotenvy = "0.15"
|
dotenvy = "0.15"
|
||||||
fastembed = { version = "6.0.2", default-features = false, features = ["hf-hub-rustls-tls", "ort-load-dynamic"] }
|
fastembed = { version = "6.0.2", default-features = false, features = ["hf-hub-rustls-tls", "ort-load-dynamic"] }
|
||||||
rmcp = { version = "3.2", default-features = false, features = ["client", "transport-child-process", "transport-streamable-http-client-reqwest"] }
|
rmcp = { version = "3.2", default-features = false, features = ["client", "transport-child-process", "transport-streamable-http-client-reqwest"] }
|
||||||
|
|
@ -41,3 +38,6 @@ chrono-tz = "0.10"
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
|
|
||||||
cap-std = "3"
|
cap-std = "3"
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -185,13 +185,13 @@ pub async fn llm_parts(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if is_text_mime(&stored.mime_type) {
|
} else if is_text_mime(&stored.mime_type) {
|
||||||
if let Some(bytes) = bytes {
|
if let Some(bytes) = bytes
|
||||||
if let Some(text) = utf8_preview(&bytes) {
|
&& let Some(text) = utf8_preview(&bytes)
|
||||||
parts.push(UserContent::text(format!(
|
{
|
||||||
"Contents of {}:\n```\n{text}\n```",
|
parts.push(UserContent::text(format!(
|
||||||
stored.name
|
"Contents of {}:\n```\n{text}\n```",
|
||||||
)));
|
stored.name
|
||||||
}
|
)));
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
notes.push(
|
notes.push(
|
||||||
|
|
@ -220,12 +220,12 @@ pub async fn sweep_all_inboxes(data_dir: &str) {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
sweep_cap_inbox(&home, "inbox");
|
sweep_cap_inbox(&home, "inbox");
|
||||||
if let Ok(bots) = home.open_dir("bots") {
|
if let Ok(bots) = home.open_dir("bots")
|
||||||
if let Ok(entries) = bots.entries() {
|
&& let Ok(entries) = bots.entries()
|
||||||
for bot in entries.flatten() {
|
{
|
||||||
if let Ok(dir) = bots.open_dir(bot.file_name()) {
|
for bot in entries.flatten() {
|
||||||
sweep_cap_inbox(&dir, "inbox");
|
if let Ok(dir) = bots.open_dir(bot.file_name()) {
|
||||||
}
|
sweep_cap_inbox(&dir, "inbox");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -351,9 +351,7 @@ fn parse_stored(value: &Value) -> Option<StoredAttachment> {
|
||||||
}
|
}
|
||||||
let path = value.get("path").and_then(Value::as_str).unwrap_or("");
|
let path = value.get("path").and_then(Value::as_str).unwrap_or("");
|
||||||
if !path.is_empty() {
|
if !path.is_empty() {
|
||||||
let Some(stored) = path.strip_prefix("inbox/") else {
|
let stored = path.strip_prefix("inbox/")?;
|
||||||
return None;
|
|
||||||
};
|
|
||||||
if stored.is_empty() || stored == "." || stored == ".." || stored.contains(['/', '\\']) {
|
if stored.is_empty() || stored == "." || stored == ".." || stored.contains(['/', '\\']) {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -45,7 +45,7 @@ impl AuthConfig {
|
||||||
pub fn strong_enough_for_network(&self) -> bool {
|
pub fn strong_enough_for_network(&self) -> bool {
|
||||||
self.token
|
self.token
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|token| token.as_bytes().len() >= 32 && token != "dev-token")
|
.map(|token| token.len() >= 32 && token != "dev-token")
|
||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -83,14 +83,13 @@ impl AuthConfig {
|
||||||
let key = hex::encode(Sha256::digest(value.as_bytes()));
|
let key = hex::encode(Sha256::digest(value.as_bytes()));
|
||||||
let mut sessions = self.sessions.lock().unwrap();
|
let mut sessions = self.sessions.lock().unwrap();
|
||||||
sessions.retain(|_, expires| *expires > Instant::now());
|
sessions.retain(|_, expires| *expires > Instant::now());
|
||||||
if sessions.len() >= 4096 {
|
if sessions.len() >= 4096
|
||||||
if let Some(oldest) = sessions
|
&& let Some(oldest) = sessions
|
||||||
.iter()
|
.iter()
|
||||||
.min_by_key(|(_, t)| **t)
|
.min_by_key(|(_, t)| **t)
|
||||||
.map(|(k, _)| k.clone())
|
.map(|(k, _)| k.clone())
|
||||||
{
|
{
|
||||||
sessions.remove(&oldest);
|
sessions.remove(&oldest);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
sessions.insert(key, Instant::now() + Duration::from_secs(604800));
|
sessions.insert(key, Instant::now() + Duration::from_secs(604800));
|
||||||
Some(format!(
|
Some(format!(
|
||||||
|
|
|
||||||
|
|
@ -1076,15 +1076,14 @@ async fn pause_idle_computers(state: &AppState) {
|
||||||
if computer_has_active_work(state, &computer.id).await {
|
if computer_has_active_work(state, &computer.id).await {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if let Some(computer_ref) = computer_ref(&computer) {
|
if let Some(computer_ref) = computer_ref(&computer)
|
||||||
if state
|
&& state
|
||||||
.sandbox
|
.sandbox
|
||||||
.suspend(&computer_ref, &idle_adapter(&computer, "idle"))
|
.suspend(&computer_ref, &idle_adapter(&computer, "idle"))
|
||||||
.await
|
.await
|
||||||
.is_err()
|
.is_err()
|
||||||
{
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
let _ = sqlx::query(
|
let _ = sqlx::query(
|
||||||
"UPDATE computers SET state = 'suspended', updated_at = now()
|
"UPDATE computers SET state = 'suspended', updated_at = now()
|
||||||
|
|
|
||||||
|
|
@ -33,9 +33,7 @@ fn parse_file(path: PathBuf, name: String) -> Option<FileSkill> {
|
||||||
let (header, body) = rest.split_once("\n---")?;
|
let (header, body) = rest.split_once("\n---")?;
|
||||||
(
|
(
|
||||||
header,
|
header,
|
||||||
body.trim_start_matches(|ch| ch == '\n' || ch == '\r')
|
body.trim_start_matches(['\n', '\r']).trim().to_string(),
|
||||||
.trim()
|
|
||||||
.to_string(),
|
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
("", text.trim().to_string())
|
("", text.trim().to_string())
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ mod file_skills;
|
||||||
mod mcp;
|
mod mcp;
|
||||||
mod mcp_catalog;
|
mod mcp_catalog;
|
||||||
mod memory;
|
mod memory;
|
||||||
|
mod monitor;
|
||||||
mod retention;
|
mod retention;
|
||||||
mod rooms;
|
mod rooms;
|
||||||
mod routes;
|
mod routes;
|
||||||
|
|
|
||||||
|
|
@ -570,7 +570,7 @@ async fn create_server(
|
||||||
internal(error.to_string())
|
internal(error.to_string())
|
||||||
}
|
}
|
||||||
})?;
|
})?;
|
||||||
let mut row = load_row(state.pool(), &actor, &id)
|
let row = load_row(state.pool(), &actor, &id)
|
||||||
.await
|
.await
|
||||||
.map_err(|error| internal(error.to_string()))?
|
.map_err(|error| internal(error.to_string()))?
|
||||||
.ok_or_else(|| internal("missing row".into()))?;
|
.ok_or_else(|| internal("missing row".into()))?;
|
||||||
|
|
|
||||||
|
|
@ -28,7 +28,9 @@ pub struct MemoryService {
|
||||||
|
|
||||||
enum ModelState {
|
enum ModelState {
|
||||||
Uninitialized,
|
Uninitialized,
|
||||||
Ready(TextEmbedding),
|
// Boxed: the embedding model is far larger than the other two variants and
|
||||||
|
// this enum lives inside an Arc<Mutex<..>> shared by every request.
|
||||||
|
Ready(Box<TextEmbedding>),
|
||||||
Unavailable,
|
Unavailable,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -102,7 +104,7 @@ impl MemoryService {
|
||||||
.with_cache_dir(cache_dir)
|
.with_cache_dir(cache_dir)
|
||||||
.with_show_download_progress(false);
|
.with_show_download_progress(false);
|
||||||
match TextEmbedding::try_new(options) {
|
match TextEmbedding::try_new(options) {
|
||||||
Ok(embedding) => *state = ModelState::Ready(embedding),
|
Ok(embedding) => *state = ModelState::Ready(Box::new(embedding)),
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
*state = ModelState::Unavailable;
|
*state = ModelState::Unavailable;
|
||||||
return Err(format!("FastEmbed unavailable: {error}"));
|
return Err(format!("FastEmbed unavailable: {error}"));
|
||||||
|
|
@ -151,7 +153,7 @@ impl MemoryService {
|
||||||
let content = validate_content(&input.content)?;
|
let content = validate_content(&input.content)?;
|
||||||
validate_importance(input.importance)?;
|
validate_importance(input.importance)?;
|
||||||
let embedding = self.embed(content.clone()).await;
|
let embedding = self.embed(content.clone()).await;
|
||||||
let vector = embedding.as_ref().map(vector_literal);
|
let vector = embedding.as_deref().map(vector_literal);
|
||||||
let id = Uuid::new_v4();
|
let id = Uuid::new_v4();
|
||||||
let mut tx = pool.begin().await.map_err(|error| error.to_string())?;
|
let mut tx = pool.begin().await.map_err(|error| error.to_string())?;
|
||||||
let item: MemoryItem = sqlx::query_as(
|
let item: MemoryItem = sqlx::query_as(
|
||||||
|
|
@ -212,7 +214,7 @@ impl MemoryService {
|
||||||
}
|
}
|
||||||
let limit = limit.unwrap_or(self.top_k).clamp(1, 50);
|
let limit = limit.unwrap_or(self.top_k).clamp(1, 50);
|
||||||
let embedding = self.embed(query.to_string()).await;
|
let embedding = self.embed(query.to_string()).await;
|
||||||
let rows = if let Some(vector) = embedding.as_ref().map(vector_literal) {
|
let rows = if let Some(vector) = embedding.as_deref().map(vector_literal) {
|
||||||
sqlx::query_as(
|
sqlx::query_as(
|
||||||
"SELECT id,session_id,source_run_id,source_message_id,content,importance,revision,
|
"SELECT id,session_id,source_run_id,source_message_id,content,importance,revision,
|
||||||
created_at,updated_at,deleted_at
|
created_at,updated_at,deleted_at
|
||||||
|
|
@ -269,7 +271,7 @@ impl MemoryService {
|
||||||
let vector = self
|
let vector = self
|
||||||
.embed(content.clone())
|
.embed(content.clone())
|
||||||
.await
|
.await
|
||||||
.as_ref()
|
.as_deref()
|
||||||
.map(vector_literal);
|
.map(vector_literal);
|
||||||
let mut tx = pool.begin().await.map_err(|error| error.to_string())?;
|
let mut tx = pool.begin().await.map_err(|error| error.to_string())?;
|
||||||
let item: Option<MemoryItem> = sqlx::query_as(
|
let item: Option<MemoryItem> = sqlx::query_as(
|
||||||
|
|
@ -377,7 +379,7 @@ async fn insert_revision(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn vector_literal(vector: &Vec<f32>) -> String {
|
fn vector_literal(vector: &[f32]) -> String {
|
||||||
format!(
|
format!(
|
||||||
"[{}]",
|
"[{}]",
|
||||||
vector
|
vector
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,538 @@
|
||||||
|
//! Observability surface for a run: the activity trail the chat bubble reads,
|
||||||
|
//! and the plain-language reading of a failure that turns into an action item.
|
||||||
|
use axum::extract::{Path, Query, State};
|
||||||
|
use axum::http::StatusCode;
|
||||||
|
use axum::routing::{get, post};
|
||||||
|
use axum::{Json, Router};
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
use crate::db::Actor;
|
||||||
|
use crate::state::AppState;
|
||||||
|
|
||||||
|
type ApiError = (StatusCode, Json<Value>);
|
||||||
|
|
||||||
|
/// Longest a single string inside an activity payload may get. The trail is a
|
||||||
|
/// glanceable summary, not a second copy of the transcript.
|
||||||
|
const MAX_STRING_CHARS: usize = 512;
|
||||||
|
const DEFAULT_ACTIVITY_LIMIT: i32 = 60;
|
||||||
|
const MAX_ACTIVITY_LIMIT: i32 = 200;
|
||||||
|
|
||||||
|
pub fn router() -> Router<AppState> {
|
||||||
|
Router::new()
|
||||||
|
.route("/api/runs/{id}/activity", get(activity))
|
||||||
|
.route("/api/runs/{id}/retry", post(retry))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Append one line to the run's trail. Diagnostics never fail a run: a write
|
||||||
|
/// that cannot land is logged and dropped.
|
||||||
|
pub async fn record(state: &AppState, run_id: &str, kind: &str, payload: Value) {
|
||||||
|
let result = sqlx::query("INSERT INTO run_activity (run_id,kind,payload) VALUES ($1,$2,$3)")
|
||||||
|
.bind(run_id)
|
||||||
|
.bind(kind)
|
||||||
|
.bind(clamp_strings(&payload, MAX_STRING_CHARS))
|
||||||
|
.execute(state.pool())
|
||||||
|
.await;
|
||||||
|
if let Err(error) = result {
|
||||||
|
tracing::warn!(run_id, kind, "failed to record run activity: {error}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One line of text for the trail: single line, bounded, never an image.
|
||||||
|
pub fn snippet(text: &str, max_chars: usize) -> String {
|
||||||
|
let flat = text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||||
|
if flat.chars().count() <= max_chars {
|
||||||
|
return flat;
|
||||||
|
}
|
||||||
|
let mut head: String = flat.chars().take(max_chars).collect();
|
||||||
|
head.push('…');
|
||||||
|
head
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bound every string leaf so one chatty tool result cannot bloat the trail.
|
||||||
|
pub fn clamp_strings(value: &Value, max_chars: usize) -> Value {
|
||||||
|
match value {
|
||||||
|
Value::String(text) if text.chars().count() > max_chars => {
|
||||||
|
let mut head: String = text.chars().take(max_chars).collect();
|
||||||
|
head.push('…');
|
||||||
|
Value::String(head)
|
||||||
|
}
|
||||||
|
Value::Array(items) => Value::Array(
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.map(|item| clamp_strings(item, max_chars))
|
||||||
|
.collect(),
|
||||||
|
),
|
||||||
|
Value::Object(map) => Value::Object(
|
||||||
|
map.iter()
|
||||||
|
.map(|(key, item)| (key.clone(), clamp_strings(item, max_chars)))
|
||||||
|
.collect(),
|
||||||
|
),
|
||||||
|
other => other.clone(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What the user is told, and what they can do about it. `code` is the stable
|
||||||
|
/// contract the chat card renders from; the Chinese strings are the fallback
|
||||||
|
/// body that also lives in the message history.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct RunFailure {
|
||||||
|
pub code: &'static str,
|
||||||
|
pub headline: &'static str,
|
||||||
|
pub action: &'static str,
|
||||||
|
pub retryable: bool,
|
||||||
|
pub needs_settings: bool,
|
||||||
|
pub needs_computer: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RunFailure {
|
||||||
|
fn new(
|
||||||
|
code: &'static str,
|
||||||
|
headline: &'static str,
|
||||||
|
action: &'static str,
|
||||||
|
retryable: bool,
|
||||||
|
needs_settings: bool,
|
||||||
|
needs_computer: bool,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
code,
|
||||||
|
headline,
|
||||||
|
action,
|
||||||
|
retryable,
|
||||||
|
needs_settings,
|
||||||
|
needs_computer,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Turn a raw run error into a classified, human-readable failure. Ordering is
|
||||||
|
/// the policy: the most specific and most actionable cause wins, so a 429 that
|
||||||
|
/// also mentions a timeout is reported as a quota problem, not a slow network.
|
||||||
|
pub fn classify_run_error(error: &str) -> RunFailure {
|
||||||
|
let text = error.to_lowercase();
|
||||||
|
let has = |needles: &[&str]| needles.iter().any(|needle| text.contains(needle));
|
||||||
|
if has(&["worker interrupted"]) {
|
||||||
|
return RunFailure::new(
|
||||||
|
"interrupted",
|
||||||
|
"我剛剛被打斷了，最後那個動作可能已經做下去了。",
|
||||||
|
"先按重試；不確定的話，打開它的畫面確認現狀再繼續。",
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if text.contains("tool ") && text.contains("timed out") {
|
||||||
|
return RunFailure::new(
|
||||||
|
"tool_timeout",
|
||||||
|
"有一個動作卡超過 150 秒，我無法確定它有沒有做完。",
|
||||||
|
"先打開它的畫面看一眼，再按重試；我不會重複已經成功的步驟。",
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if has(&[
|
||||||
|
"invalid api key",
|
||||||
|
"incorrect api key",
|
||||||
|
"unauthorized",
|
||||||
|
"authentication",
|
||||||
|
"forbidden",
|
||||||
|
"401",
|
||||||
|
"403",
|
||||||
|
]) {
|
||||||
|
return RunFailure::new(
|
||||||
|
"model_key",
|
||||||
|
"模型不認得這個 API 金鑰。",
|
||||||
|
"到「設定 → 模型」重新貼一次金鑰，再按重試。",
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if has(&[
|
||||||
|
"429",
|
||||||
|
"rate limit",
|
||||||
|
"ratelimit",
|
||||||
|
"too many requests",
|
||||||
|
"insufficient",
|
||||||
|
"quota",
|
||||||
|
"credit",
|
||||||
|
"billing",
|
||||||
|
]) {
|
||||||
|
return RunFailure::new(
|
||||||
|
"model_quota",
|
||||||
|
"模型那邊限流了，或是額度已經用完。",
|
||||||
|
"等一下再按重試；如果是額度用完，到「設定 → 模型」換一個或加值。",
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if has(&[
|
||||||
|
"model not found",
|
||||||
|
"unknown model",
|
||||||
|
"invalid model",
|
||||||
|
"does not exist",
|
||||||
|
"404",
|
||||||
|
]) {
|
||||||
|
return RunFailure::new(
|
||||||
|
"model_unknown",
|
||||||
|
"這個模型名稱找不到，可能已下架或打錯了。",
|
||||||
|
"到「設定 → 模型」選一個存在的模型，再按重試。",
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if has(&["timed out", "timeout", "逾時"]) {
|
||||||
|
return RunFailure::new(
|
||||||
|
"model_timeout",
|
||||||
|
"模型太久沒有回話（超過 165 秒）。",
|
||||||
|
"先按重試；每次都發生的話，到「設定 → 模型」換一個快一點的模型。",
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if has(&[
|
||||||
|
"error sending request",
|
||||||
|
"failed to connect",
|
||||||
|
"connection refused",
|
||||||
|
"dns",
|
||||||
|
"host not found",
|
||||||
|
"certificate",
|
||||||
|
"tls",
|
||||||
|
"network",
|
||||||
|
]) {
|
||||||
|
return RunFailure::new(
|
||||||
|
"network",
|
||||||
|
"連不到模型服務，網路或位址不對。",
|
||||||
|
"確認網路與「設定 → 模型」的 Base URL，再按重試。",
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if has(&[
|
||||||
|
"no such container",
|
||||||
|
"container",
|
||||||
|
"docker",
|
||||||
|
"computer not found",
|
||||||
|
"computer is not running",
|
||||||
|
]) {
|
||||||
|
return RunFailure::new(
|
||||||
|
"computer_gone",
|
||||||
|
"它的電腦不見了或已經停止。",
|
||||||
|
"先把電腦啟動，再按重試。",
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if text.contains("lease") {
|
||||||
|
return RunFailure::new(
|
||||||
|
"lease_lost",
|
||||||
|
"這份工作被另一邊接手過，我手上這份已經失效。",
|
||||||
|
"按重試重新開始這一步就好。",
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
RunFailure::new(
|
||||||
|
"unknown",
|
||||||
|
"我卡住了，這一輪沒有完成。",
|
||||||
|
"按重試看看；需要的話把下面的記錄複製起來給我。",
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The sentence that goes into the chat: what happened, how far the run got,
|
||||||
|
/// and the one thing the human can do next.
|
||||||
|
pub fn failure_message(failure: &RunFailure, last_step: Option<&str>, turn: Option<i64>) -> String {
|
||||||
|
let progress = match (last_step, turn) {
|
||||||
|
(Some(step), Some(n)) if n > 0 && !step.trim().is_empty() => {
|
||||||
|
format!("我最後在做：{}（第 {n} 輪）。", snippet(step, 80))
|
||||||
|
}
|
||||||
|
(Some(step), _) if !step.trim().is_empty() => {
|
||||||
|
format!("我最後在做：{}。", snippet(step, 80))
|
||||||
|
}
|
||||||
|
_ => "我還沒有開始動手。".to_string(),
|
||||||
|
};
|
||||||
|
format!("{} {} {}", failure.headline, progress, failure.action)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, serde::Deserialize)]
|
||||||
|
struct ActivityQuery {
|
||||||
|
after: Option<i64>,
|
||||||
|
limit: Option<i32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, sqlx::FromRow)]
|
||||||
|
struct RunRow {
|
||||||
|
status: String,
|
||||||
|
step: Option<String>,
|
||||||
|
turn: Option<i64>,
|
||||||
|
turn_limit: Option<i64>,
|
||||||
|
error: Option<String>,
|
||||||
|
started_at: Option<DateTime<Utc>>,
|
||||||
|
completed_at: Option<DateTime<Utc>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn actor(state: &AppState) -> Result<Actor, ApiError> {
|
||||||
|
state.bootstrap().await.map_err(|error| {
|
||||||
|
(
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(json!({"message": error.to_string()})),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The bubble's whole payload: where the run stands right now plus its trail.
|
||||||
|
/// `after` pages forward; the first read returns the newest window, oldest
|
||||||
|
/// first, so the UI renders it and then tails from the last id.
|
||||||
|
async fn activity(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Path(id): Path<String>,
|
||||||
|
Query(query): Query<ActivityQuery>,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
let actor = actor(&state).await?;
|
||||||
|
let run = sqlx::query_as::<_, RunRow>(
|
||||||
|
"SELECT status, checkpoint->>'step' AS step, (checkpoint->>'turn')::bigint AS turn,
|
||||||
|
(checkpoint->>'turnLimit')::bigint AS turn_limit, error, started_at, completed_at
|
||||||
|
FROM runs WHERE id=$1 AND space_id=$2 AND user_id=$3",
|
||||||
|
)
|
||||||
|
.bind(&id)
|
||||||
|
.bind(&actor.space_id)
|
||||||
|
.bind(&actor.user_id)
|
||||||
|
.fetch_optional(state.pool())
|
||||||
|
.await
|
||||||
|
.map_err(internal)?;
|
||||||
|
let Some(run) = run else {
|
||||||
|
return Err(not_found("run not found"));
|
||||||
|
};
|
||||||
|
let limit = query
|
||||||
|
.limit
|
||||||
|
.unwrap_or(DEFAULT_ACTIVITY_LIMIT)
|
||||||
|
.clamp(1, MAX_ACTIVITY_LIMIT);
|
||||||
|
let rows = sqlx::query_as::<_, (i64, String, Value, DateTime<Utc>)>(
|
||||||
|
"SELECT id, kind, payload, created_at FROM (
|
||||||
|
SELECT id, kind, payload, created_at FROM run_activity
|
||||||
|
WHERE run_id=$1 AND ($2::bigint IS NULL OR id>$2)
|
||||||
|
ORDER BY id DESC LIMIT $3
|
||||||
|
) recent ORDER BY id ASC",
|
||||||
|
)
|
||||||
|
.bind(&id)
|
||||||
|
.bind(query.after)
|
||||||
|
.bind(limit)
|
||||||
|
.fetch_all(state.pool())
|
||||||
|
.await
|
||||||
|
.map_err(internal)?;
|
||||||
|
let error = run
|
||||||
|
.error
|
||||||
|
.as_deref()
|
||||||
|
.filter(|raw| !raw.trim().is_empty())
|
||||||
|
.map(|raw| {
|
||||||
|
let failure = classify_run_error(raw);
|
||||||
|
json!({"code": failure.code, "headline": failure.headline, "raw": raw})
|
||||||
|
});
|
||||||
|
let elapsed = run.started_at.map(|started| {
|
||||||
|
(run.completed_at.unwrap_or_else(Utc::now) - started)
|
||||||
|
.num_milliseconds()
|
||||||
|
.max(0) as u64
|
||||||
|
});
|
||||||
|
Ok(Json(json!({
|
||||||
|
"runId": id,
|
||||||
|
"status": run.status,
|
||||||
|
"turn": run.turn,
|
||||||
|
"turnLimit": run.turn_limit,
|
||||||
|
"step": run.step,
|
||||||
|
"elapsedMs": elapsed,
|
||||||
|
"error": error,
|
||||||
|
"activity": rows.into_iter().map(|(row_id, kind, payload, created_at)| {
|
||||||
|
let mut entry = payload;
|
||||||
|
if let Some(map) = entry.as_object_mut() {
|
||||||
|
map.insert("id".to_string(), json!(row_id));
|
||||||
|
map.insert("kind".to_string(), json!(kind));
|
||||||
|
map.insert("createdAt".to_string(), json!(created_at.to_rfc3339()));
|
||||||
|
}
|
||||||
|
entry
|
||||||
|
}).collect::<Vec<_>>(),
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Re-queue a run that stopped on an error, keeping its harness checkpoint so
|
||||||
|
/// it continues where it stopped instead of starting the task over.
|
||||||
|
async fn retry(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Path(id): Path<String>,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
let actor = actor(&state).await?;
|
||||||
|
let exists: Option<String> =
|
||||||
|
sqlx::query_scalar("SELECT status FROM runs WHERE id=$1 AND space_id=$2 AND user_id=$3")
|
||||||
|
.bind(&id)
|
||||||
|
.bind(&actor.space_id)
|
||||||
|
.bind(&actor.user_id)
|
||||||
|
.fetch_optional(state.pool())
|
||||||
|
.await
|
||||||
|
.map_err(internal)?;
|
||||||
|
if exists.is_none() {
|
||||||
|
return Err(not_found("run not found"));
|
||||||
|
}
|
||||||
|
let queued: Option<String> = sqlx::query_scalar(
|
||||||
|
"UPDATE runs SET status='queued', retry_count=0, error=NULL, completed_at=NULL,
|
||||||
|
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||||||
|
WHERE id=$1 AND space_id=$2 AND user_id=$3
|
||||||
|
AND status IN ('failed','cancelled')
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM runs a
|
||||||
|
WHERE a.bot_id=runs.bot_id AND a.id<>runs.id
|
||||||
|
AND a.status IN ('leased','running','waiting_input','waiting_takeover')
|
||||||
|
AND (a.lease_expires_at IS NULL OR a.lease_expires_at >= now())
|
||||||
|
)
|
||||||
|
RETURNING thread_id",
|
||||||
|
)
|
||||||
|
.bind(&id)
|
||||||
|
.bind(&actor.space_id)
|
||||||
|
.bind(&actor.user_id)
|
||||||
|
.fetch_optional(state.pool())
|
||||||
|
.await
|
||||||
|
.map_err(internal)?;
|
||||||
|
if queued.is_none() {
|
||||||
|
return Err((
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
Json(json!({"message": "run is not retryable"})),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
record(&state, &id, "run", json!({"event": "retry"})).await;
|
||||||
|
tracing::info!(run_id = id.as_str(), "run re-queued from the chat");
|
||||||
|
Ok(Json(json!({"ok": true, "runId": id})))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn not_found(message: &str) -> ApiError {
|
||||||
|
(StatusCode::NOT_FOUND, Json(json!({"message": message})))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn internal(error: sqlx::Error) -> ApiError {
|
||||||
|
tracing::error!("monitor: {error}");
|
||||||
|
(
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(json!({"message": "internal error"})),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::{RunFailure, clamp_strings, classify_run_error, failure_message, snippet};
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
fn code(error: &str) -> String {
|
||||||
|
classify_run_error(error).code.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn every_failure_offers_an_action_and_a_code() {
|
||||||
|
for error in [
|
||||||
|
"boom",
|
||||||
|
"HTTP 401",
|
||||||
|
"model request timed out after 165 seconds",
|
||||||
|
"no such container",
|
||||||
|
] {
|
||||||
|
let failure = classify_run_error(error);
|
||||||
|
assert!(!failure.code.is_empty(), "{error}");
|
||||||
|
assert!(!failure.action.is_empty(), "{error}");
|
||||||
|
assert!(!failure.headline.is_empty(), "{error}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn auth_quota_and_unknown_model_are_told_apart() {
|
||||||
|
assert_eq!(
|
||||||
|
code("Api error 401 Unauthorized: invalid api key"),
|
||||||
|
"model_key"
|
||||||
|
);
|
||||||
|
assert_eq!(code("HTTP status 403 Forbidden"), "model_key");
|
||||||
|
assert_eq!(code("status 429 Too Many Requests"), "model_quota");
|
||||||
|
assert_eq!(code("insufficient quota for this project"), "model_quota");
|
||||||
|
assert_eq!(code("The model gpt-9 does not exist"), "model_unknown");
|
||||||
|
assert_eq!(code("HTTP status 404 Not Found"), "model_unknown");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_specific_timeouts_win_over_the_generic_one() {
|
||||||
|
assert_eq!(
|
||||||
|
code("tool browser timed out after 150 seconds"),
|
||||||
|
"tool_timeout"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
code("model request timed out after 165 seconds"),
|
||||||
|
"model_timeout"
|
||||||
|
);
|
||||||
|
assert_eq!(code("AI 回應逾時（150 秒）"), "model_timeout");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn infrastructure_and_network_failures_are_separate_from_the_model() {
|
||||||
|
assert_eq!(code("no such container: lazyboy-bot-1"), "computer_gone");
|
||||||
|
assert_eq!(code("computer not found"), "computer_gone");
|
||||||
|
assert_eq!(code("error sending request: dns failure"), "network");
|
||||||
|
assert_eq!(code("run lease was lost"), "lease_lost");
|
||||||
|
assert_eq!(
|
||||||
|
code("Worker interrupted after tool execution; inspect current state"),
|
||||||
|
"interrupted"
|
||||||
|
);
|
||||||
|
assert_eq!(code("something odd happened"), "unknown");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_retryable_answer_never_promises_a_setting_that_cannot_help() {
|
||||||
|
assert!(!classify_run_error("401 invalid api key").retryable);
|
||||||
|
assert!(classify_run_error("401 invalid api key").needs_settings);
|
||||||
|
assert!(classify_run_error("no such container").needs_computer);
|
||||||
|
assert!(!classify_run_error("no such container").needs_settings);
|
||||||
|
assert!(classify_run_error("status 429 rate limit").retryable);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_failure_sentence_names_the_last_step_and_turn() {
|
||||||
|
let failure = RunFailure::new("tool_timeout", "卡住了。", "按重試。", true, false, false);
|
||||||
|
let said = failure_message(&failure, Some("browser: click #12"), Some(7));
|
||||||
|
assert!(said.contains("卡住了。"), "{said}");
|
||||||
|
assert!(said.contains("browser: click #12"), "{said}");
|
||||||
|
assert!(said.contains("第 7 輪"), "{said}");
|
||||||
|
assert!(said.contains("按重試。"), "{said}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_failure_before_any_work_says_so_instead_of_inventing_a_step() {
|
||||||
|
let failure = RunFailure::new("model_key", "金鑰不對。", "改金鑰。", false, true, false);
|
||||||
|
assert!(failure_message(&failure, None, None).contains("我還沒有開始動手。"));
|
||||||
|
assert!(failure_message(&failure, Some(" "), Some(0)).contains("我還沒有開始動手。"));
|
||||||
|
assert!(failure_message(&failure, Some("思考中"), None).contains("思考中"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn snippets_stay_single_line_and_bounded() {
|
||||||
|
assert_eq!(snippet(" 多行\n文字\t在這裡 ", 40), "多行 文字 在這裡");
|
||||||
|
let long = "字".repeat(500);
|
||||||
|
let cut = snippet(&long, 200);
|
||||||
|
assert_eq!(cut.chars().count(), 201);
|
||||||
|
assert!(cut.ends_with('…'));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn payloads_are_clamped_at_every_string_leaf() {
|
||||||
|
let clamped = clamp_strings(
|
||||||
|
&json!({"error": "x".repeat(900), "nested": {"raw": "y".repeat(900)}, "n": 7}),
|
||||||
|
50,
|
||||||
|
);
|
||||||
|
assert_eq!(clamped["error"].as_str().unwrap().chars().count(), 51);
|
||||||
|
assert_eq!(
|
||||||
|
clamped["nested"]["raw"].as_str().unwrap().chars().count(),
|
||||||
|
51
|
||||||
|
);
|
||||||
|
assert_eq!(clamped["n"].as_i64(), Some(7));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -9,11 +9,13 @@ fn days(name: &str, default: i32) -> i32 {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn retention_loop(state: AppState) {
|
pub async fn retention_loop(state: AppState) {
|
||||||
|
let recording_days = days("LAZYBOY_RECORDING_RETENTION_DAYS", 30);
|
||||||
let rules = [
|
let rules = [
|
||||||
("events", include_str!("retention/events.sql"), days("LAZYBOY_EVENT_RETENTION_DAYS", 30)),
|
("events", include_str!("retention/events.sql"), days("LAZYBOY_EVENT_RETENTION_DAYS", 30)),
|
||||||
("checkpoints", include_str!("retention/checkpoints.sql"), days("LAZYBOY_CHECKPOINT_RETENTION_DAYS", 7)),
|
("checkpoints", include_str!("retention/checkpoints.sql"), days("LAZYBOY_CHECKPOINT_RETENTION_DAYS", 7)),
|
||||||
("runs", include_str!("retention/runs.sql"), days("LAZYBOY_RUN_RETENTION_DAYS", 90)),
|
("runs", include_str!("retention/runs.sql"), days("LAZYBOY_RUN_RETENTION_DAYS", 90)),
|
||||||
("recordings", include_str!("retention/recordings.sql"), days("LAZYBOY_RECORDING_RETENTION_DAYS", 30)),
|
("run_activity", include_str!("retention/run_activity.sql"), days("LAZYBOY_RUN_ACTIVITY_RETENTION_DAYS", 7)),
|
||||||
|
("recordings", include_str!("retention/recordings.sql"), recording_days),
|
||||||
("revisions", include_str!("retention/revisions.sql"), days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90)),
|
("revisions", include_str!("retention/revisions.sql"), days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90)),
|
||||||
("deleted_memories", include_str!("retention/deleted_memories.sql"), days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90)),
|
("deleted_memories", include_str!("retention/deleted_memories.sql"), days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90)),
|
||||||
("leases", include_str!("retention/leases.sql"), 7),
|
("leases", include_str!("retention/leases.sql"), 7),
|
||||||
|
|
@ -32,7 +34,7 @@ pub async fn retention_loop(state: AppState) {
|
||||||
}
|
}
|
||||||
if removed > 0 { tracing::info!(name, rows=removed, "retention cleaned expired diagnostics"); }
|
if removed > 0 { tracing::info!(name, rows=removed, "retention cleaned expired diagnostics"); }
|
||||||
}
|
}
|
||||||
if let Err(error) = clean_frames(&state, rules[3].2).await {
|
if let Err(error) = clean_frames(&state, recording_days).await {
|
||||||
tracing::warn!(%error, "recording file retention failed; retry next hour");
|
tracing::warn!(%error, "recording file retention failed; retry next hour");
|
||||||
}
|
}
|
||||||
if let Ok(bytes) = sqlx::query_scalar::<_, i64>("SELECT pg_database_size(current_database())").fetch_one(state.pool()).await {
|
if let Ok(bytes) = sqlx::query_scalar::<_, i64>("SELECT pg_database_size(current_database())").fetch_one(state.pool()).await {
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,4 @@
|
||||||
|
DELETE FROM run_activity WHERE id IN (
|
||||||
|
SELECT id FROM run_activity WHERE created_at < now() - make_interval(days => $1)
|
||||||
|
ORDER BY created_at LIMIT $2 FOR UPDATE SKIP LOCKED
|
||||||
|
)
|
||||||
|
|
@ -64,13 +64,21 @@ async fn members_for(state: &AppState, room_id: &str) -> Result<Vec<RoomMember>,
|
||||||
Ok(rows.into_iter().map(member_from_row).collect())
|
Ok(rows.into_iter().map(member_from_row).collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `room_from_id` projection: id, name, last message time, preview, unread count.
|
||||||
|
type RoomSummaryRow = (
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
Option<chrono::DateTime<chrono::Utc>>,
|
||||||
|
Option<String>,
|
||||||
|
i64,
|
||||||
|
);
|
||||||
|
|
||||||
async fn room_from_id(
|
async fn room_from_id(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
actor: &Actor,
|
actor: &Actor,
|
||||||
id: &str,
|
id: &str,
|
||||||
) -> Result<Option<Room>, sqlx::Error> {
|
) -> Result<Option<Room>, sqlx::Error> {
|
||||||
let row: Option<(String, String, Option<chrono::DateTime<chrono::Utc>>, Option<String>, i64)> =
|
let row: Option<RoomSummaryRow> = sqlx::query_as(
|
||||||
sqlx::query_as(
|
|
||||||
"SELECT r.id, r.name,
|
"SELECT r.id, r.name,
|
||||||
(SELECT MAX(m.created_at) FROM messages m JOIN threads t ON t.id=m.thread_id WHERE t.room_id=r.id),
|
(SELECT MAX(m.created_at) FROM messages m JOIN threads t ON t.id=m.thread_id WHERE t.room_id=r.id),
|
||||||
(SELECT m.body FROM messages m JOIN threads t ON t.id=m.thread_id
|
(SELECT m.body FROM messages m JOIN threads t ON t.id=m.thread_id
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@ pub fn router(state: AppState) -> Router {
|
||||||
Router::new()
|
Router::new()
|
||||||
.merge(crate::sessions::router())
|
.merge(crate::sessions::router())
|
||||||
.merge(crate::memory::router())
|
.merge(crate::memory::router())
|
||||||
|
.merge(crate::monitor::router())
|
||||||
.merge(crate::rooms::router())
|
.merge(crate::rooms::router())
|
||||||
.merge(crate::mcp::router())
|
.merge(crate::mcp::router())
|
||||||
.merge(crate::workspace::router())
|
.merge(crate::workspace::router())
|
||||||
|
|
@ -347,17 +348,16 @@ async fn delete_bot(
|
||||||
if let Some(computer) = computer
|
if let Some(computer) = computer
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.filter(|row| parse_mode(&row.scope) == ComputerMode::Dedicated)
|
.filter(|row| parse_mode(&row.scope) == ComputerMode::Dedicated)
|
||||||
|
&& let Some(computer_ref) = computer::computer_ref(computer)
|
||||||
{
|
{
|
||||||
if let Some(computer_ref) = computer::computer_ref(computer) {
|
state
|
||||||
state
|
.sandbox
|
||||||
.sandbox
|
.destroy(
|
||||||
.destroy(
|
&computer_ref,
|
||||||
&computer_ref,
|
&computer::adapter_context(&actor, &id, "delete-bot"),
|
||||||
&computer::adapter_context(&actor, &id, "delete-bot"),
|
)
|
||||||
)
|
.await
|
||||||
.await
|
.map_err(|error| bad_gateway(error.to_string()))?;
|
||||||
.map_err(|error| bad_gateway(error.to_string()))?;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut tx = state.pool().begin().await.map_err(internal_error)?;
|
let mut tx = state.pool().begin().await.map_err(internal_error)?;
|
||||||
|
|
|
||||||
|
|
@ -236,7 +236,7 @@ pub async fn send(
|
||||||
.bind(&member_run)
|
.bind(&member_run)
|
||||||
.bind(&actor.space_id)
|
.bind(&actor.space_id)
|
||||||
.bind(member_id)
|
.bind(member_id)
|
||||||
.bind(&thread_id)
|
.bind(thread_id)
|
||||||
.bind(&actor.user_id)
|
.bind(&actor.user_id)
|
||||||
.bind(&stored_body)
|
.bind(&stored_body)
|
||||||
.bind(json!({"messageSeq":seq}))
|
.bind(json!({"messageSeq":seq}))
|
||||||
|
|
@ -296,19 +296,43 @@ pub async fn send(
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `worker_loop` claim projection: run id, bot id, thread id, prompt, user id, space id.
|
||||||
|
type RetryCandidateRow = (String, String, String, String, String, String);
|
||||||
|
|
||||||
pub async fn worker_loop(state: AppState) {
|
pub async fn worker_loop(state: AppState) {
|
||||||
let inflight = Arc::new(tokio::sync::Semaphore::new(16));
|
let inflight = Arc::new(tokio::sync::Semaphore::new(16));
|
||||||
let lease_owner = format!("api-{}", Uuid::new_v4());
|
let lease_owner = format!("api-{}", Uuid::new_v4());
|
||||||
loop {
|
loop {
|
||||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||||
let _ = sqlx::query("UPDATE runs SET status='failed',error='Worker interrupted after tool execution; inspect current state before continuing.',completed_at=now(),lease_owner=NULL,lease_expires_at=NULL WHERE status IN ('leased','running') AND lease_expires_at<now() AND COALESCE((checkpoint->>'toolsStarted')::boolean,false)")
|
let interrupted: Vec<(String, String, String)> = sqlx::query_as(
|
||||||
.execute(state.pool()).await;
|
"WITH doomed AS (
|
||||||
|
UPDATE runs SET status='failed',error=$1,completed_at=now(),
|
||||||
|
lease_owner=NULL, lease_expires_at=NULL
|
||||||
|
WHERE status IN ('leased','running') AND lease_expires_at<now()
|
||||||
|
AND COALESCE((checkpoint->>'toolsStarted')::boolean,false)
|
||||||
|
RETURNING id, thread_id, bot_id
|
||||||
|
)
|
||||||
|
SELECT id, thread_id, bot_id FROM doomed LIMIT 50",
|
||||||
|
)
|
||||||
|
.bind(INTERRUPTED_ERROR)
|
||||||
|
.fetch_all(state.pool())
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
for (orphan_run, orphan_thread, orphan_bot) in interrupted {
|
||||||
|
report_run_failure(
|
||||||
|
&state,
|
||||||
|
&orphan_run,
|
||||||
|
&orphan_thread,
|
||||||
|
&orphan_bot,
|
||||||
|
INTERRUPTED_ERROR,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
let Ok(permit) = inflight.clone().try_acquire_owned() else {
|
let Ok(permit) = inflight.clone().try_acquire_owned() else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let queued: Result<Option<(String, String, String, String, String, String)>, _> =
|
let queued: Result<Option<RetryCandidateRow>, _> = sqlx::query_as(
|
||||||
sqlx::query_as(
|
"WITH candidate AS (
|
||||||
"WITH candidate AS (
|
|
||||||
SELECT r.id
|
SELECT r.id
|
||||||
FROM runs r
|
FROM runs r
|
||||||
WHERE r.retry_count < r.max_retries
|
WHERE r.retry_count < r.max_retries
|
||||||
|
|
@ -335,10 +359,10 @@ pub async fn worker_loop(state: AppState) {
|
||||||
lease_fence=lease_fence+1, retry_count=retry_count+1, updated_at=now()
|
lease_fence=lease_fence+1, retry_count=retry_count+1, updated_at=now()
|
||||||
FROM candidate c WHERE r.id=c.id
|
FROM candidate c WHERE r.id=c.id
|
||||||
RETURNING r.id,r.bot_id,r.thread_id,r.prompt,r.user_id,r.space_id",
|
RETURNING r.id,r.bot_id,r.thread_id,r.prompt,r.user_id,r.space_id",
|
||||||
)
|
)
|
||||||
.bind(&lease_owner)
|
.bind(&lease_owner)
|
||||||
.fetch_optional(state.pool())
|
.fetch_optional(state.pool())
|
||||||
.await;
|
.await;
|
||||||
let Ok(Some((run_id, bot_id, thread_id, prompt, user_id, space_id))) = queued else {
|
let Ok(Some((run_id, bot_id, thread_id, prompt, user_id, space_id))) = queued else {
|
||||||
drop(permit);
|
drop(permit);
|
||||||
continue;
|
continue;
|
||||||
|
|
@ -371,14 +395,7 @@ pub async fn worker_loop(state: AppState) {
|
||||||
.ok()
|
.ok()
|
||||||
.flatten();
|
.flatten();
|
||||||
if next_status.as_deref() == Some("failed") {
|
if next_status.as_deref() == Some("failed") {
|
||||||
let _ = append_bot_message(
|
report_run_failure(&state, &run_id, &thread_id, &bot_id, &error).await;
|
||||||
&state,
|
|
||||||
&thread_id,
|
|
||||||
&run_id,
|
|
||||||
&bot_id,
|
|
||||||
&format!("Run failed after retries: {error}"),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
let _ = crate::sessions::append_event(
|
let _ = crate::sessions::append_event(
|
||||||
&state,
|
&state,
|
||||||
&thread_id,
|
&thread_id,
|
||||||
|
|
@ -432,6 +449,13 @@ async fn execute_run(
|
||||||
}
|
}
|
||||||
let _ = crate::sessions::append_event(state, thread_id, "run.started", json!({"runId":run_id}))
|
let _ = crate::sessions::append_event(state, thread_id, "run.started", json!({"runId":run_id}))
|
||||||
.await;
|
.await;
|
||||||
|
crate::monitor::record(
|
||||||
|
state,
|
||||||
|
run_id,
|
||||||
|
"run",
|
||||||
|
json!({"event": "started", "task": crate::monitor::snippet(prompt, 160)}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
let bot = state
|
let bot = state
|
||||||
.db
|
.db
|
||||||
|
|
@ -656,6 +680,13 @@ async fn execute_run(
|
||||||
// model call with no chance of accidentally invoking any capability.
|
// model call with no chance of accidentally invoking any capability.
|
||||||
defs.clear();
|
defs.clear();
|
||||||
tracing::info!(run_id, "chat-only turn: all tools withheld");
|
tracing::info!(run_id, "chat-only turn: all tools withheld");
|
||||||
|
crate::monitor::record(
|
||||||
|
state,
|
||||||
|
run_id,
|
||||||
|
"notice",
|
||||||
|
json!({"text": "這一輪是對白，工具先收起來（不會動到電腦）。"}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
// Taught skills run long (a 24-page course is 24 clicks); plain chats stay
|
// Taught skills run long (a 24-page course is 24 clicks); plain chats stay
|
||||||
// bounded tighter so a confused model cannot burn budget for as long.
|
// bounded tighter so a confused model cannot burn budget for as long.
|
||||||
|
|
@ -668,6 +699,10 @@ async fn execute_run(
|
||||||
} else {
|
} else {
|
||||||
ExecutionMode::Bounded(40)
|
ExecutionMode::Bounded(40)
|
||||||
};
|
};
|
||||||
|
let turn_limit = match execution_mode {
|
||||||
|
ExecutionMode::Goal => None,
|
||||||
|
ExecutionMode::Bounded(limit) => Some(limit as i64),
|
||||||
|
};
|
||||||
let mut nudges: u32 = 0;
|
let mut nudges: u32 = 0;
|
||||||
let mut screenshots: u32 = 0;
|
let mut screenshots: u32 = 0;
|
||||||
let mut screenshot_bytes: u64 = 0;
|
let mut screenshot_bytes: u64 = 0;
|
||||||
|
|
@ -685,22 +720,21 @@ async fn execute_run(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let mut pending = Message::User { content: first };
|
let mut pending = Message::User { content: first };
|
||||||
if !resume_after_takeover {
|
if !resume_after_takeover
|
||||||
if let (Some(saved_history), Some(saved_pending)) = (
|
&& let (Some(saved_history), Some(saved_pending)) = (
|
||||||
checkpoint.get("harnessHistory"),
|
checkpoint.get("harnessHistory"),
|
||||||
checkpoint.get("harnessPending"),
|
checkpoint.get("harnessPending"),
|
||||||
) {
|
)
|
||||||
if let (Ok(restored), Ok(mut next)) = (
|
&& let (Ok(restored), Ok(mut next)) = (
|
||||||
serde_json::from_value::<Vec<Message>>(saved_history.clone()),
|
serde_json::from_value::<Vec<Message>>(saved_history.clone()),
|
||||||
serde_json::from_value::<Message>(saved_pending.clone()),
|
serde_json::from_value::<Message>(saved_pending.clone()),
|
||||||
) {
|
)
|
||||||
history = restored;
|
{
|
||||||
if let Message::User { content } = &mut next {
|
history = restored;
|
||||||
content.push(UserContent::text("Resumed after a completed tool batch. Do not repeat completed actions. Observe current browser/desktop before any new mutation; prior element references may be stale."));
|
if let Message::User { content } = &mut next {
|
||||||
}
|
content.push(UserContent::text("Resumed after a completed tool batch. Do not repeat completed actions. Observe current browser/desktop before any new mutation; prior element references may be stale."));
|
||||||
pending = next;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
pending = next;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut final_text = String::new();
|
let mut final_text = String::new();
|
||||||
|
|
@ -727,6 +761,13 @@ async fn execute_run(
|
||||||
{
|
{
|
||||||
Ok(items) => state.memory.durable_block(&items),
|
Ok(items) => state.memory.durable_block(&items),
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
|
crate::monitor::record(
|
||||||
|
state,
|
||||||
|
run_id,
|
||||||
|
"notice",
|
||||||
|
json!({"text": format!("記憶讀取失敗，這輪不用記憶：{error}")}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
tracing::warn!("memory retrieval failed for run {run_id}: {error}");
|
tracing::warn!("memory retrieval failed for run {run_id}: {error}");
|
||||||
String::new()
|
String::new()
|
||||||
}
|
}
|
||||||
|
|
@ -791,17 +832,17 @@ async fn execute_run(
|
||||||
preamble.push_str("\n\n");
|
preamble.push_str("\n\n");
|
||||||
preamble.push_str(&file_skill_index);
|
preamble.push_str(&file_skill_index);
|
||||||
}
|
}
|
||||||
if let Ok(accounts) = crate::vault::list_on(state.pool(), actor, bot_id).await {
|
if let Ok(accounts) = crate::vault::list_on(state.pool(), actor, bot_id).await
|
||||||
if !accounts.is_empty() {
|
&& !accounts.is_empty()
|
||||||
let names = accounts
|
{
|
||||||
.iter()
|
let names = accounts
|
||||||
.map(|item| format!("{} ({})", item.site, item.username))
|
.iter()
|
||||||
.collect::<Vec<_>>()
|
.map(|item| format!("{} ({})", item.site, item.username))
|
||||||
.join(", ");
|
.collect::<Vec<_>>()
|
||||||
preamble.push_str(&format!(
|
.join(", ");
|
||||||
|
preamble.push_str(&format!(
|
||||||
"\n\nSaved logins (passwords are not shown): {names}. At a login wall call use_saved_login with the matching accountId from list_accounts."
|
"\n\nSaved logins (passwords are not shown): {names}. At a login wall call use_saved_login with the matching accountId from list_accounts."
|
||||||
));
|
));
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -837,6 +878,16 @@ async fn execute_run(
|
||||||
guidance.join("\n")
|
guidance.join("\n")
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
crate::monitor::record(
|
||||||
|
state,
|
||||||
|
run_id,
|
||||||
|
"notice",
|
||||||
|
json!({
|
||||||
|
"turn": turns,
|
||||||
|
"text": format!("收到新指示：{}", crate::monitor::snippet(&guidance.join(" / "), 160)),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
if let Message::User { content } = &mut pending {
|
if let Message::User { content } = &mut pending {
|
||||||
content.push(UserContent::text(text));
|
content.push(UserContent::text(text));
|
||||||
}
|
}
|
||||||
|
|
@ -858,7 +909,7 @@ async fn execute_run(
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
drop_history_screenshots(&mut history, &pending);
|
drop_history_screenshots(&mut history, &pending);
|
||||||
set_run_step(state, run_id, MODEL_STEP).await;
|
set_run_progress(state, run_id, MODEL_STEP, turns, turn_limit).await;
|
||||||
let model_started = std::time::Instant::now();
|
let model_started = std::time::Instant::now();
|
||||||
let content = tokio::select! {
|
let content = tokio::select! {
|
||||||
halt = wait_for_halt(state, run_id) => {
|
halt = wait_for_halt(state, run_id) => {
|
||||||
|
|
@ -875,7 +926,14 @@ async fn execute_run(
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
result = complete_with_retry(&model, pending.clone(), &preamble, &history, &defs) => result
|
result = complete_with_retry(
|
||||||
|
&model,
|
||||||
|
pending.clone(),
|
||||||
|
&preamble,
|
||||||
|
&history,
|
||||||
|
&defs,
|
||||||
|
Trace { state, run_id, turn: turns },
|
||||||
|
) => result
|
||||||
}?;
|
}?;
|
||||||
let assistant = Message::Assistant {
|
let assistant = Message::Assistant {
|
||||||
id: None,
|
id: None,
|
||||||
|
|
@ -885,20 +943,37 @@ async fn execute_run(
|
||||||
history.push(assistant);
|
history.push(assistant);
|
||||||
|
|
||||||
let mut calls = Vec::new();
|
let mut calls = Vec::new();
|
||||||
|
let mut turn_text = String::new();
|
||||||
for item in &content {
|
for item in &content {
|
||||||
match item {
|
match item {
|
||||||
AssistantContent::Text(text) => final_text.push_str(&text.text),
|
AssistantContent::Text(text) => {
|
||||||
|
final_text.push_str(&text.text);
|
||||||
|
turn_text.push_str(&text.text);
|
||||||
|
}
|
||||||
AssistantContent::ToolCall(call) => calls.push(call.clone()),
|
AssistantContent::ToolCall(call) => calls.push(call.clone()),
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
let model_elapsed = model_started.elapsed().as_millis() as u64;
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
run_id,
|
run_id,
|
||||||
turn = turns,
|
turn = turns,
|
||||||
elapsed_ms = model_started.elapsed().as_millis() as u64,
|
elapsed_ms = model_elapsed,
|
||||||
tool_calls = calls.len(),
|
tool_calls = calls.len(),
|
||||||
"model turn"
|
"model turn"
|
||||||
);
|
);
|
||||||
|
crate::monitor::record(
|
||||||
|
state,
|
||||||
|
run_id,
|
||||||
|
"model",
|
||||||
|
json!({
|
||||||
|
"turn": turns,
|
||||||
|
"elapsedMs": model_elapsed,
|
||||||
|
"toolCalls": calls.len(),
|
||||||
|
"text": crate::monitor::snippet(&turn_text, 200),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
if calls.is_empty() {
|
if calls.is_empty() {
|
||||||
// A model that quits a playbook early, or parrots an earlier reply
|
// A model that quits a playbook early, or parrots an earlier reply
|
||||||
// instead of describing the current screen, gets pushed back to
|
// instead of describing the current screen, gets pushed back to
|
||||||
|
|
@ -946,6 +1021,22 @@ async fn execute_run(
|
||||||
verify = verify_chosen,
|
verify = verify_chosen,
|
||||||
"nudging model back to tools"
|
"nudging model back to tools"
|
||||||
);
|
);
|
||||||
|
crate::monitor::record(
|
||||||
|
state,
|
||||||
|
run_id,
|
||||||
|
"notice",
|
||||||
|
json!({
|
||||||
|
"turn": turns,
|
||||||
|
"text": if verify_chosen {
|
||||||
|
format!("要求模型先證明做完了才准結束（第 {nudges} 次攔下）。")
|
||||||
|
} else if parroted {
|
||||||
|
format!("模型重複了舊的回答，叫它看現在畫面繼續做（第 {nudges} 次）。")
|
||||||
|
} else {
|
||||||
|
format!("模型想用一句話收尾，叫它用工具繼續（第 {nudges} 次）。")
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
earlier_replies.push(final_text.trim().to_string());
|
earlier_replies.push(final_text.trim().to_string());
|
||||||
final_text.clear();
|
final_text.clear();
|
||||||
let mut content = vec![UserContent::text(text)];
|
let mut content = vec![UserContent::text(text)];
|
||||||
|
|
@ -1047,13 +1138,14 @@ async fn execute_run(
|
||||||
);
|
);
|
||||||
did_work = true;
|
did_work = true;
|
||||||
let step = describe_step(&name, &call.function.arguments);
|
let step = describe_step(&name, &call.function.arguments);
|
||||||
set_run_step(state, run_id, &step).await;
|
set_run_progress(state, run_id, &step, turns, turn_limit).await;
|
||||||
let fence=sqlx::query("UPDATE runs SET checkpoint=COALESCE(checkpoint,'{}'::jsonb)||jsonb_build_object('toolsStarted',true) WHERE id=$1 AND lease_owner=$2 AND status='running'")
|
let fence=sqlx::query("UPDATE runs SET checkpoint=COALESCE(checkpoint,'{}'::jsonb)||jsonb_build_object('toolsStarted',true) WHERE id=$1 AND lease_owner=$2 AND status='running'")
|
||||||
.bind(run_id).bind(lease_owner).execute(state.pool()).await.map_err(|e| e.to_string())?;
|
.bind(run_id).bind(lease_owner).execute(state.pool()).await.map_err(|e| e.to_string())?;
|
||||||
if fence.rows_affected() != 1 {
|
if fence.rows_affected() != 1 {
|
||||||
return Err("run lease lost before tool dispatch".into());
|
return Err("run lease lost before tool dispatch".into());
|
||||||
}
|
}
|
||||||
let tool_started = std::time::Instant::now();
|
let tool_started = std::time::Instant::now();
|
||||||
|
let mut tool_timed_out = false;
|
||||||
let outcome = tokio::select! {
|
let outcome = tokio::select! {
|
||||||
halt = wait_for_halt(state, run_id) => {
|
halt = wait_for_halt(state, run_id) => {
|
||||||
return finish_halt(
|
return finish_halt(
|
||||||
|
|
@ -1076,12 +1168,15 @@ async fn execute_run(
|
||||||
Ok(outcome) => outcome,
|
Ok(outcome) => outcome,
|
||||||
// A slow tool is a normal turn, not a dead run: say the
|
// A slow tool is a normal turn, not a dead run: say the
|
||||||
// effects are unknown and let the model re-observe.
|
// effects are unknown and let the model re-observe.
|
||||||
Err(_) => ToolOutcome {
|
Err(_) => {
|
||||||
|
tool_timed_out = true;
|
||||||
|
ToolOutcome {
|
||||||
text: format!("tool {name} timed out after 150 seconds. Its effects are unknown: observe the current screen or files before anything else, and never repeat a step that already worked."),
|
text: format!("tool {name} timed out after 150 seconds. Its effects are unknown: observe the current screen or files before anything else, and never repeat a step that already worked."),
|
||||||
image: None,
|
image: None,
|
||||||
pause: false,
|
pause: false,
|
||||||
blocks: Vec::new(),
|
blocks: Vec::new(),
|
||||||
},
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
|
|
@ -1094,6 +1189,20 @@ async fn execute_run(
|
||||||
pause = outcome.pause,
|
pause = outcome.pause,
|
||||||
"tool call"
|
"tool call"
|
||||||
);
|
);
|
||||||
|
crate::monitor::record(
|
||||||
|
state,
|
||||||
|
run_id,
|
||||||
|
"tool",
|
||||||
|
json!({
|
||||||
|
"turn": turns,
|
||||||
|
"name": name.clone(),
|
||||||
|
"step": step.clone(),
|
||||||
|
"status": tool_status(tool_timed_out, outcome.pause, &outcome.text),
|
||||||
|
"elapsedMs": tool_started.elapsed().as_millis() as u64,
|
||||||
|
"snippet": crate::monitor::snippet(&outcome.text, 200),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
// xAI rejects images inside tool results. Attach a changed
|
// xAI rejects images inside tool results. Attach a changed
|
||||||
// screenshot as a following user image instead.
|
// screenshot as a following user image instead.
|
||||||
if let Some(image) = outcome.image {
|
if let Some(image) = outcome.image {
|
||||||
|
|
@ -1161,7 +1270,16 @@ async fn execute_run(
|
||||||
results.extend(screenshot_parts(png));
|
results.extend(screenshot_parts(png));
|
||||||
}
|
}
|
||||||
pending = Message::User { content: results };
|
pending = Message::User { content: results };
|
||||||
save_harness_checkpoint(state, run_id, lease_owner, &history, &pending, turns, steering_seq).await?;
|
save_harness_checkpoint(
|
||||||
|
state,
|
||||||
|
run_id,
|
||||||
|
lease_owner,
|
||||||
|
&history,
|
||||||
|
&pending,
|
||||||
|
turns,
|
||||||
|
steering_seq,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let status: Option<String> = sqlx::query_scalar("SELECT status FROM runs WHERE id = $1")
|
let status: Option<String> = sqlx::query_scalar("SELECT status FROM runs WHERE id = $1")
|
||||||
|
|
@ -1214,8 +1332,21 @@ async fn execute_run(
|
||||||
}
|
}
|
||||||
let needs_input = goal_mode && goal_outcome(&final_text) == GoalOutcome::NeedsInput;
|
let needs_input = goal_mode && goal_outcome(&final_text) == GoalOutcome::NeedsInput;
|
||||||
if needs_input {
|
if needs_input {
|
||||||
let next = Message::User { content: vec![UserContent::text("The goal was paused for required user input. Read the user's new information and continue from completed work.")] };
|
let next = Message::User {
|
||||||
save_harness_checkpoint(state, run_id, lease_owner, &history, &next, turns, steering_seq).await?;
|
content: vec![UserContent::text(
|
||||||
|
"The goal was paused for required user input. Read the user's new information and continue from completed work.",
|
||||||
|
)],
|
||||||
|
};
|
||||||
|
save_harness_checkpoint(
|
||||||
|
state,
|
||||||
|
run_id,
|
||||||
|
lease_owner,
|
||||||
|
&history,
|
||||||
|
&next,
|
||||||
|
turns,
|
||||||
|
steering_seq,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
}
|
}
|
||||||
let final_text = final_text
|
let final_text = final_text
|
||||||
.replace("[GOAL_COMPLETE]", "")
|
.replace("[GOAL_COMPLETE]", "")
|
||||||
|
|
@ -1238,6 +1369,16 @@ async fn execute_run(
|
||||||
if completed.rows_affected() != 1 {
|
if completed.rows_affected() != 1 {
|
||||||
return Err("run lease was lost before completion".into());
|
return Err("run lease was lost before completion".into());
|
||||||
}
|
}
|
||||||
|
crate::monitor::record(
|
||||||
|
state,
|
||||||
|
run_id,
|
||||||
|
"run",
|
||||||
|
json!({
|
||||||
|
"event": if needs_input { "waiting_input" } else { "completed" },
|
||||||
|
"turns": turns,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
let click_misses = *ctx.click_misses.lock().unwrap();
|
let click_misses = *ctx.click_misses.lock().unwrap();
|
||||||
let takeover = *ctx.takeover_requested.lock().unwrap();
|
let takeover = *ctx.takeover_requested.lock().unwrap();
|
||||||
record_run_metrics(
|
record_run_metrics(
|
||||||
|
|
@ -1331,12 +1472,21 @@ fn retryable_run_error(error: &str) -> bool {
|
||||||
!matches!(code, 400..=499 if !matches!(code, 408 | 409 | 425 | 429))
|
!matches!(code, 400..=499 if !matches!(code, 408 | 409 | 425 | 429))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Where a helper that is not the run loop itself writes its diagnostics.
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct Trace<'a> {
|
||||||
|
state: &'a AppState,
|
||||||
|
run_id: &'a str,
|
||||||
|
turn: u32,
|
||||||
|
}
|
||||||
|
|
||||||
async fn complete_with_retry(
|
async fn complete_with_retry(
|
||||||
model: &DynModel,
|
model: &DynModel,
|
||||||
pending: Message,
|
pending: Message,
|
||||||
preamble: &str,
|
preamble: &str,
|
||||||
history: &[Message],
|
history: &[Message],
|
||||||
defs: &[ToolDefinition],
|
defs: &[ToolDefinition],
|
||||||
|
trace: Trace<'_>,
|
||||||
) -> Result<Vec<AssistantContent>, String> {
|
) -> Result<Vec<AssistantContent>, String> {
|
||||||
let mut last = String::new();
|
let mut last = String::new();
|
||||||
for attempt in 0..3 {
|
for attempt in 0..3 {
|
||||||
|
|
@ -1349,6 +1499,18 @@ async fn complete_with_retry(
|
||||||
match result {
|
match result {
|
||||||
Ok(Ok(content)) => return Ok(content),
|
Ok(Ok(content)) => return Ok(content),
|
||||||
Ok(Err(error)) => {
|
Ok(Err(error)) => {
|
||||||
|
crate::monitor::record(
|
||||||
|
trace.state,
|
||||||
|
trace.run_id,
|
||||||
|
"retry",
|
||||||
|
json!({
|
||||||
|
"turn": trace.turn,
|
||||||
|
"attempt": attempt + 1,
|
||||||
|
"error": error,
|
||||||
|
"gaveUp": !retryable_run_error(&error),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
if !retryable_run_error(&error) {
|
if !retryable_run_error(&error) {
|
||||||
return Err(error);
|
return Err(error);
|
||||||
}
|
}
|
||||||
|
|
@ -1360,8 +1522,19 @@ async fn complete_with_retry(
|
||||||
last = error;
|
last = error;
|
||||||
}
|
}
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
tracing::warn!(attempt = attempt + 1, "model attempt exceeded its 165 second budget");
|
let error = "model request timed out after 165 seconds";
|
||||||
last = "model request timed out after 165 seconds".into();
|
crate::monitor::record(
|
||||||
|
trace.state,
|
||||||
|
trace.run_id,
|
||||||
|
"retry",
|
||||||
|
json!({"turn": trace.turn, "attempt": attempt + 1, "error": error}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
tracing::warn!(
|
||||||
|
attempt = attempt + 1,
|
||||||
|
"model attempt exceeded its 165 second budget"
|
||||||
|
);
|
||||||
|
last = error.into();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if attempt < 2 {
|
if attempt < 2 {
|
||||||
|
|
@ -1702,6 +1875,13 @@ async fn finish_halt(
|
||||||
.bind(run_id)
|
.bind(run_id)
|
||||||
.execute(state.pool())
|
.execute(state.pool())
|
||||||
.await;
|
.await;
|
||||||
|
crate::monitor::record(
|
||||||
|
state,
|
||||||
|
run_id,
|
||||||
|
"run",
|
||||||
|
json!({"event": "paused", "reason": "takeover", "turns": turns}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
let click_misses = *ctx.click_misses.lock().unwrap();
|
let click_misses = *ctx.click_misses.lock().unwrap();
|
||||||
record_run_metrics(
|
record_run_metrics(
|
||||||
state,
|
state,
|
||||||
|
|
@ -1792,6 +1972,18 @@ async fn pause_for_answer(
|
||||||
if paused.rows_affected() != 1 {
|
if paused.rows_affected() != 1 {
|
||||||
return Err("run lease was lost before pausing".into());
|
return Err("run lease was lost before pausing".into());
|
||||||
}
|
}
|
||||||
|
crate::monitor::record(
|
||||||
|
state,
|
||||||
|
run_id,
|
||||||
|
"run",
|
||||||
|
json!({
|
||||||
|
"event": "paused",
|
||||||
|
"reason": req.reason.as_str(),
|
||||||
|
"turns": req.turns,
|
||||||
|
"limit": req.limit,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
append_bot_message_with(
|
append_bot_message_with(
|
||||||
state,
|
state,
|
||||||
thread_id,
|
thread_id,
|
||||||
|
|
@ -2369,7 +2561,7 @@ fn is_greeting(normalized: &str) -> bool {
|
||||||
"🙂",
|
"🙂",
|
||||||
"😀",
|
"😀",
|
||||||
];
|
];
|
||||||
if EXACT.iter().any(|item| normalized == *item) {
|
if EXACT.contains(&normalized) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
const PREFIXES: &[&str] = &["hi ", "hey ", "hello ", "嗨", "你好"];
|
const PREFIXES: &[&str] = &["hi ", "hey ", "hello ", "嗨", "你好"];
|
||||||
|
|
@ -2641,6 +2833,113 @@ fn describe_step(name: &str, args: &Value) -> String {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The error the claim sweep writes when a worker died holding a tool open.
|
||||||
|
const INTERRUPTED_ERROR: &str =
|
||||||
|
"Worker interrupted after tool execution; inspect current state before continuing.";
|
||||||
|
|
||||||
|
/// Step plus turn counter: what the bubble header reads out of the checkpoint.
|
||||||
|
pub(crate) async fn set_run_progress(
|
||||||
|
state: &AppState,
|
||||||
|
run_id: &str,
|
||||||
|
step: &str,
|
||||||
|
turn: u32,
|
||||||
|
limit: Option<i64>,
|
||||||
|
) {
|
||||||
|
let result = sqlx::query(
|
||||||
|
"UPDATE runs SET checkpoint = COALESCE(checkpoint, '{}'::jsonb)
|
||||||
|
|| jsonb_build_object('step', $2::text, 'stepAt', now(),
|
||||||
|
'turn', $3::bigint, 'turnLimit', $4::bigint),
|
||||||
|
updated_at = now()
|
||||||
|
WHERE id = $1",
|
||||||
|
)
|
||||||
|
.bind(run_id)
|
||||||
|
.bind(step)
|
||||||
|
.bind(turn as i64)
|
||||||
|
.bind(limit)
|
||||||
|
.execute(state.pool())
|
||||||
|
.await;
|
||||||
|
if let Err(error) = result {
|
||||||
|
tracing::warn!(run_id, "failed to record run progress: {error}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tell the human a run died, in their language, with the one next action. The
|
||||||
|
/// raw error stays available through the run activity endpoint.
|
||||||
|
async fn report_run_failure(
|
||||||
|
state: &AppState,
|
||||||
|
run_id: &str,
|
||||||
|
thread_id: &str,
|
||||||
|
bot_id: &str,
|
||||||
|
error: &str,
|
||||||
|
) {
|
||||||
|
let progress: Option<(Option<String>, Option<i64>)> = sqlx::query_as(
|
||||||
|
"SELECT checkpoint->>'step', (checkpoint->>'turn')::bigint FROM runs WHERE id=$1",
|
||||||
|
)
|
||||||
|
.bind(run_id)
|
||||||
|
.fetch_optional(state.pool())
|
||||||
|
.await
|
||||||
|
.unwrap_or(None);
|
||||||
|
let (last_step, turn) = progress.unwrap_or((None, None));
|
||||||
|
let failure = crate::monitor::classify_run_error(error);
|
||||||
|
let body = crate::monitor::failure_message(&failure, last_step.as_deref(), turn);
|
||||||
|
let _ = append_bot_message_with(
|
||||||
|
state,
|
||||||
|
thread_id,
|
||||||
|
run_id,
|
||||||
|
bot_id,
|
||||||
|
&body,
|
||||||
|
json!([{
|
||||||
|
"kind": "error",
|
||||||
|
"code": failure.code,
|
||||||
|
"retryable": failure.retryable,
|
||||||
|
"runId": run_id,
|
||||||
|
"turn": turn,
|
||||||
|
"step": last_step,
|
||||||
|
}]),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
crate::monitor::record(
|
||||||
|
state,
|
||||||
|
run_id,
|
||||||
|
"run",
|
||||||
|
json!({"event": "failed", "error": error}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Display-only reading of a tool result. Tools report failure in prose, so the
|
||||||
|
/// trail marks the shapes it recognises rather than pretending to know more.
|
||||||
|
fn tool_status(timed_out: bool, pause: bool, text: &str) -> &'static str {
|
||||||
|
if timed_out {
|
||||||
|
return "timed_out";
|
||||||
|
}
|
||||||
|
if pause {
|
||||||
|
return "paused";
|
||||||
|
}
|
||||||
|
const FAILURES: [&str; 14] = [
|
||||||
|
"error",
|
||||||
|
"failed",
|
||||||
|
"failure",
|
||||||
|
"unable to",
|
||||||
|
"cannot ",
|
||||||
|
"can't ",
|
||||||
|
"timed out",
|
||||||
|
"denied",
|
||||||
|
"no such",
|
||||||
|
"exception",
|
||||||
|
"not available",
|
||||||
|
"失敗",
|
||||||
|
"錯誤",
|
||||||
|
"無法",
|
||||||
|
];
|
||||||
|
let head: String = text.to_lowercase().chars().take(160).collect();
|
||||||
|
if FAILURES.iter().any(|needle| head.contains(needle)) {
|
||||||
|
"error"
|
||||||
|
} else {
|
||||||
|
"ok"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn set_run_step(state: &AppState, run_id: &str, step: &str) {
|
pub(crate) async fn set_run_step(state: &AppState, run_id: &str, step: &str) {
|
||||||
let result = sqlx::query(
|
let result = sqlx::query(
|
||||||
"UPDATE runs SET checkpoint = COALESCE(checkpoint, '{}'::jsonb)
|
"UPDATE runs SET checkpoint = COALESCE(checkpoint, '{}'::jsonb)
|
||||||
|
|
@ -2696,11 +2995,23 @@ mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
RunHalt, SCREENSHOT_CAPTION, describe_step, drop_history_screenshots, halt_from_status,
|
RunHalt, SCREENSHOT_CAPTION, describe_step, drop_history_screenshots, halt_from_status,
|
||||||
history_window_start, is_plain_chat, retryable_run_error, screenshot_parts, tool_needs_gui,
|
history_window_start, is_plain_chat, retryable_run_error, screenshot_parts, tool_needs_gui,
|
||||||
tool_needs_sandbox,
|
tool_needs_sandbox, tool_status,
|
||||||
};
|
};
|
||||||
use rig_core::completion::message::{Message, UserContent};
|
use rig_core::completion::message::{Message, UserContent};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_trail_reads_a_tool_result_as_ok_error_timeout_or_pause() {
|
||||||
|
assert_eq!(tool_status(false, false, "clicked element #12"), "ok");
|
||||||
|
assert_eq!(
|
||||||
|
tool_status(false, false, "Error: selector not found"),
|
||||||
|
"error"
|
||||||
|
);
|
||||||
|
assert_eq!(tool_status(false, false, "操作失敗：視窗關閉"), "error");
|
||||||
|
assert_eq!(tool_status(true, false, "tool timed out"), "timed_out");
|
||||||
|
assert_eq!(tool_status(false, true, "需要人先登入"), "paused");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn chat_tools_do_not_need_the_desktop() {
|
fn chat_tools_do_not_need_the_desktop() {
|
||||||
assert!(!tool_needs_sandbox("remember"));
|
assert!(!tool_needs_sandbox("remember"));
|
||||||
|
|
|
||||||
|
|
@ -617,7 +617,7 @@ pub fn describe_cron(expr: &str) -> String {
|
||||||
if let Ok(Some(seconds)) = interval_seconds(expr) {
|
if let Ok(Some(seconds)) = interval_seconds(expr) {
|
||||||
return format!("每隔 {} 分鐘（固定間隔）", seconds / 60);
|
return format!("每隔 {} 分鐘（固定間隔）", seconds / 60);
|
||||||
}
|
}
|
||||||
let parts: Vec<&str> = expr.trim().split_whitespace().collect();
|
let parts: Vec<&str> = expr.split_whitespace().collect();
|
||||||
if parts.len() != 5 {
|
if parts.len() != 5 {
|
||||||
return expr.to_string();
|
return expr.to_string();
|
||||||
}
|
}
|
||||||
|
|
@ -625,36 +625,40 @@ pub fn describe_cron(expr: &str) -> String {
|
||||||
if expr.trim() == "* * * * *" {
|
if expr.trim() == "* * * * *" {
|
||||||
return "每分鐘".into();
|
return "每分鐘".into();
|
||||||
}
|
}
|
||||||
if let Some(rest) = min.strip_prefix("*/") {
|
if let Some(rest) = min.strip_prefix("*/")
|
||||||
if hour == "*" && dom == "*" && month == "*" && dow == "*" {
|
&& hour == "*"
|
||||||
return if rest
|
&& dom == "*"
|
||||||
.parse::<u32>()
|
&& month == "*"
|
||||||
.ok()
|
&& dow == "*"
|
||||||
.is_some_and(|n| n > 0 && 60 % n == 0)
|
{
|
||||||
{
|
return if rest
|
||||||
format!("每 {rest} 分鐘")
|
.parse::<u32>()
|
||||||
} else {
|
.ok()
|
||||||
format!("日曆排程：{expr}")
|
.is_some_and(|n| n > 0 && 60 % n == 0)
|
||||||
};
|
{
|
||||||
}
|
format!("每 {rest} 分鐘")
|
||||||
|
} else {
|
||||||
|
format!("日曆排程：{expr}")
|
||||||
|
};
|
||||||
}
|
}
|
||||||
if min == "0" && hour == "*" && dom == "*" && month == "*" && dow == "*" {
|
if min == "0" && hour == "*" && dom == "*" && month == "*" && dow == "*" {
|
||||||
return "每小時".into();
|
return "每小時".into();
|
||||||
}
|
}
|
||||||
if min == "0" {
|
if min == "0"
|
||||||
if let Some(rest) = hour.strip_prefix("*/") {
|
&& let Some(rest) = hour.strip_prefix("*/")
|
||||||
if dom == "*" && month == "*" && dow == "*" {
|
&& dom == "*"
|
||||||
return if rest
|
&& month == "*"
|
||||||
.parse::<u32>()
|
&& dow == "*"
|
||||||
.ok()
|
{
|
||||||
.is_some_and(|n| n > 0 && 24 % n == 0)
|
return if rest
|
||||||
{
|
.parse::<u32>()
|
||||||
format!("每 {rest} 小時")
|
.ok()
|
||||||
} else {
|
.is_some_and(|n| n > 0 && 24 % n == 0)
|
||||||
format!("日曆排程：{expr}")
|
{
|
||||||
};
|
format!("每 {rest} 小時")
|
||||||
}
|
} else {
|
||||||
}
|
format!("日曆排程：{expr}")
|
||||||
|
};
|
||||||
}
|
}
|
||||||
if min.parse::<u32>().is_ok() && hour.parse::<u32>().is_ok() && month == "*" {
|
if min.parse::<u32>().is_ok() && hour.parse::<u32>().is_ok() && month == "*" {
|
||||||
let at = format!("{hour:0>2}:{min:0>2}");
|
let at = format!("{hour:0>2}:{min:0>2}");
|
||||||
|
|
|
||||||
|
|
@ -241,10 +241,10 @@ async fn proxy_socket(mut client: WebSocket, host: String, port: u16, rest: Stri
|
||||||
if client.send(AxumMessage::Text(text.to_string().into())).await.is_err() { break; }
|
if client.send(AxumMessage::Text(text.to_string().into())).await.is_err() { break; }
|
||||||
}
|
}
|
||||||
Some(Ok(WsMessage::Ping(data))) => {
|
Some(Ok(WsMessage::Ping(data))) => {
|
||||||
if client.send(AxumMessage::Ping(data.into())).await.is_err() { break; }
|
if client.send(AxumMessage::Ping(data)).await.is_err() { break; }
|
||||||
}
|
}
|
||||||
Some(Ok(WsMessage::Pong(data))) => {
|
Some(Ok(WsMessage::Pong(data))) => {
|
||||||
if client.send(AxumMessage::Pong(data.into())).await.is_err() { break; }
|
if client.send(AxumMessage::Pong(data)).await.is_err() { break; }
|
||||||
}
|
}
|
||||||
Some(Ok(WsMessage::Close(_))) | Some(Ok(WsMessage::Frame(_))) | None => break,
|
Some(Ok(WsMessage::Close(_))) | Some(Ok(WsMessage::Frame(_))) | None => break,
|
||||||
Some(Err(_)) => break,
|
Some(Err(_)) => break,
|
||||||
|
|
|
||||||
|
|
@ -220,9 +220,7 @@ async fn delete_session(
|
||||||
Json(json!({"message":"session not found"})),
|
Json(json!({"message":"session not found"})),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
cancel_session_runs(&state, &id)
|
cancel_session_runs(&state, &id).await.map_err(internal)?;
|
||||||
.await
|
|
||||||
.map_err(|error| internal(error))?;
|
|
||||||
let mut tx = state
|
let mut tx = state
|
||||||
.pool()
|
.pool()
|
||||||
.begin()
|
.begin()
|
||||||
|
|
@ -286,9 +284,7 @@ async fn clear_messages(
|
||||||
Json(json!({"message":"session not found"})),
|
Json(json!({"message":"session not found"})),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
cancel_session_runs(&state, &id)
|
cancel_session_runs(&state, &id).await.map_err(internal)?;
|
||||||
.await
|
|
||||||
.map_err(|error| internal(error))?;
|
|
||||||
let mut tx = state
|
let mut tx = state
|
||||||
.pool()
|
.pool()
|
||||||
.begin()
|
.begin()
|
||||||
|
|
@ -364,9 +360,7 @@ async fn stop_session(
|
||||||
Json(json!({"message":"session not found"})),
|
Json(json!({"message":"session not found"})),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
cancel_session_runs(&state, &id)
|
cancel_session_runs(&state, &id).await.map_err(internal)?;
|
||||||
.await
|
|
||||||
.map_err(|error| internal(error))?;
|
|
||||||
Ok(Json(json!({"ok":true})))
|
Ok(Json(json!({"ok":true})))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -447,26 +441,29 @@ pub async fn default_session_for_bot(
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `messages_for_session` projection: message columns joined with the speaking bot.
|
||||||
|
type MessageWithSpeakerRow = (
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
i32,
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
Value,
|
||||||
|
Option<String>,
|
||||||
|
Option<String>,
|
||||||
|
chrono::DateTime<chrono::Utc>,
|
||||||
|
Option<String>,
|
||||||
|
Option<String>,
|
||||||
|
Option<String>,
|
||||||
|
Option<String>,
|
||||||
|
);
|
||||||
|
|
||||||
pub async fn messages_for_session(
|
pub async fn messages_for_session(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
actor: &Actor,
|
actor: &Actor,
|
||||||
id: &str,
|
id: &str,
|
||||||
) -> Result<Vec<SessionMessage>, ApiError> {
|
) -> Result<Vec<SessionMessage>, ApiError> {
|
||||||
let rows: Vec<(
|
let rows: Vec<MessageWithSpeakerRow> = sqlx::query_as(
|
||||||
String,
|
|
||||||
String,
|
|
||||||
i32,
|
|
||||||
String,
|
|
||||||
String,
|
|
||||||
Value,
|
|
||||||
Option<String>,
|
|
||||||
Option<String>,
|
|
||||||
chrono::DateTime<chrono::Utc>,
|
|
||||||
Option<String>,
|
|
||||||
Option<String>,
|
|
||||||
Option<String>,
|
|
||||||
Option<String>,
|
|
||||||
)> = sqlx::query_as(
|
|
||||||
"SELECT m.id, m.thread_id, m.seq, m.role, m.body, m.blocks, m.run_id,
|
"SELECT m.id, m.thread_id, m.seq, m.role, m.body, m.blocks, m.run_id,
|
||||||
m.client_nonce, m.created_at, m.speaker_bot_id, b.name, b.avatar_color, b.avatar_shape
|
m.client_nonce, m.created_at, m.speaker_bot_id, b.name, b.avatar_color, b.avatar_shape
|
||||||
FROM messages m
|
FROM messages m
|
||||||
|
|
|
||||||
|
|
@ -1394,7 +1394,11 @@ pub fn fallback_playbook(goal: &str, events: &[Value]) -> Value {
|
||||||
})
|
})
|
||||||
.filter_map(|event| describe_event(event, t0))
|
.filter_map(|event| describe_event(event, t0))
|
||||||
.map(|line| {
|
.map(|line| {
|
||||||
let text = line.splitn(2, ' ').nth(1).unwrap_or(&line).to_string();
|
let text = line
|
||||||
|
.split_once(' ')
|
||||||
|
.map(|x| x.1)
|
||||||
|
.unwrap_or(&line)
|
||||||
|
.to_string();
|
||||||
json!({ "do": text, "expect": "", "note": "" })
|
json!({ "do": text, "expect": "", "note": "" })
|
||||||
})
|
})
|
||||||
.take(40)
|
.take(40)
|
||||||
|
|
|
||||||
|
|
@ -728,20 +728,20 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
||||||
request[key] = value.clone();
|
request[key] = value.clone();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if request.get("selector").and_then(Value::as_str).is_none() {
|
if request.get("selector").and_then(Value::as_str).is_none()
|
||||||
if let Some(id) = element_id(args.get("element").or_else(|| args.get("id"))) {
|
&& let Some(id) = element_id(args.get("element").or_else(|| args.get("id")))
|
||||||
let elements = ctx.elements.lock().unwrap().clone();
|
{
|
||||||
match elements
|
let elements = ctx.elements.lock().unwrap().clone();
|
||||||
.iter()
|
match elements
|
||||||
.find(|element| u64::from(element.id) == id)
|
.iter()
|
||||||
.and_then(|element| element.selector.clone())
|
.find(|element| u64::from(element.id) == id)
|
||||||
{
|
.and_then(|element| element.selector.clone())
|
||||||
Some(selector) => request["selector"] = json!(selector),
|
{
|
||||||
None if matches!(action, "click" | "type") => {
|
Some(selector) => request["selector"] = json!(selector),
|
||||||
return pause_unknown_element(ctx, id as u32, &elements);
|
None if matches!(action, "click" | "type") => {
|
||||||
}
|
return pause_unknown_element(ctx, id as u32, &elements);
|
||||||
None => {}
|
|
||||||
}
|
}
|
||||||
|
None => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if matches!(action, "click") && request.get("selector").and_then(Value::as_str).is_none() {
|
if matches!(action, "click") && request.get("selector").and_then(Value::as_str).is_none() {
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ use aes_gcm::aead::{Aead, KeyInit};
|
||||||
use aes_gcm::{Aes256Gcm, Nonce};
|
use aes_gcm::{Aes256Gcm, Nonce};
|
||||||
use axum::extract::{Path, State};
|
use axum::extract::{Path, State};
|
||||||
use axum::http::StatusCode;
|
use axum::http::StatusCode;
|
||||||
use axum::routing::{delete, get, patch, post};
|
use axum::routing::{get, patch};
|
||||||
use axum::{Json, Router};
|
use axum::{Json, Router};
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
use rand::RngCore;
|
use rand::RngCore;
|
||||||
|
|
@ -161,14 +161,18 @@ pub async fn list_on(
|
||||||
.map_err(|error| error.to_string())
|
.map_err(|error| error.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_secret(
|
/// `get_secret_on` projection: vault columns plus the encrypted password.
|
||||||
state: &AppState,
|
type SecretRow = (
|
||||||
actor: &Actor,
|
String,
|
||||||
bot_id: &str,
|
String,
|
||||||
account_id: &str,
|
String,
|
||||||
) -> Result<Option<(VaultAccount, String, String)>, String> {
|
String,
|
||||||
get_secret_on(state.pool(), actor, bot_id, account_id).await
|
String,
|
||||||
}
|
String,
|
||||||
|
DateTime<Utc>,
|
||||||
|
DateTime<Utc>,
|
||||||
|
String,
|
||||||
|
);
|
||||||
|
|
||||||
pub async fn get_secret_on(
|
pub async fn get_secret_on(
|
||||||
pool: &sqlx::PgPool,
|
pool: &sqlx::PgPool,
|
||||||
|
|
@ -176,8 +180,7 @@ pub async fn get_secret_on(
|
||||||
bot_id: &str,
|
bot_id: &str,
|
||||||
account_id: &str,
|
account_id: &str,
|
||||||
) -> Result<Option<(VaultAccount, String, String)>, String> {
|
) -> Result<Option<(VaultAccount, String, String)>, String> {
|
||||||
let row: Option<(String, String, String, String, String, String, DateTime<Utc>, DateTime<Utc>, String)> =
|
let row: Option<SecretRow> = sqlx::query_as(
|
||||||
sqlx::query_as(
|
|
||||||
"SELECT id, bot_id, site, host, username, notes, created_at, updated_at, password_ciphertext
|
"SELECT id, bot_id, site, host, username, notes, created_at, updated_at, password_ciphertext
|
||||||
FROM vault_accounts
|
FROM vault_accounts
|
||||||
WHERE id=$1 AND bot_id=$2 AND space_id=$3 AND user_id=$4",
|
WHERE id=$1 AND bot_id=$2 AND space_id=$3 AND user_id=$4",
|
||||||
|
|
@ -368,6 +371,9 @@ mod tests {
|
||||||
use super::{decrypt, encrypt, normalize_host};
|
use super::{decrypt, encrypt, normalize_host};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
// Rust 2024 marks `set_var` unsafe; this only seeds a test-only key and no
|
||||||
|
// other test in this binary reads it.
|
||||||
|
#[allow(unsafe_code)]
|
||||||
fn round_trips_a_password() {
|
fn round_trips_a_password() {
|
||||||
unsafe { std::env::set_var("LAZYBOY_VAULT_KEY", "test-vault-key-for-unit-tests") };
|
unsafe { std::env::set_var("LAZYBOY_VAULT_KEY", "test-vault-key-for-unit-tests") };
|
||||||
let packed = encrypt("s3cret!").unwrap();
|
let packed = encrypt("s3cret!").unwrap();
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
name = "lazyboy-contracts"
|
name = "lazyboy-contracts"
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
publish.workspace = true
|
publish.workspace = true
|
||||||
|
|
||||||
|
|
@ -11,3 +12,6 @@ serde_json.workspace = true
|
||||||
thiserror.workspace = true
|
thiserror.workspace = true
|
||||||
chrono.workspace = true
|
chrono.workspace = true
|
||||||
base64.workspace = true
|
base64.workspace = true
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -113,8 +113,10 @@ pub fn computer_home_key(
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "kebab-case")]
|
#[serde(rename_all = "kebab-case")]
|
||||||
|
#[derive(Default)]
|
||||||
pub enum BrowserProfileMode {
|
pub enum BrowserProfileMode {
|
||||||
Shared,
|
Shared,
|
||||||
|
#[default]
|
||||||
PerBot,
|
PerBot,
|
||||||
PerTask,
|
PerTask,
|
||||||
}
|
}
|
||||||
|
|
@ -129,12 +131,6 @@ impl BrowserProfileMode {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for BrowserProfileMode {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::PerBot
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::str::FromStr for BrowserProfileMode {
|
impl std::str::FromStr for BrowserProfileMode {
|
||||||
type Err = UnknownProfileMode;
|
type Err = UnknownProfileMode;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
name = "lazyboy-control"
|
name = "lazyboy-control"
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
publish.workspace = true
|
publish.workspace = true
|
||||||
|
|
||||||
|
|
@ -15,3 +16,6 @@ hex.workspace = true
|
||||||
chrono.workspace = true
|
chrono.workspace = true
|
||||||
async-trait = "0.1"
|
async-trait = "0.1"
|
||||||
image.workspace = true
|
image.workspace = true
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -130,7 +130,6 @@ mod tests {
|
||||||
kind: Some("a11y".into()),
|
kind: Some("a11y".into()),
|
||||||
selector: Some(format!("0/{id}")),
|
selector: Some(format!("0/{id}")),
|
||||||
role: Some("push button".into()),
|
role: Some("push button".into()),
|
||||||
..UiElement::default()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -200,22 +200,22 @@ pub fn parse_computer_actions(value: &Value) -> Result<Vec<ComputerAction>, Acti
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
match kind {
|
match kind {
|
||||||
"click" | "move" | "down" | "up" => {
|
"click" | "move" | "down" | "up" => {
|
||||||
if kind == "click" {
|
if kind == "click"
|
||||||
if let Some(target) = ref_target(action) {
|
&& let Some(target) = ref_target(action)
|
||||||
let pointer = ComputerAction::Ref {
|
{
|
||||||
verb: RefVerb::Click,
|
let pointer = ComputerAction::Ref {
|
||||||
target,
|
verb: RefVerb::Click,
|
||||||
ref_kind: ref_kind(action),
|
target,
|
||||||
text: None,
|
ref_kind: ref_kind(action),
|
||||||
};
|
text: None,
|
||||||
let doubled = action.get("double").and_then(Value::as_bool) == Some(true);
|
};
|
||||||
actions.push(pointer.clone());
|
let doubled = action.get("double").and_then(Value::as_bool) == Some(true);
|
||||||
if doubled {
|
actions.push(pointer.clone());
|
||||||
actions.push(ComputerAction::Wait { ms: 70 });
|
if doubled {
|
||||||
actions.push(pointer);
|
actions.push(ComputerAction::Wait { ms: 70 });
|
||||||
}
|
actions.push(pointer);
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
let x = coordinate(action.get("x"), "x")?;
|
let x = coordinate(action.get("x"), "x")?;
|
||||||
let y = coordinate(action.get("y"), "y")?;
|
let y = coordinate(action.get("y"), "y")?;
|
||||||
|
|
@ -558,7 +558,6 @@ mod tests {
|
||||||
y: 20,
|
y: 20,
|
||||||
w: 80,
|
w: 80,
|
||||||
h: 24,
|
h: 24,
|
||||||
..UiElement::default()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -150,7 +150,7 @@ fn draw_badge(img: &mut RgbImage, x: u32, y: u32, id: u32) {
|
||||||
2,
|
2,
|
||||||
);
|
);
|
||||||
for (i, ch) in label.bytes().enumerate() {
|
for (i, ch) in label.bytes().enumerate() {
|
||||||
if !(b'0'..=b'9').contains(&ch) {
|
if !ch.is_ascii_digit() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let dx = bx + pad + i as u32 * digit_w;
|
let dx = bx + pad + i as u32 * digit_w;
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ use lazyboy_contracts::{ComputerAction, ComputerCapabilities, ComputerObservatio
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||||
pub struct AdapterContext {
|
pub struct AdapterContext {
|
||||||
pub operation_id: String,
|
pub operation_id: String,
|
||||||
pub space_id: String,
|
pub space_id: String,
|
||||||
|
|
@ -20,23 +20,6 @@ pub struct AdapterContext {
|
||||||
pub profile_path: Option<String>,
|
pub profile_path: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for AdapterContext {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self {
|
|
||||||
operation_id: String::new(),
|
|
||||||
space_id: String::new(),
|
|
||||||
user_id: String::new(),
|
|
||||||
bot_id: None,
|
|
||||||
run_id: None,
|
|
||||||
screen_lease_id: None,
|
|
||||||
screen_id: None,
|
|
||||||
screen_slot: None,
|
|
||||||
display: None,
|
|
||||||
profile_path: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct ComputerRef {
|
pub struct ComputerRef {
|
||||||
pub id: String,
|
pub id: String,
|
||||||
|
|
@ -53,7 +36,7 @@ pub struct ProvisionRequest {
|
||||||
pub provider_ref: Option<String>,
|
pub provider_ref: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||||
pub struct CommandRequest {
|
pub struct CommandRequest {
|
||||||
pub argv: Vec<String>,
|
pub argv: Vec<String>,
|
||||||
pub cwd: Option<String>,
|
pub cwd: Option<String>,
|
||||||
|
|
@ -64,17 +47,6 @@ pub struct CommandRequest {
|
||||||
pub stdin: Option<String>,
|
pub stdin: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for CommandRequest {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self {
|
|
||||||
argv: Vec::new(),
|
|
||||||
cwd: None,
|
|
||||||
timeout_ms: None,
|
|
||||||
stdin: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct CommandResult {
|
pub struct CommandResult {
|
||||||
pub stdout: String,
|
pub stdout: String,
|
||||||
|
|
|
||||||
|
|
@ -386,6 +386,17 @@ pub fn screenshot_command_on(display: &str) -> Vec<String> {
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Native clipboard path; text is supplied on stdin, never process arguments.
|
||||||
|
pub fn paste_command_on(display: &str) -> Vec<String> {
|
||||||
|
vec![
|
||||||
|
"env".into(),
|
||||||
|
display_env(display),
|
||||||
|
"python3".into(),
|
||||||
|
"-c".into(),
|
||||||
|
include_str!("clipboard.py").into(),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
@ -490,14 +501,3 @@ mod tests {
|
||||||
assert!(parse_ui_elements("not-json").is_empty());
|
assert!(parse_ui_elements("not-json").is_empty());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Native clipboard path; text is supplied on stdin, never process arguments.
|
|
||||||
pub fn paste_command_on(display: &str) -> Vec<String> {
|
|
||||||
vec![
|
|
||||||
"env".into(),
|
|
||||||
display_env(display),
|
|
||||||
"python3".into(),
|
|
||||||
"-c".into(),
|
|
||||||
include_str!("clipboard.py").into(),
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
name = "lazyboy-controld"
|
name = "lazyboy-controld"
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
publish.workspace = true
|
publish.workspace = true
|
||||||
|
|
||||||
|
|
@ -10,9 +11,10 @@ lazyboy-contracts.workspace = true
|
||||||
lazyboy-control.workspace = true
|
lazyboy-control.workspace = true
|
||||||
axum.workspace = true
|
axum.workspace = true
|
||||||
tokio.workspace = true
|
tokio.workspace = true
|
||||||
serde.workspace = true
|
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
thiserror.workspace = true
|
|
||||||
tracing.workspace = true
|
tracing.workspace = true
|
||||||
tracing-subscriber.workspace = true
|
tracing-subscriber.workspace = true
|
||||||
base64.workspace = true
|
base64.workspace = true
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -2,12 +2,12 @@
|
||||||
name = "lazyboy-harness"
|
name = "lazyboy-harness"
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
publish.workspace = true
|
publish.workspace = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
lazyboy-contracts.workspace = true
|
lazyboy-contracts.workspace = true
|
||||||
serde.workspace = true
|
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
thiserror.workspace = true
|
thiserror.workspace = true
|
||||||
rig-core.workspace = true
|
rig-core.workspace = true
|
||||||
|
|
@ -21,3 +21,6 @@ http = "1"
|
||||||
reqwest.workspace = true
|
reqwest.workspace = true
|
||||||
rustls.workspace = true
|
rustls.workspace = true
|
||||||
rustls-native-certs = "0.8"
|
rustls-native-certs = "0.8"
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
name = "lazyboy-sandbox"
|
name = "lazyboy-sandbox"
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
publish.workspace = true
|
publish.workspace = true
|
||||||
|
|
||||||
|
|
@ -10,10 +11,8 @@ lazyboy-contracts.workspace = true
|
||||||
lazyboy-control.workspace = true
|
lazyboy-control.workspace = true
|
||||||
async-trait = "0.1"
|
async-trait = "0.1"
|
||||||
reqwest.workspace = true
|
reqwest.workspace = true
|
||||||
serde.workspace = true
|
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
tokio.workspace = true
|
|
||||||
base64.workspace = true
|
base64.workspace = true
|
||||||
chrono.workspace = true
|
|
||||||
sha2.workspace = true
|
[lints]
|
||||||
hex.workspace = true
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -36,20 +36,20 @@ impl DockerSandbox {
|
||||||
if let Some(bot_id) = &context.bot_id {
|
if let Some(bot_id) = &context.bot_id {
|
||||||
headers.insert("x-lazyboy-bot-id", bot_id.parse().unwrap());
|
headers.insert("x-lazyboy-bot-id", bot_id.parse().unwrap());
|
||||||
}
|
}
|
||||||
if let Some(display) = &context.display {
|
if let Some(display) = &context.display
|
||||||
if let Ok(value) = display.parse() {
|
&& let Ok(value) = display.parse()
|
||||||
headers.insert("x-lazyboy-display", value);
|
{
|
||||||
}
|
headers.insert("x-lazyboy-display", value);
|
||||||
}
|
}
|
||||||
if let Some(profile) = &context.profile_path {
|
if let Some(profile) = &context.profile_path
|
||||||
if let Ok(value) = profile.parse() {
|
&& let Ok(value) = profile.parse()
|
||||||
headers.insert("x-lazyboy-profile", value);
|
{
|
||||||
}
|
headers.insert("x-lazyboy-profile", value);
|
||||||
}
|
}
|
||||||
if let Some(slot) = context.screen_slot {
|
if let Some(slot) = context.screen_slot
|
||||||
if let Ok(value) = slot.to_string().parse() {
|
&& let Ok(value) = slot.to_string().parse()
|
||||||
headers.insert("x-lazyboy-screen-slot", value);
|
{
|
||||||
}
|
headers.insert("x-lazyboy-screen-slot", value);
|
||||||
}
|
}
|
||||||
headers
|
headers
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -55,22 +55,22 @@ impl SandboxProvider for FakeSandbox {
|
||||||
request: CommandRequest,
|
request: CommandRequest,
|
||||||
_context: &AdapterContext,
|
_context: &AdapterContext,
|
||||||
) -> Result<CommandResult, SandboxError> {
|
) -> Result<CommandResult, SandboxError> {
|
||||||
if request.argv.get(0).map(String::as_str) == Some("mkdir") {
|
if request.argv.first().map(String::as_str) == Some("mkdir") {
|
||||||
return Ok(CommandResult {
|
return Ok(CommandResult {
|
||||||
stdout: String::new(),
|
stdout: String::new(),
|
||||||
stderr: String::new(),
|
stderr: String::new(),
|
||||||
code: 0,
|
code: 0,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if request.argv.get(0).map(String::as_str) == Some("touch") {
|
if request.argv.first().map(String::as_str) == Some("touch")
|
||||||
if let Some(path) = request.argv.get(1) {
|
&& let Some(path) = request.argv.get(1)
|
||||||
self.files
|
{
|
||||||
.lock()
|
self.files
|
||||||
.unwrap()
|
.lock()
|
||||||
.entry(computer.home_key.clone())
|
.unwrap()
|
||||||
.or_default()
|
.entry(computer.home_key.clone())
|
||||||
.insert(path.clone(), Vec::new());
|
.or_default()
|
||||||
}
|
.insert(path.clone(), Vec::new());
|
||||||
}
|
}
|
||||||
Ok(CommandResult {
|
Ok(CommandResult {
|
||||||
stdout: request.argv.join(" "),
|
stdout: request.argv.join(" "),
|
||||||
|
|
|
||||||
|
|
@ -2,26 +2,26 @@
|
||||||
name = "lazyboy-supervisor"
|
name = "lazyboy-supervisor"
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
publish.workspace = true
|
publish.workspace = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
lazyboy-contracts.workspace = true
|
|
||||||
lazyboy-control.workspace = true
|
lazyboy-control.workspace = true
|
||||||
axum.workspace = true
|
axum.workspace = true
|
||||||
tokio.workspace = true
|
tokio.workspace = true
|
||||||
serde.workspace = true
|
serde.workspace = true
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
thiserror.workspace = true
|
|
||||||
tracing.workspace = true
|
tracing.workspace = true
|
||||||
tracing-subscriber.workspace = true
|
tracing-subscriber.workspace = true
|
||||||
bollard.workspace = true
|
bollard.workspace = true
|
||||||
uuid.workspace = true
|
|
||||||
base64.workspace = true
|
base64.workspace = true
|
||||||
reqwest.workspace = true
|
reqwest.workspace = true
|
||||||
async-trait = "0.1"
|
|
||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
|
|
||||||
hmac.workspace = true
|
hmac.workspace = true
|
||||||
sha2.workspace = true
|
sha2.workspace = true
|
||||||
hex.workspace = true
|
hex.workspace = true
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -34,20 +34,20 @@ pub struct Provisioned {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct ObservePayload {
|
pub struct ObservePayload {
|
||||||
pub png: Vec<u8>,
|
|
||||||
pub json: serde_json::Value,
|
pub json: serde_json::Value,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ObservePayload {
|
impl ObservePayload {
|
||||||
fn from_json(value: serde_json::Value) -> Result<Self, String> {
|
fn from_json(value: serde_json::Value) -> Result<Self, String> {
|
||||||
let encoded = value
|
// The screenshot travels inside `json` as base64; only its presence is
|
||||||
|
// checked here so a broken capture fails fast instead of returning an
|
||||||
|
// empty observation.
|
||||||
|
value
|
||||||
.get("png_base64")
|
.get("png_base64")
|
||||||
.and_then(serde_json::Value::as_str)
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.filter(|encoded| !encoded.is_empty())
|
||||||
.ok_or_else(|| "missing png".to_string())?;
|
.ok_or_else(|| "missing png".to_string())?;
|
||||||
let png = base64::engine::general_purpose::STANDARD
|
Ok(Self { json: value })
|
||||||
.decode(encoded)
|
|
||||||
.map_err(|error| error.to_string())?;
|
|
||||||
Ok(Self { png, json: value })
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -76,7 +76,7 @@ impl DockerHost {
|
||||||
return Err("invalid home key".into());
|
return Err("invalid home key".into());
|
||||||
}
|
}
|
||||||
let expected = PathBuf::from(&data_dir).join("homes").join(home_key);
|
let expected = PathBuf::from(&data_dir).join("homes").join(home_key);
|
||||||
if PathBuf::from(home_path) != expected {
|
if *home_path != expected {
|
||||||
return Err("home path is outside managed homes".into());
|
return Err("home path is outside managed homes".into());
|
||||||
}
|
}
|
||||||
// Work on the container-local path, not the host daemon's bind path.
|
// Work on the container-local path, not the host daemon's bind path.
|
||||||
|
|
@ -251,42 +251,6 @@ impl DockerHost {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn exec(&self, id: &str, request: CommandRequest) -> Result<CommandResult, String> {
|
|
||||||
let cwd = match request.cwd {
|
|
||||||
Some(cwd) if PathBuf::from(&cwd).is_absolute() => cwd,
|
|
||||||
Some(cwd) => {
|
|
||||||
let relative = normalize_workspace_path(&cwd).map_err(|error| error.to_string())?;
|
|
||||||
if relative.is_empty() {
|
|
||||||
HOME.to_string()
|
|
||||||
} else {
|
|
||||||
format!("{HOME}/{relative}")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
None => HOME.to_string(),
|
|
||||||
};
|
|
||||||
let timeout_ms = request.timeout_ms.unwrap_or(30_000).clamp(100, 120_000);
|
|
||||||
let argv = if request.argv.is_empty() {
|
|
||||||
vec!["/bin/echo".into(), "ready".into()]
|
|
||||||
} else {
|
|
||||||
request.argv
|
|
||||||
};
|
|
||||||
let mut bounded = vec![
|
|
||||||
"timeout".into(),
|
|
||||||
"--signal=TERM".into(),
|
|
||||||
"--kill-after=2s".into(),
|
|
||||||
format!("{}s", timeout_ms as f64 / 1000.0),
|
|
||||||
];
|
|
||||||
bounded.extend(argv);
|
|
||||||
self.exec_raw_cmd(
|
|
||||||
id,
|
|
||||||
&bounded,
|
|
||||||
Some(&cwd),
|
|
||||||
&ScreenTarget::default(),
|
|
||||||
request.stdin,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn exec_on(
|
pub async fn exec_on(
|
||||||
&self,
|
&self,
|
||||||
id: &str,
|
id: &str,
|
||||||
|
|
@ -322,13 +286,6 @@ impl DockerHost {
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn observe(&self, id: &str) -> Result<Vec<u8>, String> {
|
|
||||||
Ok(self
|
|
||||||
.observe_payload(id, &ScreenTarget::default())
|
|
||||||
.await?
|
|
||||||
.png)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn observe_payload(
|
pub async fn observe_payload(
|
||||||
&self,
|
&self,
|
||||||
id: &str,
|
id: &str,
|
||||||
|
|
@ -352,23 +309,22 @@ impl DockerHost {
|
||||||
let mut body = serde_json::json!({
|
let mut body = serde_json::json!({
|
||||||
"png_base64": base64::engine::general_purpose::STANDARD.encode(&stdout)
|
"png_base64": base64::engine::general_purpose::STANDARD.encode(&stdout)
|
||||||
});
|
});
|
||||||
if let Ok(meta) = self.pointer_state(id, target).await {
|
if let Ok(meta) = self.pointer_state(id, target).await
|
||||||
if let serde_json::Value::Object(map) = meta {
|
&& let serde_json::Value::Object(map) = meta
|
||||||
if let Some(obj) = body.as_object_mut() {
|
&& let Some(obj) = body.as_object_mut()
|
||||||
if let (Some(x), Some(y)) = (map.get("x"), map.get("y")) {
|
{
|
||||||
obj.insert("cursor".into(), serde_json::json!({ "x": x, "y": y }));
|
if let (Some(x), Some(y)) = (map.get("x"), map.get("y")) {
|
||||||
}
|
obj.insert("cursor".into(), serde_json::json!({ "x": x, "y": y }));
|
||||||
if map
|
}
|
||||||
.get("id")
|
if map
|
||||||
.and_then(serde_json::Value::as_str)
|
.get("id")
|
||||||
.is_some_and(|id| !id.is_empty())
|
.and_then(serde_json::Value::as_str)
|
||||||
{
|
.is_some_and(|id| !id.is_empty())
|
||||||
obj.insert(
|
{
|
||||||
"activeWindow".into(),
|
obj.insert(
|
||||||
serde_json::json!({ "id": map.get("id"), "title": map.get("title") }),
|
"activeWindow".into(),
|
||||||
);
|
serde_json::json!({ "id": map.get("id"), "title": map.get("title") }),
|
||||||
}
|
);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
body
|
body
|
||||||
|
|
|
||||||
|
|
@ -387,10 +387,9 @@ async fn managed_boundary(
|
||||||
.path()
|
.path()
|
||||||
.strip_prefix("/computers/")
|
.strip_prefix("/computers/")
|
||||||
.and_then(|p| p.split('/').next())
|
.and_then(|p| p.split('/').next())
|
||||||
|
&& app.docker.container_control_token(id).await.is_err()
|
||||||
{
|
{
|
||||||
if app.docker.container_control_token(id).await.is_err() {
|
return StatusCode::NOT_FOUND.into_response();
|
||||||
return StatusCode::NOT_FOUND.into_response();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
next.run(req).await
|
next.run(req).await
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,58 @@
|
||||||
|
# Supply-chain policy for the LazyBoy workspace, checked by `make audit`.
|
||||||
|
# cargo-deny is not part of the Rust toolchain:
|
||||||
|
# cargo install --locked cargo-deny (or grab a release binary)
|
||||||
|
#
|
||||||
|
# Three things are enforced: known vulnerabilities (RustSec), the license set we
|
||||||
|
# are willing to ship, and where dependencies may come from.
|
||||||
|
|
||||||
|
[graph]
|
||||||
|
targets = [
|
||||||
|
{ triple = "x86_64-unknown-linux-gnu" },
|
||||||
|
{ triple = "aarch64-unknown-linux-gnu" },
|
||||||
|
]
|
||||||
|
|
||||||
|
[advisories]
|
||||||
|
yanked = "deny"
|
||||||
|
ignore = [
|
||||||
|
# `paste` is an eager-macro helper pulled in by fastembed -> tokenizers. It is
|
||||||
|
# archived upstream (no safe upgrade), only used while compiling macros, and we
|
||||||
|
# keep embeddings optional. Re-check on the next fastembed/tokenizers bump.
|
||||||
|
{ id = "RUSTSEC-2024-0436", reason = "transitive proc-macro helper behind fastembed; unmaintained, no upgrade available" },
|
||||||
|
]
|
||||||
|
|
||||||
|
[licenses]
|
||||||
|
# Permissive set only; copyleft candidates (MPL-2.0) stay file-level.
|
||||||
|
confidence-threshold = 0.93
|
||||||
|
allow = [
|
||||||
|
"0BSD",
|
||||||
|
"Apache-2.0",
|
||||||
|
"Apache-2.0 WITH LLVM-exception",
|
||||||
|
"BSD-2-Clause",
|
||||||
|
"BSD-3-Clause",
|
||||||
|
"BSL-1.0",
|
||||||
|
"CC0-1.0",
|
||||||
|
"CDLA-Permissive-2.0",
|
||||||
|
"ISC",
|
||||||
|
"MIT",
|
||||||
|
"MIT-0",
|
||||||
|
"MPL-2.0",
|
||||||
|
"Unicode-3.0",
|
||||||
|
"Unlicense",
|
||||||
|
"Zlib",
|
||||||
|
]
|
||||||
|
|
||||||
|
[bans]
|
||||||
|
# Duplicate major/minor versions are normal in an application graph (they are
|
||||||
|
# compiled once each); we only want to know when a dependency disappears.
|
||||||
|
multiple-versions = "allow"
|
||||||
|
wildcards = "deny"
|
||||||
|
# Workspace members are path dependencies and carry no version requirement.
|
||||||
|
allow-workspace = true
|
||||||
|
allow-wildcard-paths = true
|
||||||
|
highlight = "all"
|
||||||
|
|
||||||
|
[sources]
|
||||||
|
unknown-registry = "deny"
|
||||||
|
unknown-git = "deny"
|
||||||
|
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
|
||||||
|
allow-git = []
|
||||||
|
|
@ -1,5 +1,14 @@
|
||||||
# Opt-in host access for local Rust development only.
|
# Opt-in host access for local Rust development only.
|
||||||
|
# The project `database` network is internal, and Docker silently ignores published
|
||||||
|
# ports for containers attached only to internal networks. Postgres therefore also
|
||||||
|
# joins this host bridge, so 127.0.0.1:5434 really listens for `cargo test`.
|
||||||
services:
|
services:
|
||||||
postgres:
|
postgres:
|
||||||
ports:
|
ports:
|
||||||
- "127.0.0.1:5434:5432"
|
- "127.0.0.1:5434:5432"
|
||||||
|
networks:
|
||||||
|
- database
|
||||||
|
- dev_host
|
||||||
|
networks:
|
||||||
|
dev_host:
|
||||||
|
name: lazyboy_dev_host
|
||||||
|
|
|
||||||
|
|
@ -21,9 +21,12 @@ npm run dev # http://127.0.0.1:5173
|
||||||
### 檢查與測試
|
### 檢查與測試
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make fmt
|
# 有資料庫的 Rust 測試需要 127.0.0.1:5434，先用 dev overlay 啟動 Postgres
|
||||||
make clippy
|
make postgres
|
||||||
make test
|
|
||||||
|
make lint # clippy（-D warnings），政策見下方
|
||||||
|
make test # Rust 測試（需要 127.0.0.1:5434）
|
||||||
|
make audit # 依賴漏洞、授權、來源掃描（需要 cargo-deny）
|
||||||
|
|
||||||
cd apps/web
|
cd apps/web
|
||||||
npm run typecheck
|
npm run typecheck
|
||||||
|
|
@ -34,11 +37,46 @@ node --test tests/frontend.test.mjs
|
||||||
python3 tests/control.test.py
|
python3 tests/control.test.py
|
||||||
python3 tests/log-rotation.test.py
|
python3 tests/log-rotation.test.py
|
||||||
|
|
||||||
# 需要已啟動的 PostgreSQL Compose service
|
# Python 整合測試用 docker compose exec 連進 Postgres，自己建一次性資料庫後清掉
|
||||||
docker compose up -d postgres
|
|
||||||
python3 tests/retention.test.py
|
python3 tests/retention.test.py
|
||||||
|
python3 tests/run-resume.test.py
|
||||||
|
python3 tests/run-activity.test.py
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`make postgres` 才會發布 `127.0.0.1:5434`（`docker-compose.dev.yml`）；整套堆疊已經用
|
||||||
|
基礎設定跑著時，請改用 `export COMPOSE_FILE=docker-compose.yml:docker-compose.dev.yml`
|
||||||
|
再 `docker compose up -d`，否則後續的 `docker compose` 指令會拿基礎設定重建 Postgres、
|
||||||
|
把發布埠拿掉。
|
||||||
|
|
||||||
|
### Rust 品質檢查
|
||||||
|
|
||||||
|
工具鏈以穩定版為準：workspace 宣告 `rust-version = "1.98"`（MSRV，代表 let-chains
|
||||||
|
等語法下限），容器用 `rust:1-*` 映像會自動跟最新小版。本機更新只需：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
rustup update stable && rustc --version
|
||||||
|
```
|
||||||
|
|
||||||
|
Lint 政策集中在三處，新增 crate 時只要補 `[lints] workspace = true` 就會繼承：
|
||||||
|
|
||||||
|
- 根 `Cargo.toml` 的 `[workspace.lints]`：不安全程式碼預設警告、未使用的
|
||||||
|
`Result`/`Future` 直接拒絕，並用 `unused_crate_dependencies` 抓多餘依賴
|
||||||
|
（不需要另外裝 `cargo-machete`）。
|
||||||
|
- `clippy.toml`：閾值類設定。`too-many-arguments-threshold = 10` 是因為 handler
|
||||||
|
與 run/vault/voice 協助函式本來就要帶 state + actor + 多個 id，超過 10 個參數才會警告。
|
||||||
|
`cargo-clippy` 只看得到啟動目錄下的 `clippy.toml`，請一律在 repo 根目錄跑 `make lint`。
|
||||||
|
- 各 crate 的 `[lints] workspace = true`。
|
||||||
|
|
||||||
|
要放寬一條 lint 時，請在**呼叫點**加 `#[allow(...)]` 並附一句理由（例：
|
||||||
|
`crates/api/src/vault.rs` 測試裡的 `#[allow(unsafe_code)]`），不要在工作區層級關掉規則。
|
||||||
|
|
||||||
|
`make fmt` 會重排整個 workspace；目前倉庫仍有歷史格式偏差，`make fmt-check` 會列出來，
|
||||||
|
等到一次性重整後再併入 CI。CI 目前只需 `make lint` + `make test`。
|
||||||
|
|
||||||
|
供應鏈檢查用 `cargo-deny`（`make audit`，設定見 `deny.toml`）：RustSec 漏洞、授權白名單、
|
||||||
|
依賴來源。它不是內建工具，第一次要先 `cargo install --locked cargo-deny`。
|
||||||
|
已知無法升級的項目會寫進 `deny.toml` 的 `ignore` 並附原因與重檢時機。
|
||||||
|
|
||||||
### 專案結構
|
### 專案結構
|
||||||
|
|
||||||
```text
|
```text
|
||||||
|
|
@ -59,6 +97,8 @@ LazyBoy/
|
||||||
├── docs/hero.html README 首頁視覺原稿
|
├── docs/hero.html README 首頁視覺原稿
|
||||||
├── docs/workflow.html 可互動產品流程圖
|
├── docs/workflow.html 可互動產品流程圖
|
||||||
├── docker-compose.yml 正式堆疊
|
├── docker-compose.yml 正式堆疊
|
||||||
|
├── clippy.toml Clippy 閾值（要在 repo 根目錄執行才讀得到）
|
||||||
|
├── deny.toml cargo-deny 供應鏈政策（make audit）
|
||||||
└── Makefile 常用開發與部署指令
|
└── Makefile 常用開發與部署指令
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -77,6 +77,37 @@ docker compose up -d api
|
||||||
- 暫時性跨網存取建議維持 `127.0.0.1` 綁定改用隧道：`ssh -L 3101:127.0.0.1:3101 <host>`。
|
- 暫時性跨網存取建議維持 `127.0.0.1` 綁定改用隧道：`ssh -L 3101:127.0.0.1:3101 <host>`。
|
||||||
- 桌面 noVNC 走 `LAZYBOY_SCREEN_NETWORK` 這條 internal 網路，不佔主機埠；同機跑多組 LazyBoy 時請為每組取不同名稱，compose 與 supervisor 會共用同一個值。
|
- 桌面 noVNC 走 `LAZYBOY_SCREEN_NETWORK` 這條 internal 網路，不佔主機埠；同機跑多組 LazyBoy 時請為每組取不同名稱，compose 與 supervisor 會共用同一個值。
|
||||||
|
|
||||||
|
## 執行記錄與錯誤診斷
|
||||||
|
|
||||||
|
任務執行中的每一步會寫進 `run_activity`：模型每一輪在想什麼、哪個動作成功或
|
||||||
|
失敗、花了幾秒、現在是第幾輪。這屬於診斷資料，不是對話內容，因此可以被清理。
|
||||||
|
|
||||||
|
**看即時記錄。** 在對話框把滑鼠移到「思考中」的頭像上（或點一下釘住、用 `Tab`
|
||||||
|
聚焦），會浮出即時記錄面板：
|
||||||
|
|
||||||
|
- 標題列顯示「第 {n}/{limit} 輪」、已運行時間與現在在做什麼；goal 模式沒有輪次
|
||||||
|
上限時只顯示目前輪次，不會假裝有上限。
|
||||||
|
- 記錄由舊到新排列並自動捲到底，包含模型回合、動作成敗與逾時、重試第幾次，以及
|
||||||
|
停下來等你的原因，最後停在哪裡一目了然。
|
||||||
|
- 「複製記錄」可把整份純文字記錄貼給別人除錯；錯誤原文以等寬字型原樣顯示。
|
||||||
|
- 面板只在開啟時每 1.5 秒增量抓取 `GET /api/runs/{id}/activity`，run 結束
|
||||||
|
（完成／失敗／取消）後停止輪詢，不開就完全不打 API。
|
||||||
|
|
||||||
|
**出錯時說什麼。** 任務失敗會在對話框出現錯誤卡，而不是一行紅字：
|
||||||
|
|
||||||
|
- 一句人話講清楚原因與下一步，例如「模型不認得這個 API 金鑰。到「設定 →
|
||||||
|
模型」重新貼一次金鑰，再按重試。」
|
||||||
|
- 按鈕依錯誤類型給：`重試`（從中斷的地方接著做，保留 checkpoint，不重複已成功的
|
||||||
|
步驟）、`開模型設定`、`打開它的畫面`。金鑰錯誤不會只給重試，電腦消失也不會叫
|
||||||
|
你去看設定。
|
||||||
|
- `ⓘ` 直接展開同一個即時記錄面板看詳細 log；錯誤代碼與原文都在裡面。
|
||||||
|
- 重試走 `POST /api/runs/{id}/retry`，只對失敗或已取消的 run 生效；同一台電腦還有
|
||||||
|
別的工作在跑時會被擋下，並明確告訴你是因為還有別的工作。
|
||||||
|
|
||||||
|
**保留多久。** `LAZYBOY_RUN_ACTIVITY_RETENTION_DAYS` 控制記錄保留天數，預設 7 天，
|
||||||
|
每小時清理一次。想留更久的除錯軌跡就調大，在意資料庫體積就調小；run 本身被
|
||||||
|
`LAZYBOY_RUN_RETENTION_DAYS` 清掉時，它的記錄一併消失。
|
||||||
|
|
||||||
## 網站連線驗證
|
## 網站連線驗證
|
||||||
|
|
||||||
AI 遇到可辨識的 Cloudflare 連線驗證頁時，會先取得最新桌面截圖，再使用
|
AI 遇到可辨識的 Cloudflare 連線驗證頁時，會先取得最新桌面截圖，再使用
|
||||||
|
|
@ -95,3 +126,31 @@ AI 遇到可辨識的 Cloudflare 連線驗證頁時，會先取得最新桌面
|
||||||
這會重建桌面容器，讓 `lazyboy` 使用者能執行免密碼 `sudo`；可用
|
這會重建桌面容器，讓 `lazyboy` 使用者能執行免密碼 `sudo`；可用
|
||||||
`sudo -n id -u` 檢查，預期輸出 `0`。既有容器只暫停／恢復不會套用此設定。
|
`sudo -n id -u` 檢查，預期輸出 `0`。既有容器只暫停／恢復不會套用此設定。
|
||||||
重新啟動前請儲存工作；家目錄保留，容器系統層自行安裝的套件需重新安裝。
|
重新啟動前請儲存工作；家目錄保留，容器系統層自行安裝的套件需重新安裝。
|
||||||
|
|
||||||
|
## Postgres collation 版本不符
|
||||||
|
|
||||||
|
`pgvector/pgvector:pg16` 是浮動 tag。重新 `docker compose pull` 之後，容器內的 glibc
|
||||||
|
版本可能和 `pgdata` 卷建立時不同，Postgres 會在使用每個資料庫時抱怨：
|
||||||
|
|
||||||
|
```text
|
||||||
|
WARNING: database "template1" has a collation version mismatch
|
||||||
|
DETAIL: The database was created using collation version 2.41, but the operating system provides version 2.36.
|
||||||
|
```
|
||||||
|
|
||||||
|
此時 `CREATE DATABASE` 直接被拒（`ERROR: template database "template1" has a collation
|
||||||
|
version mismatch`），凡是需要建立資料庫的測試都會失敗：`cargo test` 裡的 `#[sqlx::test]`
|
||||||
|
表現在連線逾時（`PoolTimedOut`），`tests/retention.test.py`、`tests/run-resume.test.py`
|
||||||
|
則在 `CREATE DATABASE` 就掛掉。
|
||||||
|
|
||||||
|
原地修好，資料不遺失（會重建索引，資料量大時安排在離峰）：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose stop api
|
||||||
|
make pg-collation
|
||||||
|
docker compose up -d api
|
||||||
|
```
|
||||||
|
|
||||||
|
它對 `template1`、`postgres`、`lazyboy` 各跑一次 `REINDEX DATABASE` 與
|
||||||
|
`ALTER DATABASE ... REFRESH COLLATION VERSION`。collation provider 是 libc，排序規則
|
||||||
|
真的可能跟著 glibc 變，所以先重建索引再更新記錄的版號，不要只改版號。想避免重複發生，
|
||||||
|
把 Postgres 映像改成固定 digest，讓同一個資料卷不會被不同 glibc 開起來。
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
-- Per-run activity trail for the chat's live monitor bubble. Diagnostics only:
|
||||||
|
-- never user-authored content, and cheap enough to write once per model turn.
|
||||||
|
CREATE TABLE IF NOT EXISTS run_activity (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
run_id TEXT NOT NULL REFERENCES runs (id) ON DELETE CASCADE,
|
||||||
|
kind TEXT NOT NULL,
|
||||||
|
payload JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS run_activity_run_idx ON run_activity (run_id, id);
|
||||||
|
CREATE INDEX IF NOT EXISTS run_activity_retention_idx ON run_activity (created_at);
|
||||||
|
|
@ -241,3 +241,90 @@ test('view-only pointer controls never send mouse input and trackpad asks for co
|
||||||
assert.equal(v.pointerEvents.length,0);
|
assert.equal(v.pointerEvents.length,0);
|
||||||
assert.equal(v.sent.at(-1).type,'lazyboy-request-control');
|
assert.equal(v.sent.at(-1).type,'lazyboy-request-control');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const monitorJs=ts.transpileModule(fs.readFileSync('apps/web/src/run-monitor.tsx','utf8'),{compilerOptions:{module:ts.ModuleKind.CommonJS,jsx:ts.JsxEmit.ReactJSX}}).outputText;
|
||||||
|
const monitorBox={exports:{},require:name=>{
|
||||||
|
if(name==='react')return{useCallback:fn=>fn,useEffect:()=>{},useRef:()=>({current:null}),useState:()=>[null,()=>{}]};
|
||||||
|
if(name==='react/jsx-runtime')return{jsx:()=>null,jsxs:()=>null,Fragment:'Fragment'};
|
||||||
|
if(name==='./api')return{api:async()=>({activity:[]})};
|
||||||
|
if(name==='./i18n')return{t:i18n.t,getLocale:()=>i18n.locale};
|
||||||
|
throw new Error('unexpected import '+name);
|
||||||
|
}};
|
||||||
|
vm.runInNewContext(monitorJs,monitorBox);
|
||||||
|
const {formatElapsed,shortDuration,errorActions,errorTitle,trailText}=monitorBox.exports;
|
||||||
|
const FAILURE_CODES=['interrupted','tool_timeout','model_key','model_quota','model_unknown','model_timeout','network','computer_gone','lease_lost','unknown'];
|
||||||
|
|
||||||
|
test('run timings stay on one glanceable line',()=>{
|
||||||
|
assert.equal(formatElapsed(0),'0:00');
|
||||||
|
assert.equal(formatElapsed(9_500),'0:09');
|
||||||
|
assert.equal(formatElapsed(65_000),'1:05');
|
||||||
|
assert.equal(formatElapsed(3_725_000),'1:02:05');
|
||||||
|
assert.equal(formatElapsed(-5),'0:00');
|
||||||
|
assert.equal(formatElapsed(NaN),'0:00');
|
||||||
|
assert.equal(shortDuration(340),'340ms');
|
||||||
|
assert.equal(shortDuration(6_400),'6.4s');
|
||||||
|
assert.equal(shortDuration(65_000),'1:05');
|
||||||
|
assert.equal(shortDuration(null),'');
|
||||||
|
assert.equal(shortDuration(-1),'');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('every failure code offers at least one action, in the right order',()=>{
|
||||||
|
const buttons=code=>errorActions(code).join(',');
|
||||||
|
assert.equal(buttons('model_key'),'settings,retry');
|
||||||
|
assert.equal(buttons('model_unknown'),'settings,retry');
|
||||||
|
assert.equal(buttons('model_quota'),'retry,settings');
|
||||||
|
assert.equal(buttons('model_timeout'),'retry,settings');
|
||||||
|
assert.equal(buttons('network'),'retry,settings');
|
||||||
|
assert.equal(buttons('computer_gone'),'screen,retry');
|
||||||
|
assert.equal(buttons('tool_timeout'),'screen,retry');
|
||||||
|
assert.equal(buttons('interrupted'),'screen,retry');
|
||||||
|
assert.equal(buttons('lease_lost'),'retry');
|
||||||
|
for(const code of [...FAILURE_CODES,'made_up',undefined,null]){
|
||||||
|
const actions=errorActions(code);
|
||||||
|
assert.ok(actions.length>0,String(code));
|
||||||
|
assert.ok(actions.every(action=>['retry','screen','settings'].includes(action)),code);
|
||||||
|
assert.equal(new Set(actions).size,actions.length,'no duplicate buttons for '+code);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('failure titles resolve in both catalogs and never repeat themselves',()=>{
|
||||||
|
for(const locale of Object.keys(catalogs)){
|
||||||
|
i18n.locale=locale;
|
||||||
|
const titles=FAILURE_CODES.map(code=>errorTitle(code));
|
||||||
|
for(const [index,title] of titles.entries()){
|
||||||
|
assert.notEqual(title,title.startsWith('errorTitle')?title:`${title}-missing`,FAILURE_CODES[index]);
|
||||||
|
assert.ok(!title.startsWith('errorTitle'),`${locale} ${FAILURE_CODES[index]} has no copy`);
|
||||||
|
assert.ok(title.trim().length>0);
|
||||||
|
}
|
||||||
|
assert.equal(new Set(titles).size,titles.length,locale);
|
||||||
|
assert.ok(errorTitle('made_up').length>0);
|
||||||
|
}
|
||||||
|
i18n.locale='en';
|
||||||
|
});
|
||||||
|
|
||||||
|
test('trail lines read as sentences with the detail a stuck run needs',()=>{
|
||||||
|
assert.equal(trailText({id:1,kind:'model',createdAt:'',turn:3,elapsedMs:6400,text:'先打開網頁'}),'turn 3, thought for 6.4s — 先打開網頁');
|
||||||
|
assert.match(trailText({id:2,kind:'tool',createdAt:'',step:'browser: click #12',status:'ok',elapsedMs:400}),/browser: click #12 · ok · 400ms/);
|
||||||
|
assert.match(trailText({id:3,kind:'tool',createdAt:'',step:'shell: npm test',status:'timed_out',snippet:'killed after 150s'}),/timed out — killed after 150s/);
|
||||||
|
assert.match(trailText({id:4,kind:'run',createdAt:'',event:'started',task:'整理下載資料'}),/整理下載資料/);
|
||||||
|
assert.match(trailText({id:5,kind:'run',createdAt:'',event:'completed',turns:9}),/Done in 9 turns/);
|
||||||
|
assert.match(trailText({id:6,kind:'run',createdAt:'',event:'waiting_input',reason:'登入'}),/Waiting for you: 登入/);
|
||||||
|
assert.match(trailText({id:7,kind:'run',createdAt:'',event:'retry'}),/Re-queued/);
|
||||||
|
assert.match(trailText({id:8,kind:'retry',createdAt:'',attempt:2,gaveUp:true,error:'429 rate limit'}),/attempt 2 failed, retrying · gave up — 429 rate limit/);
|
||||||
|
assert.match(trailText({id:9,kind:'notice',createdAt:'',text:'這輪不需要電腦'}),/這輪不需要電腦/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('the bubble reads the run activity endpoint the API actually mounts',()=>{
|
||||||
|
const monitor=fs.readFileSync('crates/api/src/monitor.rs','utf8');
|
||||||
|
assert.match(monitor,/\.route\("\/api\/runs\/\{id\}\/activity", get\(activity\)\)/);
|
||||||
|
assert.match(monitor,/\.route\("\/api\/runs\/\{id\}\/retry", post\(retry\)\)/);
|
||||||
|
const app=fs.readFileSync('apps/web/src/App.tsx','utf8');
|
||||||
|
assert.match(app,/chip\.kind==="error"\?/);
|
||||||
|
assert.match(app,/errorActions\(chip\.code\)/);
|
||||||
|
assert.match(app,/`\/api\/runs\/\$\{runId\}\/retry`,\{method:"POST",body:"\{\}"\}/);
|
||||||
|
assert.match(app,/<RunProbe runId=\{chip\.runId\|\|null\} align="end" label=\{t\("errorDetails"\)\}/);
|
||||||
|
assert.match(app,/<RunProbe runId=\{member\.id===computer\.botId\?computer\.busyRunId:null\}><Avatar/);
|
||||||
|
const probe=fs.readFileSync('apps/web/src/run-monitor.tsx','utf8');
|
||||||
|
assert.match(probe,/`\/api\/runs\/\$\{runId\}\/activity\$\{after\}`/);
|
||||||
|
assert.match(fs.readFileSync('apps/web/src/main.tsx','utf8'),/import "\.\/monitor\.css";/);
|
||||||
|
});
|
||||||
|
|
|
||||||
|
|
@ -44,6 +44,11 @@ try:
|
||||||
FROM unnest(ARRAY['completed','failed','cancelled','running','queued','leased','waiting_input','waiting_takeover']) status;
|
FROM unnest(ARRAY['completed','failed','cancelled','running','queued','leased','waiting_input','waiting_takeover']) status;
|
||||||
INSERT INTO runs(id,space_id,bot_id,thread_id,user_id,status,checkpoint,updated_at,completed_at)
|
INSERT INTO runs(id,space_id,bot_id,thread_id,user_id,status,checkpoint,updated_at,completed_at)
|
||||||
VALUES ('recent-completed','s','b','t','u','completed','{"keep":true}',now(),now());
|
VALUES ('recent-completed','s','b','t','u','completed','{"keep":true}',now(),now());
|
||||||
|
INSERT INTO run_activity(run_id,kind,payload,created_at)
|
||||||
|
SELECT run_id,'notice','{"text":"step from an old run"}',now()-interval '10 days'
|
||||||
|
FROM unnest(ARRAY['completed','failed','running','recent-completed']) run_id;
|
||||||
|
INSERT INTO run_activity(run_id,kind,payload,created_at)
|
||||||
|
VALUES ('recent-completed','notice','{"text":"fresh step"}',now());
|
||||||
INSERT INTO memory_items(id,space_id,user_id,bot_id,content,revision,source_run_id)
|
INSERT INTO memory_items(id,space_id,user_id,bot_id,content,revision,source_run_id)
|
||||||
VALUES ('00000000-0000-0000-0000-000000000001','s','u','b','keep current memory',15,'completed');
|
VALUES ('00000000-0000-0000-0000-000000000001','s','u','b','keep current memory',15,'completed');
|
||||||
INSERT INTO memory_revisions(memory_id,revision,space_id,user_id,bot_id,content,importance,action)
|
INSERT INTO memory_revisions(memory_id,revision,space_id,user_id,bot_id,content,importance,action)
|
||||||
|
|
@ -69,6 +74,8 @@ try:
|
||||||
clean('runs',90)
|
clean('runs',90)
|
||||||
check('(SELECT count(*) FROM runs)=6')
|
check('(SELECT count(*) FROM runs)=6')
|
||||||
check("EXISTS(SELECT 1 FROM memory_items WHERE content='keep current memory' AND source_run_id IS NULL)")
|
check("EXISTS(SELECT 1 FROM memory_items WHERE content='keep current memory' AND source_run_id IS NULL)")
|
||||||
|
clean('run_activity',7)
|
||||||
|
check("(SELECT count(*) FROM run_activity)=1 AND EXISTS(SELECT 1 FROM run_activity WHERE run_id='recent-completed' AND payload->>'text'='fresh step')")
|
||||||
clean('recordings',30)
|
clean('recordings',30)
|
||||||
check("(SELECT count(*) FROM taught_skills WHERE recording='{}')=3 AND (SELECT count(*) FROM taught_skills WHERE playbook->>'keep'='true')=5")
|
check("(SELECT count(*) FROM taught_skills WHERE recording='{}')=3 AND (SELECT count(*) FROM taught_skills WHERE playbook->>'keep'='true')=5")
|
||||||
clean('revisions',90)
|
clean('revisions',90)
|
||||||
|
|
@ -80,9 +87,9 @@ try:
|
||||||
check("(SELECT count(*) FROM computer_execution_leases)=1 AND EXISTS(SELECT 1 FROM computer_execution_leases WHERE id='active')")
|
check("(SELECT count(*) FROM computer_execution_leases)=1 AND EXISTS(SELECT 1 FROM computer_execution_leases WHERE id='active')")
|
||||||
check("(SELECT count(*) FROM computer_profile_locks)=1 AND EXISTS(SELECT 1 FROM computer_profile_locks WHERE profile_key='active')")
|
check("(SELECT count(*) FROM computer_profile_locks)=1 AND EXISTS(SELECT 1 FROM computer_profile_locks WHERE profile_key='active')")
|
||||||
check("EXISTS(SELECT 1 FROM messages WHERE body='keep my conversation')")
|
check("EXISTS(SELECT 1 FROM messages WHERE body='keep my conversation')")
|
||||||
for name,age in [('events',30),('checkpoints',7),('runs',90),('recordings',30),('revisions',90),('deleted_memories',90),('leases',7),('profile_locks',7)]:
|
for name,age in [('events',30),('checkpoints',7),('runs',90),('run_activity',7),('recordings',30),('revisions',90),('deleted_memories',90),('leases',7),('profile_locks',7)]:
|
||||||
clean(name,age)
|
clean(name,age)
|
||||||
check('(SELECT count(*) FROM runs)=6 AND (SELECT count(*) FROM memory_revisions)=9')
|
check('(SELECT count(*) FROM runs)=6 AND (SELECT count(*) FROM memory_revisions)=9')
|
||||||
print('PASS: all 8 retention rules; bounded batches; live work and user content preserved; repeat-safe')
|
print('PASS: all 9 retention rules; bounded batches; live work and user content preserved; repeat-safe')
|
||||||
finally:
|
finally:
|
||||||
sql(f'DROP DATABASE {DB};','postgres')
|
sql(f'DROP DATABASE {DB};','postgres')
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,210 @@
|
||||||
|
"""Integration regression: the run trail the hover bubble reads, and the manual
|
||||||
|
retry that re-queues a dead run without losing where it stopped.
|
||||||
|
Requires the project's Postgres Compose service; never modifies the app database.
|
||||||
|
"""
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
DB = 'run_activity_test_' + uuid.uuid4().hex
|
||||||
|
MONITOR_RS = (ROOT/'crates/api/src/monitor.rs').read_text()
|
||||||
|
RUNS_RS = (ROOT/'crates/api/src/runs.rs').read_text()
|
||||||
|
ACTIVITY_SQL = (ROOT/'crates/api/src/retention/run_activity.sql').read_text()
|
||||||
|
|
||||||
|
|
||||||
|
def sql(text, database=DB):
|
||||||
|
result = subprocess.run(['docker','compose','exec','-T','postgres','psql','-X','-q','-v','ON_ERROR_STOP=1','-U','lazyboy','-d',database], input=text, text=True, capture_output=True, cwd=ROOT)
|
||||||
|
if result.returncode:
|
||||||
|
raise AssertionError(result.stderr)
|
||||||
|
return result.stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def query(text):
|
||||||
|
result = subprocess.run(['docker','compose','exec','-T','postgres','psql','-X','-q','-t','-A','-U','lazyboy','-d',DB], input=text, text=True, capture_output=True, cwd=ROOT)
|
||||||
|
if result.returncode:
|
||||||
|
raise AssertionError(result.stderr)
|
||||||
|
return result.stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def check(condition, label):
|
||||||
|
if query(f"SELECT ({condition});") != 't':
|
||||||
|
raise AssertionError(f'run-activity assertion failed: {label}')
|
||||||
|
|
||||||
|
|
||||||
|
def flatten(text):
|
||||||
|
return re.sub(r'\s+',' ',text).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def trail(run, after=None, limit=60):
|
||||||
|
"""The handler's paging query: newest window first read, then forward by id."""
|
||||||
|
cursor = 'NULL' if after is None else str(after)
|
||||||
|
statement = """SELECT id FROM ( SELECT id FROM run_activity WHERE run_id='%s' AND (%s::bigint IS NULL OR id>%s) ORDER BY id DESC LIMIT %s ) recent ORDER BY id ASC""" % (run, cursor, cursor, limit)
|
||||||
|
return query(statement)
|
||||||
|
|
||||||
|
|
||||||
|
def row_ids(run):
|
||||||
|
return query(f"SELECT id FROM run_activity WHERE run_id='{run}' ORDER BY id;").split()
|
||||||
|
|
||||||
|
|
||||||
|
def retry(run):
|
||||||
|
"""The handler's retry UPDATE, verbatim apart from the bind parameters."""
|
||||||
|
statement = """
|
||||||
|
UPDATE runs SET status='queued', retry_count=0, error=NULL, completed_at=NULL,
|
||||||
|
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||||||
|
WHERE id='%s' AND space_id='s' AND user_id='u'
|
||||||
|
AND status IN ('failed','cancelled')
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM runs a
|
||||||
|
WHERE a.bot_id=runs.bot_id AND a.id<>runs.id
|
||||||
|
AND a.status IN ('leased','running','waiting_input','waiting_takeover')
|
||||||
|
AND (a.lease_expires_at IS NULL OR a.lease_expires_at >= now())
|
||||||
|
)
|
||||||
|
RETURNING thread_id""" % run
|
||||||
|
return query(statement) or '<null>'
|
||||||
|
|
||||||
|
|
||||||
|
# The bubble is only as honest as what the run loop writes. Pin the shapes so a
|
||||||
|
# silent rename shows up here instead of as a blank panel.
|
||||||
|
monitor = flatten(MONITOR_RS)
|
||||||
|
for fragment in [
|
||||||
|
'.route("/api/runs/{id}/activity", get(activity))',
|
||||||
|
'.route("/api/runs/{id}/retry", post(retry))',
|
||||||
|
'INSERT INTO run_activity (run_id,kind,payload)',
|
||||||
|
'SELECT id, kind, payload, created_at FROM run_activity WHERE run_id=$1 AND ($2::bigint IS NULL OR id>$2) ORDER BY id DESC LIMIT $3',
|
||||||
|
"(checkpoint->>'turn')::bigint AS turn",
|
||||||
|
"(checkpoint->>'turnLimit')::bigint AS turn_limit",
|
||||||
|
"AND status IN ('failed','cancelled')",
|
||||||
|
'a.bot_id=runs.bot_id AND a.id<>runs.id',
|
||||||
|
"a.status IN ('leased','running','waiting_input','waiting_takeover')",
|
||||||
|
'"run is not retryable"',
|
||||||
|
]:
|
||||||
|
assert flatten(fragment) in monitor, f'monitor SQL changed, update tests/run-activity.test.py: {fragment}'
|
||||||
|
|
||||||
|
runs_source = flatten(RUNS_RS)
|
||||||
|
for fragment in [
|
||||||
|
'"kind": "error"',
|
||||||
|
'"code": failure.code',
|
||||||
|
'"Worker interrupted after tool execution',
|
||||||
|
"'step', $2::text, 'stepAt', now(), 'turn', $3::bigint, 'turnLimit', $4::bigint",
|
||||||
|
'"event": "started"',
|
||||||
|
'"event": "failed"',
|
||||||
|
'"status": tool_status(',
|
||||||
|
'"elapsedMs": model_elapsed',
|
||||||
|
]:
|
||||||
|
assert flatten(fragment) in runs_source, f'run loop instrumentation changed, update tests/run-activity.test.py: {fragment}'
|
||||||
|
|
||||||
|
# template0, not template1: hosts whose Postgres collation version drifted refuse to
|
||||||
|
# copy template1, and C collation keeps this test's ordering deterministic anyway.
|
||||||
|
sql(f"CREATE DATABASE {DB} TEMPLATE template0 LC_COLLATE 'C' LC_CTYPE 'C';",'postgres')
|
||||||
|
try:
|
||||||
|
for migration in sorted((ROOT/'migrations').glob('*.sql')):
|
||||||
|
sql(migration.read_text())
|
||||||
|
|
||||||
|
# 015 has to survive being applied twice, like every other migration.
|
||||||
|
sql((ROOT/'migrations/015_run_activity.sql').read_text())
|
||||||
|
check("""(SELECT count(*) FROM pg_indexes WHERE tablename='run_activity' AND indexname IN ('run_activity_run_idx','run_activity_retention_idx'))=2""",
|
||||||
|
'the trail needs its run index and its retention index')
|
||||||
|
|
||||||
|
sql("""
|
||||||
|
INSERT INTO users(id,name) VALUES ('u','test');
|
||||||
|
INSERT INTO spaces(id,user_id,name) VALUES ('s','u','test');
|
||||||
|
INSERT INTO computers(id,space_id,user_id,scope,scope_key,home_key) VALUES ('c','s','u','bot','c','c');
|
||||||
|
INSERT INTO bots(id,space_id,user_id,name,computer_id) VALUES
|
||||||
|
('b_live','s','u','live','c'),('b_idle','s','u','idle','c'),
|
||||||
|
('b_busy','s','u','busy','c'),('b_other','s','u','other','c');
|
||||||
|
INSERT INTO threads(id,space_id,bot_id,user_id) VALUES
|
||||||
|
('t_live','s','b_live','u'),('t_idle','s','b_idle','u'),
|
||||||
|
('t_busy','s','b_busy','u'),('t_other','s','b_other','u');
|
||||||
|
INSERT INTO runs(id,space_id,bot_id,thread_id,user_id,status,prompt,checkpoint) VALUES
|
||||||
|
('r_live','s','b_live','t_live','u','running','整理報價','{"step":"browser: click #12","turn":3,"turnLimit":40}'),
|
||||||
|
('r_goal','s','b_live','t_live','u','running','/goal 每天彙整日報','{"step":"shell: npm test","turn":7,"turnLimit":null}'),
|
||||||
|
('r_failed','s','b_idle','t_idle','u','failed','整理報價','{"step":"browser: click #12","turn":7}'),
|
||||||
|
('r_cancelled','s','b_idle','t_idle','u','cancelled','整理報價','{"keep":true}'),
|
||||||
|
('r_busy_running','s','b_busy','t_busy','u','running','還沒結束','{}'),
|
||||||
|
('r_busy_failed','s','b_busy','t_busy','u','failed','整理報價','{}'),
|
||||||
|
('r_other','s','b_other','t_other','u','running','別人的工作正在跑','{}');
|
||||||
|
UPDATE runs SET error='HTTP status 401 Unauthorized: invalid api key' WHERE id='r_failed';
|
||||||
|
UPDATE runs SET retry_count=3, lease_owner='stale-worker', lease_expires_at=now()+interval '5 minutes' WHERE id='r_failed';
|
||||||
|
""")
|
||||||
|
|
||||||
|
# The header the bubble reads: turn, limit and step must come back typed.
|
||||||
|
check("""(SELECT (checkpoint->>'turn')::bigint FROM runs WHERE id='r_live')=3
|
||||||
|
AND (SELECT (checkpoint->>'turnLimit')::bigint FROM runs WHERE id='r_live')=40""",
|
||||||
|
'turn and turn limit must be readable as numbers')
|
||||||
|
check("""(SELECT (checkpoint->>'turnLimit')::bigint IS NULL FROM runs WHERE id='r_goal')""",
|
||||||
|
'a goal run has no limit instead of a broken one')
|
||||||
|
|
||||||
|
# A trail, oldest first, with the payload the panel renders.
|
||||||
|
sql("""
|
||||||
|
INSERT INTO run_activity(run_id,kind,payload) VALUES
|
||||||
|
('r_live','run','{"event":"started","task":"整理報價"}'),
|
||||||
|
('r_live','model','{"turn":1,"elapsedMs":6400,"toolCalls":1,"text":"先打開網頁"}'),
|
||||||
|
('r_live','tool','{"turn":1,"name":"browser","step":"browser: click #12","status":"ok","elapsedMs":400}'),
|
||||||
|
('r_live','retry','{"turn":2,"attempt":1,"error":"429 rate limit","gaveUp":false}'),
|
||||||
|
('r_live','tool','{"turn":2,"name":"browser","step":"browser: type #13","status":"timed_out","elapsedMs":150000}');
|
||||||
|
INSERT INTO run_activity(run_id,kind,payload) VALUES ('r_failed','run','{"event":"failed","error":"HTTP status 401"}');
|
||||||
|
""")
|
||||||
|
assert len(row_ids('r_live')) == 5, 'the trail collected every turn'
|
||||||
|
check("""(SELECT payload->>'step' FROM run_activity WHERE kind='tool' ORDER BY id LIMIT 1)='browser: click #12'""",
|
||||||
|
'a tool line carries the step it is on')
|
||||||
|
|
||||||
|
# Paging: the first read is the newest window (oldest first so it renders in
|
||||||
|
# order), later reads only carry what is newer than the row already shown.
|
||||||
|
live = row_ids('r_live')
|
||||||
|
assert len(live) == 5, f'expected five trail rows, got {live}'
|
||||||
|
assert trail('r_live', None, 60).split() == live, 'a full window is the whole trail, oldest first'
|
||||||
|
assert trail('r_live', None, 2).split() == live[3:], f'the first read must be the newest window, got {trail("r_live", None, 2).split()}'
|
||||||
|
assert trail('r_live', live[2], 60).split() == live[3:], 'a cursor must never serve a row twice'
|
||||||
|
assert trail('r_live', live[4], 60) == '', 'a tail read on a quiet run stays empty'
|
||||||
|
assert trail('r_nothing', None, 60) == '', 'a run with no trail shows an empty panel, not an error'
|
||||||
|
sql("""
|
||||||
|
INSERT INTO run_activity(run_id,kind,payload) VALUES
|
||||||
|
('r_live','tool','{"turn":3,"name":"shell","step":"shell: npm run build","status":"ok"}'),
|
||||||
|
('r_live','notice','{"turn":3,"text":"這輪不需要電腦"}');
|
||||||
|
""")
|
||||||
|
fresh = row_ids('r_live')[5:]
|
||||||
|
assert len(fresh) == 2, f'two new rows expected, got {fresh}'
|
||||||
|
assert trail('r_live', live[4], 60).split() == fresh, 'new rows extend the panel in arrival order'
|
||||||
|
assert trail('r_live', live[4], 1) == fresh[1], 'a small window keeps the freshest line, not the oldest'
|
||||||
|
assert len(set(trail('r_live', None, 60).split())) == 7, 'the window never repeats a row'
|
||||||
|
|
||||||
|
# Manual retry: back to queued, counters reset, checkpoint kept for the resume.
|
||||||
|
assert retry('r_failed') == 't_idle', 'a failed run must be re-queueable from the chat'
|
||||||
|
check("""(SELECT status FROM runs WHERE id='r_failed')='queued'""", 'a retry goes back to the queue')
|
||||||
|
check("""(SELECT retry_count FROM runs WHERE id='r_failed')=0""", 'a human retry is not the automatic retry')
|
||||||
|
check("""(SELECT error FROM runs WHERE id='r_failed') IS NULL""", 'the stale failure must stop being shown')
|
||||||
|
check("""(SELECT lease_owner FROM runs WHERE id='r_failed') IS NULL""", 'a re-queued run must not keep the dead worker lease')
|
||||||
|
check("""(SELECT checkpoint->>'step' FROM runs WHERE id='r_failed')='browser: click #12'""",
|
||||||
|
'the checkpoint survives so the run continues where it stopped')
|
||||||
|
assert retry('r_failed') == '<null>', 'a queued run cannot be re-queued twice'
|
||||||
|
assert retry('r_cancelled') == 't_idle', 'a cancelled run can be restarted too'
|
||||||
|
|
||||||
|
# Only the bot that is actually idle may be restarted, and only for itself.
|
||||||
|
assert retry('r_busy_running') == '<null>', 'a live run is never hijacked by a retry'
|
||||||
|
assert retry('r_busy_failed') == '<null>', 'a bot with work in flight cannot be double-booked'
|
||||||
|
sql("UPDATE runs SET status='cancelled' WHERE id='r_busy_running'")
|
||||||
|
assert retry('r_busy_failed') == 't_busy', 'once its other run is gone the retry works'
|
||||||
|
assert retry('r_other') == '<null>', 'another bot is none of this retry\'s business'
|
||||||
|
|
||||||
|
# Diagnostics are expendable: old trail lines expire, fresh ones stay.
|
||||||
|
sql("""
|
||||||
|
INSERT INTO run_activity(run_id,kind,payload,created_at)
|
||||||
|
SELECT 'r_live','notice','{"text":"old"}',now()-interval '30 days' FROM generate_series(1,3) n;
|
||||||
|
UPDATE run_activity SET created_at=now()-interval '30 days' WHERE payload->>'text'='old';
|
||||||
|
""")
|
||||||
|
stale = query("SELECT count(*) FROM run_activity WHERE created_at < now()-interval '7 days';")
|
||||||
|
sql(ACTIVITY_SQL.replace('$1','7').replace('$2','1000')+';')
|
||||||
|
removed = int(stale) - int(query("SELECT count(*) FROM run_activity WHERE created_at < now()-interval '7 days';"))
|
||||||
|
assert removed == int(stale), f'retention must remove every expired row ({stale}), removed {removed}'
|
||||||
|
check("""(SELECT count(*) FROM run_activity WHERE run_id='r_live')=7""", 'fresh trail rows are never collected')
|
||||||
|
sql(ACTIVITY_SQL.replace('$1','7').replace('$2','1000')+';')
|
||||||
|
check("""(SELECT count(*) FROM run_activity WHERE run_id='r_live')=7""", 'repeat retention batches stay safe')
|
||||||
|
|
||||||
|
# And the trail dies with its run, never as an orphan holding the table open.
|
||||||
|
sql("DELETE FROM runs WHERE id='r_live';")
|
||||||
|
check("""(SELECT count(*) FROM run_activity WHERE run_id='r_live')=0""", 'deleting a run must delete its trail')
|
||||||
|
print('PASS: trail paging, typed turn/limit header, guarded manual retry that keeps the checkpoint, expiring diagnostics, cascading cleanup')
|
||||||
|
finally:
|
||||||
|
sql(f'DROP DATABASE {DB};','postgres')
|
||||||
Loading…
Reference in New Issue