daniel.w
/
lazyBoy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat/cua-driver-poc #7

Merged
daniel.w merged 9 commits from feat/cua-driver-poc into main 2026-09-08 16:57:24 +00:00
Conversation 0 Commits 9 Files Changed 159 +7136 -540
52 changed files with 7136 additions and 540 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit f6585d216d - Show all commits

4
.env.example
View File

@ -21,6 +21,10 @@ LAZYBOY_COMPUTER_CPUS=2
LAZYBOY_COMPUTER_PIDS=2048
# Only affects the Agent desktop container. Disabled by default.
LAZYBOY_COMPUTER_SUDO=false
# Computer-control backend inside each desktop container. Rebuild/recreate
# computers after changing. `cua` is Cua Driver (default); `legacy` is
# CDP/AT-SPI/xdotool rollback.
LAZYBOY_COMPUTER_DRIVER=cua
# Linux only (optional): point this at the host's LXCFS root to make htop/free
# report the per-Agent cgroup quota. Leave the default empty directory on macOS.
LAZYBOY_LXCFS_ROOT=./data/lxcfs

5
Cargo.lock generated
View File

@ -1936,6 +1936,7 @@ name = "lazyboy-control"
version = "0.1.0"
dependencies = [
"async-trait",
"base64 0.22.1",
"chrono",
"hex",
"image",
@ -1944,6 +1945,8 @@ dependencies = [
"serde_json",
"sha2",
"thiserror",
"tokio",
"tracing",
]
[[package]]
@ -1951,8 +1954,6 @@ name = "lazyboy-controld"
version = "0.1.0"
dependencies = [
"axum",
"base64 0.22.1",
"lazyboy-contracts",
"lazyboy-control",
"serde_json",
"tokio",

5
Makefile
View File

@ -12,6 +12,7 @@ DATA_DIR ?= ./data
.PHONY: help env env-force \
up logs ps health down purge \
computer postgres postgres-down pg-collation \
cua-smoke \
build build-api build-supervisor build-controld \
fmt fmt-check clippy lint audit test clean \
web \
@ -31,6 +32,7 @@ help: ## Show this help
@echo ""
@echo " Individual pieces:"
@echo " make computer Build the heavy Debian desktop image (lazyboy/computer:local)"
@echo " make cua-smoke Run Cua Driver smoke test in a disposable desktop container"
@echo " make postgres Start only postgres (127.0.0.1:5434) and wait for ready"
@echo " make postgres-down Stop postgres"
@echo " make pg-collation Repair a Postgres collation version mismatch (see docs)"
@ -93,6 +95,9 @@ purge: ## Stop containers and delete the postgres data volume
computer: ## Build the Debian desktop image used to spawn bot computers
docker build -f image/computer/Dockerfile -t $(COMPUTER_IMAGE) .
cua-smoke: computer ## Run the Cua Driver smoke test inside a disposable desktop container
./scripts/cua-smoke-test.sh --docker --image $(COMPUTER_IMAGE)
postgres: ## Start only postgres and wait until it is ready
$(COMPOSE) -f docker-compose.yml -f docker-compose.dev.yml up -d postgres
@echo "waiting for postgres (127.0.0.1:5434) to be ready..."

2208
a.md Normal file
View File

File diff suppressed because it is too large Load Diff

90
crates/api/src/file_skills.rs
View File

@ -17,7 +17,9 @@ fn valid_name(name: &str) -> bool {
&& name != "goal"
&& name != "skills"
&& name != "help"
&& name.bytes().all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
&& name
.bytes()
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
}
fn parse_file(path: PathBuf, name: String) -> Option<FileSkill> {
@ -38,23 +40,51 @@ fn parse_file(path: PathBuf, name: String) -> Option<FileSkill> {
} else {
("", text.trim().to_string())
};
let description = frontmatter.lines().find_map(|line| {
let (key, value) = line.split_once(':')?;
(key.trim() == "description")
.then(|| value.trim().trim_matches(|ch| ch == '"' || ch == '\'').to_string())
}).unwrap_or_else(|| instructions.lines().next().unwrap_or("自訂技能").chars().take(120).collect());
(!instructions.is_empty()).then_some(FileSkill { name, description, instructions })
let description = frontmatter
.lines()
.find_map(|line| {
let (key, value) = line.split_once(':')?;
(key.trim() == "description").then(|| {
value
.trim()
.trim_matches(|ch| ch == '"' || ch == '\'')
.to_string()
})
})
.unwrap_or_else(|| {
instructions
.lines()
.next()
.unwrap_or("自訂技能")
.chars()
.take(120)
.collect()
});
(!instructions.is_empty()).then_some(FileSkill {
name,
description,
instructions,
})
}
pub fn list(data_dir: &str) -> Vec<FileSkill> {
let root = Path::new(data_dir).join("skills");
let Ok(entries) = std::fs::read_dir(&root) else { return Vec::new() };
let mut skills = entries.filter_map(Result::ok).filter_map(|entry| {
let file_type = entry.file_type().ok()?;
if !file_type.is_dir() { return None; }
let name = entry.file_name().to_string_lossy().to_string();
valid_name(&name).then(|| parse_file(entry.path().join("SKILL.md"), name)).flatten()
}).collect::<Vec<_>>();
let Ok(entries) = std::fs::read_dir(&root) else {
return Vec::new();
};
let mut skills = entries
.filter_map(Result::ok)
.filter_map(|entry| {
let file_type = entry.file_type().ok()?;
if !file_type.is_dir() {
return None;
}
let name = entry.file_name().to_string_lossy().to_string();
valid_name(&name)
.then(|| parse_file(entry.path().join("SKILL.md"), name))
.flatten()
})
.collect::<Vec<_>>();
skills.sort_by(|left, right| left.name.cmp(&right.name));
skills
}
@ -62,15 +92,25 @@ pub fn list(data_dir: &str) -> Vec<FileSkill> {
pub fn slash(prompt: &str, data_dir: &str) -> Option<(FileSkill, String)> {
let mut words = prompt.trim().splitn(2, char::is_whitespace);
let command = words.next()?.strip_prefix('/')?;
if !valid_name(command) { return None; }
let skill = list(data_dir).into_iter().find(|skill| skill.name == command)?;
if !valid_name(command) {
return None;
}
let skill = list(data_dir)
.into_iter()
.find(|skill| skill.name == command)?;
Some((skill, words.next().unwrap_or("").trim().to_string()))
}
pub fn index(data_dir: &str) -> String {
let skills = list(data_dir);
if skills.is_empty() { return String::new(); }
let lines = skills.iter().map(|skill| format!("/{:<18} {}", skill.name, skill.description)).collect::<Vec<_>>().join("\n");
if skills.is_empty() {
return String::new();
}
let lines = skills
.iter()
.map(|skill| format!("/{:<18} {}", skill.name, skill.description))
.collect::<Vec<_>>()
.join("\n");
format!("可用的檔案技能（唯讀）：\n{lines}")
}
@ -81,12 +121,20 @@ mod tests {
#[test]
fn loads_only_safe_skill_names_and_resolves_arguments() {
let root = std::env::temp_dir().join(format!("lazyboy-file-skills-{}", uuid::Uuid::new_v4()));
let root =
std::env::temp_dir().join(format!("lazyboy-file-skills-{}", uuid::Uuid::new_v4()));
fs::create_dir_all(root.join("skills/open-site")).unwrap();
fs::write(root.join("skills/open-site/SKILL.md"), "---\ndescription: Open a site\n---\nUse the browser.\n").unwrap();
fs::write(
root.join("skills/open-site/SKILL.md"),
"---\ndescription: Open a site\n---\nUse the browser.\n",
)
.unwrap();
fs::create_dir_all(root.join("skills/goal")).unwrap();
let root_text = root.to_str().unwrap();
assert_eq!(slash("/open-site example.com", root_text).unwrap().1, "example.com");
assert_eq!(
slash("/open-site example.com", root_text).unwrap().1,
"example.com"
);
assert!(slash("/goal do it", root_text).is_none());
assert!(slash("/missing", root_text).is_none());
let _ = fs::remove_dir_all(root);

4
crates/api/src/main.rs
View File

@ -57,7 +57,9 @@ async fn main() {
});
let retention_state = state.clone();
tokio::spawn(async move { retention::retention_loop(retention_state).await; });
tokio::spawn(async move {
retention::retention_loop(retention_state).await;
});
let worker_state = state.clone();
tokio::spawn(async move {

151
crates/api/src/retention.rs
View File

@ -1,25 +1,60 @@
//! Bounded maintenance of expendable diagnostics, never user-authored content.
use std::time::{Duration};
use sqlx::PgPool;
use crate::state::AppState;
use sqlx::PgPool;
use std::time::Duration;
fn days(name: &str, default: i32) -> i32 {
std::env::var(name).ok().and_then(|v| v.parse::<i32>().ok())
.filter(|v| (1..=3650).contains(v)).unwrap_or(default)
std::env::var(name)
.ok()
.and_then(|v| v.parse::<i32>().ok())
.filter(|v| (1..=3650).contains(v))
.unwrap_or(default)
}
pub async fn retention_loop(state: AppState) {
let recording_days = days("LAZYBOY_RECORDING_RETENTION_DAYS", 30);
let rules = [
("events", include_str!("retention/events.sql"), days("LAZYBOY_EVENT_RETENTION_DAYS", 30)),
("checkpoints", include_str!("retention/checkpoints.sql"), days("LAZYBOY_CHECKPOINT_RETENTION_DAYS", 7)),
("runs", include_str!("retention/runs.sql"), days("LAZYBOY_RUN_RETENTION_DAYS", 90)),
("run_activity", include_str!("retention/run_activity.sql"), days("LAZYBOY_RUN_ACTIVITY_RETENTION_DAYS", 7)),
("recordings", include_str!("retention/recordings.sql"), recording_days),
("revisions", include_str!("retention/revisions.sql"), days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90)),
("deleted_memories", include_str!("retention/deleted_memories.sql"), days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90)),
(
"events",
include_str!("retention/events.sql"),
days("LAZYBOY_EVENT_RETENTION_DAYS", 30),
),
(
"checkpoints",
include_str!("retention/checkpoints.sql"),
days("LAZYBOY_CHECKPOINT_RETENTION_DAYS", 7),
),
(
"runs",
include_str!("retention/runs.sql"),
days("LAZYBOY_RUN_RETENTION_DAYS", 90),
),
(
"run_activity",
include_str!("retention/run_activity.sql"),
days("LAZYBOY_RUN_ACTIVITY_RETENTION_DAYS", 7),
),
(
"recordings",
include_str!("retention/recordings.sql"),
recording_days,
),
(
"revisions",
include_str!("retention/revisions.sql"),
days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90),
),
(
"deleted_memories",
include_str!("retention/deleted_memories.sql"),
days("LAZYBOY_MEMORY_HISTORY_RETENTION_DAYS", 90),
),
("leases", include_str!("retention/leases.sql"), 7),
("profile_locks", include_str!("retention/profile_locks.sql"), 7),
(
"profile_locks",
include_str!("retention/profile_locks.sql"),
7,
),
];
loop {
for (name, query, age) in rules {
@ -27,20 +62,48 @@ pub async fn retention_loop(state: AppState) {
// Limit both transaction size and work per hour; defer excess backlog.
for _ in 0..20 {
match batch(state.pool(), query, age).await {
Ok(count) => { removed += count; if count < 1000 { break; } }
Err(error) => { tracing::warn!(name, %error, "retention batch failed; retry next hour"); break; }
Ok(count) => {
removed += count;
if count < 1000 {
break;
}
}
Err(error) => {
tracing::warn!(name, %error, "retention batch failed; retry next hour");
break;
}
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
if removed > 0 { tracing::info!(name, rows=removed, "retention cleaned expired diagnostics"); }
if removed > 0 {
tracing::info!(
name,
rows = removed,
"retention cleaned expired diagnostics"
);
}
}
if let Err(error) = clean_frames(&state, recording_days).await {
tracing::warn!(%error, "recording file retention failed; retry next hour");
}
if let Ok(bytes) = sqlx::query_scalar::<_, i64>("SELECT pg_database_size(current_database())").fetch_one(state.pool()).await {
let warn_mb = std::env::var("LAZYBOY_DB_WARN_MB").ok().and_then(|v| v.parse::<i64>().ok()).filter(|v| *v > 0 && *v < 1_000_000).unwrap_or(1024);
if let Ok(bytes) =
sqlx::query_scalar::<_, i64>("SELECT pg_database_size(current_database())")
.fetch_one(state.pool())
.await
{
let warn_mb = std::env::var("LAZYBOY_DB_WARN_MB")
.ok()
.and_then(|v| v.parse::<i64>().ok())
.filter(|v| *v > 0 && *v < 1_000_000)
.unwrap_or(1024);
tracing::info!(bytes, "database size after retention");
if bytes > warn_mb * 1024 * 1024 { tracing::warn!(bytes, warn_mb, "database exceeds configured size warning; review retained conversations and memories"); }
if bytes > warn_mb * 1024 * 1024 {
tracing::warn!(
bytes,
warn_mb,
"database exceeds configured size warning; review retained conversations and memories"
);
}
}
tokio::time::sleep(Duration::from_secs(3600)).await;
}
@ -49,16 +112,31 @@ pub async fn retention_loop(state: AppState) {
async fn batch(pool: &PgPool, query: &str, age: i32) -> Result<u64, sqlx::Error> {
let mut tx = pool.begin().await?;
// One maintenance writer, even when multiple API processes start together.
let acquired: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(72189431)").fetch_one(&mut *tx).await?;
if !acquired { return Ok(0); }
sqlx::query("SET LOCAL statement_timeout = '10s'").execute(&mut *tx).await?;
sqlx::query("SET LOCAL lock_timeout = '1s'").execute(&mut *tx).await?;
let result = sqlx::query(query).bind(age).bind(1000_i64).execute(&mut *tx).await?;
let acquired: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(72189431)")
.fetch_one(&mut *tx)
.await?;
if !acquired {
return Ok(0);
}
sqlx::query("SET LOCAL statement_timeout = '10s'")
.execute(&mut *tx)
.await?;
sqlx::query("SET LOCAL lock_timeout = '1s'")
.execute(&mut *tx)
.await?;
let result = sqlx::query(query)
.bind(age)
.bind(1000_i64)
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(result.rows_affected())
}
async fn clean_frames(state: &AppState, age: i32) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
async fn clean_frames(
state: &AppState,
age: i32,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let root = std::path::Path::new(&state.data_dir).join("teach");
let mut dirs = match tokio::fs::read_dir(&root).await {
Ok(dirs) => dirs,
@ -67,20 +145,35 @@ async fn clean_frames(state: &AppState, age: i32) -> Result<(), Box<dyn std::err
};
let mut cleaned = 0;
while let Some(entry) = dirs.next_entry().await? {
if cleaned >= 1000 { break; }
if cleaned >= 1000 {
break;
}
// Ignore symlinks and unexpected names; never traverse a user's home.
if !entry.file_type().await?.is_dir() { continue; }
if !entry.file_type().await?.is_dir() {
continue;
}
let id = entry.file_name().to_string_lossy().into_owned();
if uuid::Uuid::parse_str(&id).is_err() { continue; }
if uuid::Uuid::parse_str(&id).is_err() {
continue;
}
let eligible: Option<bool> = sqlx::query_scalar(
"SELECT status IN ('saved','failed','draft') AND updated_at < now() - make_interval(days => $2) FROM taught_skills WHERE id=$1"
).bind(&id).bind(age).fetch_optional(state.pool()).await?;
let old_orphan = eligible.is_none() && entry.metadata().await?.modified()?.elapsed().unwrap_or(Duration::ZERO) > Duration::from_secs(age as u64 * 86400);
let old_orphan = eligible.is_none()
&& entry
.metadata()
.await?
.modified()?
.elapsed()
.unwrap_or(Duration::ZERO)
> Duration::from_secs(age as u64 * 86400);
if eligible == Some(true) || old_orphan {
tokio::fs::remove_dir_all(entry.path()).await?;
cleaned += 1;
}
}
if cleaned > 0 { tracing::info!(cleaned, "removed expired teaching frame directories"); }
if cleaned > 0 {
tracing::info!(cleaned, "removed expired teaching frame directories");
}
Ok(())
}

4
crates/api/src/routes.rs
View File

@ -609,7 +609,9 @@ async fn stop(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Json<Value>, (StatusCode, Json<Value>)> {
let actor = actor(&state).await.map_err(|status| (status, Json(json!({"message":"無法取得工作區"}))))?;
let actor = actor(&state)
.await
.map_err(|status| (status, Json(json!({"message":"無法取得工作區"}))))?;
computer::stop(&state, &actor, &id)
.await
.map(|status| Json(serde_json::to_value(status).unwrap()))

122
crates/api/src/runs.rs
View File

@ -1,7 +1,7 @@
use lazyboy_harness::execution::{
ExecutionMode, GoalOutcome, MAX_NUDGES_GOAL, MAX_NUDGES_PLAIN, NEEDS_INPUT_MARKER, StopReason,
VERIFY_BEFORE_DONE, asks_for_input, goal_outcome, goal_request, stop_reason, GOAL_CONTINUE,
GOAL_INSTRUCTIONS,
ExecutionMode, GOAL_CONTINUE, GOAL_INSTRUCTIONS, GoalOutcome, MAX_NUDGES_GOAL,
MAX_NUDGES_PLAIN, NEEDS_INPUT_MARKER, StopReason, VERIFY_BEFORE_DONE, asks_for_input,
goal_outcome, goal_request, stop_reason,
};
use lazyboy_harness::policy::{ActionObserved, LoopGuard, RunPolicy, Verdict};
use std::sync::Arc;
@ -26,6 +26,8 @@ use crate::tools::{ToolCtx, ToolOutcome, dispatch, tool_definitions};
const SCREENSHOT_CAPTION: &str = "Desktop screenshot (1280x800) with yellow numbered marks. Click by those element ids. The live VNC view has no marks.";
const TAKEOVER_RESUME_PROMPT: &str = "The user finished collaborating and released control. Continue the original task from the CURRENT screen. Do not restart from scratch. Element ids and page refs from before the handoff are invalid; use only the fresh observation below.";
const SYSTEM_CHAT: &str = "You are this bot's assistant. This message is conversation — a greeting, small talk, a question you can answer from knowledge, planning, or explaining.
Reply in text only. Do not try to use the desktop, browser, files, or shell, and do not narrate that you are checking a screen. You have a Linux desktop for later if the user asks you to operate it; this turn does not need it.
@ -179,7 +181,10 @@ pub async fn send(
checkpoint=checkpoint-'awaitResume', updated_at=now()
WHERE id=$1 AND status IN ('waiting_input','waiting_takeover')",
)
.bind(&run_id).execute(&mut *tx).await.map_err(|error| error.to_string())?;
.bind(&run_id)
.execute(&mut *tx)
.await
.map_err(|error| error.to_string())?;
}
let message_id = Uuid::new_v4().to_string();
let seq: i32 = sqlx::query_scalar(
@ -472,7 +477,7 @@ async fn execute_run(
"run",
json!({"event": "started", "task": crate::monitor::snippet(prompt, 160)}),
)
.await;
.await;
let bot = state
.db
@ -625,16 +630,20 @@ async fn execute_run(
None
};
let initial_prompt = if goal_mode {
format!("Execute this goal until it is verified complete:\n{}", goal_text)
format!(
"Execute this goal until it is verified complete:\n{}",
goal_text
)
} else if let Some((skill, args)) = &file_skill {
format!("Run the /{} skill with these arguments: {}", skill.name, args)
format!(
"Run the /{} skill with these arguments: {}",
skill.name, args
)
} else {
prompt.to_string()
};
let mut first = if resume_after_takeover {
vec![UserContent::text(
"The user finished collaborating and released control. Continue the original task from the CURRENT screen. Do not restart from scratch.",
)]
vec![UserContent::text(TAKEOVER_RESUME_PROMPT)]
} else {
vec![UserContent::text(initial_prompt)]
};
@ -660,7 +669,11 @@ async fn execute_run(
if !resume_after_takeover {
// The user named a taught skill: hand the model the full playbook up
// front so it does not have to guess or call use_skill first.
if let Some(skill) = if goal_mode || file_skill.is_some() { None } else { crate::skills::skill_for_prompt(state.pool(), bot_id, prompt).await } {
if let Some(skill) = if goal_mode || file_skill.is_some() {
None
} else {
crate::skills::skill_for_prompt(state.pool(), bot_id, prompt).await
} {
first.push(UserContent::text(crate::skills::format_playbook_for_run(
&skill,
)));
@ -689,8 +702,12 @@ async fn execute_run(
.any(|block| block.get("kind").and_then(Value::as_str) == Some("file"));
// Greetings and small talk must not even *see* desktop tools: models
// otherwise "check the screen" or `ls` the home on "hi" and boot Docker.
let chat_only =
!resume_after_takeover && !goal_mode && file_skill.is_none() && skill_check.is_none() && !workspace_file && is_plain_chat(prompt);
let chat_only = !resume_after_takeover
&& !goal_mode
&& file_skill.is_none()
&& skill_check.is_none()
&& !workspace_file
&& is_plain_chat(prompt);
if chat_only {
// Memory is recalled separately and injected into the preamble below.
// Do not expose even memory tools here: a plain greeting must be one
@ -761,7 +778,11 @@ async fn execute_run(
// steering input. Keep the run alive and deliver each new message once
// before the next model turn instead of waiting for a second run to win
// the bot lease.
let mut steering_seq = checkpoint.get("steeringSeq").and_then(Value::as_i64).map(|seq| seq as i32).unwrap_or(current_seq);
let mut steering_seq = checkpoint
.get("steeringSeq")
.and_then(Value::as_i64)
.map(|seq| seq as i32)
.unwrap_or(current_seq);
let mut used_gui = false;
let mut did_work = false;
// One verification demand per run: enough to catch "I'm done" that isn't,
@ -1057,7 +1078,11 @@ async fn execute_run(
goal_outcome(&final_text),
GoalOutcome::Complete | GoalOutcome::NeedsInput
);
let nudge_limit = if goal_mode { MAX_NUDGES_GOAL } else { MAX_NUDGES_PLAIN };
let nudge_limit = if goal_mode {
MAX_NUDGES_GOAL
} else {
MAX_NUDGES_PLAIN
};
let mut verify_chosen = false;
let nudge = if goal_mode {
match goal_outcome(&final_text) {
@ -1114,9 +1139,7 @@ async fn execute_run(
if used_gui || skill_check.is_some() {
prepare_run_computer(state, actor, bot_id, run_id, &ctx, true).await?;
}
if (used_gui || skill_check.is_some())
&& ctx.gui_block.lock().unwrap().is_none()
{
if (used_gui || skill_check.is_some()) && ctx.gui_block.lock().unwrap().is_none() {
set_run_step(state, run_id, "computer_observe: 重新確認畫面").await;
let outcome = dispatch(&ctx, "computer_observe", &json!({})).await;
content.push(UserContent::text(outcome.text));
@ -1511,7 +1534,11 @@ async fn execute_run(
state,
thread_id,
run_id,
if needs_input { "run.paused" } else { "run.completed" },
if needs_input {
"run.paused"
} else {
"run.completed"
},
turns,
screenshots,
screenshot_bytes,
@ -1618,10 +1645,16 @@ async fn complete_with_retry(
for attempt in 0..3 {
if attempt > 0 {
let failure = crate::monitor::classify_run_error(&last);
set_run_step(trace.state, trace.run_id, &format!(
"{} 正在自動重試模型（第 {}/3 次）；保留已完成的操作。",
failure.headline, attempt + 1
)).await;
set_run_step(
trace.state,
trace.run_id,
&format!(
"{} 正在自動重試模型（第 {}/3 次）；保留已完成的操作。",
failure.headline,
attempt + 1
),
)
.await;
}
let started = std::time::Instant::now();
let result = tokio::time::timeout(
@ -1779,7 +1812,10 @@ fn shrink_checkpoint(history: &mut Vec<Message>, pending: &mut Message) {
cap_parts(pending);
history.iter_mut().for_each(cap_parts);
let fits = |turns: &[Message]| {
json!({"harnessHistory": turns, "harnessPending": pending}).to_string().len() <= LIMIT
json!({"harnessHistory": turns, "harnessPending": pending})
.to_string()
.len()
<= LIMIT
};
while !fits(history) && history.len() > 1 {
history.remove(0);
@ -1810,7 +1846,10 @@ async fn save_harness_checkpoint(
// Only a checkpoint that still cannot fit after shrinking fails closed, and
// loudly: the run keeps its uncertain-effects flag instead of freezing.
if value.to_string().len() > 1024 * 1024 {
tracing::error!(run_id, "harness checkpoint still exceeds 1 MB after shrinking");
tracing::error!(
run_id,
"harness checkpoint still exceeds 1 MB after shrinking"
);
return Ok(());
}
let result=sqlx::query("UPDATE runs SET checkpoint=COALESCE(checkpoint,'{}'::jsonb)||$3,updated_at=now() WHERE id=$1 AND lease_owner=$2 AND status='running'")
@ -2090,7 +2129,9 @@ async fn pause_for_answer(
}
} else if !asks_for_input(req.draft) {
// A model's optimistic draft must not hide a harness-detected stall.
format!("{draft}\n\n任務尚未確認完成。停止原因：{stall}。已保存操作進度；回覆下一步指示或接管確認現況後可繼續。")
format!(
"{draft}\n\n任務尚未確認完成。停止原因：{stall}。已保存操作進度；回覆下一步指示或接管確認現況後可繼續。"
)
} else {
draft
};
@ -2959,15 +3000,17 @@ fn describe_step(name: &str, args: &Value) -> String {
}
"shell" => {
// The terminal does four different things; the feed says which.
let session = get("session")
.filter(|name| !name.trim().is_empty() && *name != "main");
let session = get("session").filter(|name| !name.trim().is_empty() && *name != "main");
let suffix = session.map(|name| format!(" ·{name}")).unwrap_or_default();
if args.get("reset").and_then(Value::as_bool).unwrap_or(false) {
format!("重開終端機{suffix}")
} else if let Some(keys) = get("keys") {
format!("輸入 {}{suffix}", short(Some(keys), 20))
} else {
match get("command").or(get("cmd")).filter(|line| !line.trim().is_empty()) {
match get("command")
.or(get("cmd"))
.filter(|line| !line.trim().is_empty())
{
Some(command) => format!("{}{suffix}", short(Some(command), 60)),
None => format!("讀終端機{suffix}"),
}
@ -3114,8 +3157,8 @@ fn tool_status(timed_out: bool, pause: bool, text: &str) -> &'static str {
fn action_changes_state(name: &str, args: &Value) -> bool {
match name {
"computer_act" | "shell" | "write_file" | "launch_app" | "open_path"
| "create_schedule" | "cancel_schedule" | "remember" | "forget_memory" | "use_saved_login"
| "request_takeover" => true,
| "create_schedule" | "cancel_schedule" | "remember" | "forget_memory"
| "use_saved_login" | "request_takeover" => true,
"browser" => matches!(
args.get("action").and_then(Value::as_str),
Some("click") | Some("type") | Some("navigate") | Some("press")
@ -3179,13 +3222,24 @@ async fn record_run_metrics(
#[cfg(test)]
mod tests {
use super::{
RunHalt, SCREENSHOT_CAPTION, describe_step, drop_history_screenshots, halt_from_status,
history_window_start, is_plain_chat, retryable_run_error, screenshot_parts, tool_needs_gui,
tool_needs_sandbox, tool_status,
RunHalt, SCREENSHOT_CAPTION, TAKEOVER_RESUME_PROMPT, describe_step,
drop_history_screenshots, halt_from_status, history_window_start, is_plain_chat,
retryable_run_error, screenshot_parts, tool_needs_gui, tool_needs_sandbox, tool_status,
};
use rig_core::completion::message::{Message, UserContent};
use serde_json::json;
#[test]
fn takeover_resume_invalidates_pre_handoff_refs() {
assert!(
TAKEOVER_RESUME_PROMPT
.contains("Element ids and page refs from before the handoff are invalid")
);
assert!(TAKEOVER_RESUME_PROMPT.contains("fresh observation"));
assert!(tool_needs_gui("browser"));
assert!(tool_needs_gui("computer_observe"));
}
#[test]
fn the_trail_reads_a_tool_result_as_ok_error_timeout_or_pause() {
assert_eq!(tool_status(false, false, "clicked element #12"), "ok");

14
crates/api/src/sessions.rs
View File

@ -390,7 +390,14 @@ async fn events(
// afterwards would open a window in which a commit could knock on a channel
// this reader is not listening to yet.
let wakes = state.wakes.subscribe();
let stream_state = (state, id, actor, after, Vec::<(i32, String, Value)>::new(), wakes);
let stream_state = (
state,
id,
actor,
after,
Vec::<(i32, String, Value)>::new(),
wakes,
);
let output = stream::unfold(
stream_state,
|(state, id, actor, mut after, mut pending, mut wakes)| async move {
@ -402,10 +409,7 @@ async fn events(
.event(kind)
.json_data(payload)
.unwrap_or_else(|_| Event::default().event("error").data("{}"));
return Some((
Ok(event),
(state, id, actor, after, pending, wakes),
));
return Some((Ok(event), (state, id, actor, after, pending, wakes)));
}
match sqlx::query_as::<_, (i32, String, Value)>(
"SELECT e.seq,e.type,e.payload FROM events e

72
crates/api/src/skills.rs
View File

@ -19,8 +19,7 @@ use axum::{Json, Router};
use base64::Engine;
use chrono::{DateTime, TimeDelta, Utc};
use lazyboy_control::{
AdapterContext, CommandRequest, ComputerRef, cdp_record_command_on, cdp_record_stop_command,
frame_signature, signatures_similar, teach_recorder_output,
AdapterContext, ComputerRef, RecordingRequest, frame_signature, signatures_similar,
};
use rig_core::completion::message::{
AssistantContent, ImageDetail, ImageMediaType, Message, UserContent,
@ -282,17 +281,14 @@ async fn start_skill(
})?;
let ctx = computer::adapter_context_for(&actor, &bot_id, "teach", screen.as_ref(), None);
let display = ctx.display.clone().unwrap_or_else(|| ":1".into());
let argv = cdp_record_command_on(&display, ctx.profile_path.as_deref(), &skill_id);
if let Err(error) = state
.sandbox
.execute(
.start_recording(
&computer_ref,
CommandRequest {
argv,
cwd: None,
timeout_ms: Some(10_000),
stdin: None,
RecordingRequest {
skill_id: skill_id.clone(),
display: ctx.display.clone(),
profile_path: ctx.profile_path.clone(),
},
&ctx,
)
@ -889,6 +885,14 @@ async fn record_loop(
}
}
fn recording_request(skill_id: &str, ctx: &AdapterContext) -> RecordingRequest {
RecordingRequest {
skill_id: skill_id.to_string(),
display: ctx.display.clone(),
profile_path: ctx.profile_path.clone(),
}
}
async fn stop_recorder(
state: &AppState,
computer_ref: &ComputerRef,
@ -897,16 +901,7 @@ async fn stop_recorder(
) {
let _ = state
.sandbox
.execute(
computer_ref,
CommandRequest {
argv: cdp_record_stop_command(skill_id),
cwd: None,
timeout_ms: Some(5_000),
stdin: None,
},
ctx,
)
.stop_recording(computer_ref, recording_request(skill_id, ctx), ctx)
.await;
}
@ -916,34 +911,17 @@ async fn collect_browser_events(
ctx: &AdapterContext,
skill_id: &str,
) -> Vec<Value> {
let out = teach_recorder_output(skill_id);
let result = state
match state
.sandbox
.execute(
computer_ref,
CommandRequest {
argv: vec![
"sh".into(),
"-c".into(),
"cat \"$0\" 2>/dev/null; rm -f \"$0\"".into(),
out,
],
cwd: None,
timeout_ms: Some(10_000),
stdin: None,
},
ctx,
)
.await;
let Ok(result) = result else {
return Vec::new();
};
result
.stdout
.lines()
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
.filter(|event| event.get("t").and_then(Value::as_str) != Some("recorder"))
.collect()
.collect_recording(computer_ref, recording_request(skill_id, ctx), ctx)
.await
{
Ok(result) => result.events,
Err(error) => {
tracing::warn!("teach {skill_id}: collect recording failed: {error}");
Vec::new()
}
}
}
/// Stop recording, gather the trace, hand the desktop back and distil the

5
crates/api/src/state.rs
View File

@ -47,7 +47,10 @@ pub struct CallLease {
impl Drop for CallLease {
fn drop(&mut self) {
let mut map = self.registry.lock();
if map.get(&self.bot_id).is_some_and(|held| *held == self.call_id) {
if map
.get(&self.bot_id)
.is_some_and(|held| *held == self.call_id)
{
map.remove(&self.bot_id);
}
}

239
crates/api/src/tools.rs
View File

@ -4,10 +4,10 @@ use lazyboy_contracts::{
ComputerAction, ComputerMode, ComputerObservation, PointerType, UiElement,
};
use lazyboy_control::{
ActionError, ActionRequest, AdapterContext, CdpPage, CommandRequest, ComputerRef,
SandboxProvider, a11y_command_on, apply_element_targets, browser_gui_block, cdp_command_on,
click_fingerprint, element_id, format_ui_elements, frames_match, merge_ui_elements,
cdp_stdin_command_on, overlay_elements, parse_a11y_page, parse_cdp_page, parse_computer_actions,
ActionError, ActionRequest, AdapterContext, BrowserRequest, CdpPage, CommandRequest,
ComputerRef, SandboxProvider, a11y_command_on, apply_element_targets, browser_gui_block,
cdp_stdin_command_on, click_fingerprint, element_id, format_ui_elements, frames_match,
merge_ui_elements, overlay_elements, parse_a11y_page, parse_cdp_page, parse_computer_actions,
resolve_bot_workspace_cwd, resolve_bot_workspace_path, should_block_stale_click,
};
use rig_core::completion::ToolDefinition;
@ -454,9 +454,11 @@ fn vision_guard(ctx: &ToolCtx) -> Option<ToolOutcome> {
}
fn observation_text(note: &str, observation: &ComputerObservation, unchanged: bool) -> String {
let label = if observation.elements.iter().any(|element| {
matches!(element.kind.as_deref(), Some("dom") | Some("a11y"))
}) {
let label = if observation
.elements
.iter()
.any(|element| matches!(element.kind.as_deref(), Some("dom") | Some("a11y")))
{
"Clickable controls"
} else {
"Clickable windows"
@ -481,7 +483,11 @@ async fn observe(ctx: &ToolCtx) -> ToolOutcome {
if let Some(blocked) = vision_guard(ctx) {
return blocked;
}
match ctx.sandbox.observe(&ctx.computer_ref(), &ctx.adapter()).await {
match ctx
.sandbox
.observe(&ctx.computer_ref(), &ctx.adapter())
.await
{
Ok(observation) => {
let (observation, note) =
attach_ui_elements(ctx, observation, "computer observed").await;
@ -509,7 +515,11 @@ async fn wait_then_observe(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
if vision_guard(ctx).is_some() {
return text_outcome(format!("waited {seconds:.0}s"));
}
match ctx.sandbox.observe(&ctx.computer_ref(), &ctx.adapter()).await {
match ctx
.sandbox
.observe(&ctx.computer_ref(), &ctx.adapter())
.await
{
Ok(observation) => {
let note = format!("waited {seconds:.0}s");
let (observation, note) = attach_ui_elements(ctx, observation, &note).await;
@ -559,11 +569,7 @@ async fn a11y_snapshot(ctx: &ToolCtx, include_browser: bool) -> Option<lazyboy_c
json!({"action": "snapshot", "includeBrowser": include_browser}),
)
.await;
if page.ok {
Some(page)
} else {
None
}
if page.ok { Some(page) } else { None }
}
async fn a11y_call(ctx: &ToolCtx, request: serde_json::Value) -> lazyboy_control::A11yPage {
@ -607,41 +613,26 @@ async fn cdp_snapshot(ctx: &ToolCtx, ensure: bool) -> Option<CdpPage> {
async fn cdp_call(ctx: &ToolCtx, request: Value) -> CdpPage {
let adapter = ctx.adapter();
let display = adapter.display.as_deref().unwrap_or(":1");
// Clicks may sit through a page's stay timer (CLICK_WAIT_MS in cdp.py).
let timeout_ms = if request.get("action").and_then(Value::as_str) == Some("click") {
let wait = request
.get("waitMs")
.and_then(Value::as_u64)
.unwrap_or(45_000)
.min(120_000);
wait + 25_000
} else {
20_000
};
let argv = cdp_command_on(display, adapter.profile_path.as_deref(), &request);
let mut browser_request: BrowserRequest =
serde_json::from_value(request.clone()).unwrap_or_default();
if browser_request.action.is_empty() {
browser_request.action = request
.get("action")
.and_then(Value::as_str)
.unwrap_or("snapshot")
.to_string();
}
if let Some(ensure) = request.get("ensure").and_then(Value::as_bool) {
browser_request.ensure = ensure;
}
browser_request.display = adapter.display.clone();
browser_request.profile_path = adapter.profile_path.clone();
match ctx
.sandbox
.execute(
&ctx.computer_ref(),
CommandRequest {
argv,
cwd: None,
timeout_ms: Some(timeout_ms),
stdin: None,
},
&ctx.adapter(),
)
.browser(&ctx.computer_ref(), browser_request, &adapter)
.await
{
Ok(result) => {
let raw = if result.stdout.trim().is_empty() {
result.stderr
} else {
result.stdout
};
parse_cdp_page(&raw)
}
Ok(page) => page,
Err(error) => CdpPage {
ok: false,
error: Some(error.to_string()),
@ -652,33 +643,55 @@ async fn cdp_call(ctx: &ToolCtx, request: Value) -> CdpPage {
fn is_connection_check(page: &CdpPage) -> bool {
let text = format!("{} {}", page.title, page.text).to_lowercase();
page.ok && (text.contains("需要確認您的連線是安全") || (text.contains("cloudflare") && [
"verify you are human", "verifying you are human", "checking your browser",
"checking if the site connection is secure", "needs to review the security",
"驗證您是人類", "验证您是人类", "確認您的連線是安全", "確認您的人類身分",
].iter().any(|marker| text.contains(marker))))
page.ok
&& (text.contains("需要確認您的連線是安全")
|| (text.contains("cloudflare")
&& [
"verify you are human",
"verifying you are human",
"checking your browser",
"checking if the site connection is secure",
"needs to review the security",
"驗證您是人類",
"验证您是人类",
"確認您的連線是安全",
"確認您的人類身分",
]
.iter()
.any(|marker| text.contains(marker))))
}
fn connection_takeover(ctx: &ToolCtx, reason: &str) -> ToolOutcome {
*ctx.takeover_requested.lock().unwrap() = true;
ToolOutcome {
text: reason.into(), image: None, pause: true,
text: reason.into(),
image: None,
pause: true,
blocks: login_blocks(&json!({"reason":reason,"site":"網站連線驗證",
"why":"完成驗證後，繼續原本的瀏覽任務。"})),
}
}
async fn connection_check(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
if let Some(blocked) = vision_guard(ctx) { return blocked; }
let (Some(x), Some(y)) = (args.get("x").and_then(Value::as_u64), args.get("y").and_then(Value::as_u64)) else {
return text_outcome("Take a fresh computer_observe and provide the visible checkbox's non-negative screen x/y.");
if let Some(blocked) = vision_guard(ctx) {
return blocked;
}
let (Some(x), Some(y)) = (
args.get("x").and_then(Value::as_u64),
args.get("y").and_then(Value::as_u64),
) else {
return text_outcome(
"Take a fresh computer_observe and provide the visible checkbox's non-negative screen x/y.",
);
};
if x > i32::MAX as u64 || y > i32::MAX as u64 {
return text_outcome("Checkbox coordinates are out of range.");
}
let page = cdp_call(ctx, json!({"action":"snapshot","ensure":false})).await;
if !is_connection_check(&page) {
return text_outcome("No supported Cloudflare connection-check page was confirmed. Re-observe; use request_takeover for other CAPTCHA, login or 2FA. No click was sent.");
return text_outcome(
"No supported Cloudflare connection-check page was confirmed. Re-observe; use request_takeover for other CAPTCHA, login or 2FA. No click was sent.",
);
}
let already_attempted = {
let mut attempted = ctx.connection_check_attempted.lock().unwrap();
@ -694,10 +707,20 @@ async fn connection_check(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
Err(error) => return text_outcome(error.to_string()),
};
let adapter = ctx.adapter();
let result = ctx.sandbox.act(&ctx.computer_ref(), ActionRequest {
actions, observe:false, settle_ms:350,
display:adapter.display.clone(), profile_path:adapter.profile_path.clone(),
}, &adapter).await;
let result = ctx
.sandbox
.act(
&ctx.computer_ref(),
ActionRequest {
actions,
observe: false,
settle_ms: 350,
display: adapter.display.clone(),
profile_path: adapter.profile_path.clone(),
},
&adapter,
)
.await;
if result.is_err() {
return connection_takeover(ctx, "無法確認驗證點擊是否完成，請接管檢查。");
}
@ -708,12 +731,18 @@ async fn connection_check(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
// can also remove the checkbox. Return evidence for the task check.
if after.ok && !is_connection_check(&after) && !after.text.trim().is_empty() {
let mut outcome = observe(ctx).await;
outcome.text = format!("Connection-check markers disappeared. This is NOT proof of success. Confirm the requested content is actually visible before continuing; if a challenge/error remains, request_takeover.\n{}\n{}",
browser_result_text("snapshot", &after), outcome.text);
outcome.text = format!(
"Connection-check markers disappeared. This is NOT proof of success. Confirm the requested content is actually visible before continuing; if a challenge/error remains, request_takeover.\n{}\n{}",
browser_result_text("snapshot", &after),
outcome.text
);
return outcome;
}
}
connection_takeover(ctx, "已嘗試一次驗證並等待，仍無法確認通過。請接管完成驗證，之後繼續原任務。")
connection_takeover(
ctx,
"已嘗試一次驗證並等待，仍無法確認通過。請接管完成驗證，之後繼續原任務。",
)
}
async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
@ -802,7 +831,11 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
blocks: Vec::new(),
};
}
match ctx.sandbox.observe(&ctx.computer_ref(), &ctx.adapter()).await {
match ctx
.sandbox
.observe(&ctx.computer_ref(), &ctx.adapter())
.await
{
Ok(observation) => {
let (observation, note) = attach_ui_elements(ctx, observation, &text).await;
pack_observation(ctx, &note, observation)
@ -819,7 +852,10 @@ async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
fn browser_result_text(action: &str, page: &CdpPage) -> String {
format!(
"browser {action}\nPage: {} {}\nClickable page elements: {}\nVisible text:\n{}",
page.title, page.url, format_ui_elements(&page.elements), page.text
page.title,
page.url,
format_ui_elements(&page.elements),
page.text
)
}
@ -1201,13 +1237,11 @@ async fn shell(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
Ok(result) => result,
Err(error) => return text_outcome(error.to_string()),
};
text_outcome(format!(
"{}\n{}",
result.stdout.trim_end(),
result.stderr.trim_end()
text_outcome(
format!("{}\n{}", result.stdout.trim_end(), result.stderr.trim_end())
.trim_end()
.to_string(),
)
.trim_end()
.to_string())
}
async fn list_files(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
@ -1295,7 +1329,12 @@ async fn write_file(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
};
match ctx
.sandbox
.write_file(&ctx.computer_ref(), &stored, content.as_bytes(), &ctx.adapter())
.write_file(
&ctx.computer_ref(),
&stored,
content.as_bytes(),
&ctx.adapter(),
)
.await
{
Ok(()) => ToolOutcome {
@ -1438,7 +1477,9 @@ async fn list_saved_accounts(ctx: &ToolCtx) -> ToolOutcome {
})
.collect();
if slim.is_empty() {
text_outcome("No saved logins. Ask the human to add one under 帳號, or call request_takeover so they can sign in on the screen.")
text_outcome(
"No saved logins. Ask the human to add one under 帳號, or call request_takeover so they can sign in on the screen.",
)
} else {
text_outcome(json!({"accounts": slim}).to_string())
}
@ -1640,13 +1681,33 @@ mod connection_check_tests {
use super::*;
#[test]
fn recognizes_connection_wall_but_not_cloudflare_footer() {
for text in ["Cloudflare 驗證您是人類", "Cloudflare Verify you are human", "Dcard 需要確認您的連線是安全的"] {
assert!(is_connection_check(&CdpPage { ok:true, text:text.into(), ..Default::default() }));
for text in [
"Cloudflare 驗證您是人類",
"Cloudflare Verify you are human",
"Dcard 需要確認您的連線是安全的",
] {
assert!(is_connection_check(&CdpPage {
ok: true,
text: text.into(),
..Default::default()
}));
}
for text in ["Article text. Protected by Cloudflare", "Sign in with your password", ""] {
assert!(!is_connection_check(&CdpPage { ok:true, text:text.into(), ..Default::default() }));
for text in [
"Article text. Protected by Cloudflare",
"Sign in with your password",
"",
] {
assert!(!is_connection_check(&CdpPage {
ok: true,
text: text.into(),
..Default::default()
}));
}
assert!(!is_connection_check(&CdpPage { ok:false, text:"Cloudflare Verify you are human".into(), ..Default::default() }));
assert!(!is_connection_check(&CdpPage {
ok: false,
text: "Cloudflare Verify you are human".into(),
..Default::default()
}));
}
}
@ -1686,29 +1747,41 @@ mod shell_session_tests {
argv(json!({"session":"build","keys":"C-c"})),
["lazyboy-shell", "keys", "build", "C-c"]
);
assert_eq!(argv(json!({"reset":true})), ["lazyboy-shell", "reset", "main"]);
assert_eq!(
argv(json!({"reset":true})),
["lazyboy-shell", "reset", "main"]
);
assert!(shell_argv(&json!({"keys":" "}), "main", None).is_err());
}
#[test]
fn cwd_moves_this_command_into_a_directory_and_survives_quotes() {
let argv = shell_argv(&json!({"command":"make\ntest"}), "main", Some("/tmp/a b'c"))
.expect("argv");
let argv =
shell_argv(&json!({"command":"make\ntest"}), "main", Some("/tmp/a b'c")).expect("argv");
assert_eq!(argv[4], "cd -- '/tmp/a b'\\''c' && {\nmake\ntest\n}");
}
#[test]
fn waits_stay_inside_the_range_the_desktop_can_honour() {
assert_eq!(shell_wait_ms(&json!({"wait_ms":900})), 1_000);
assert_eq!(shell_wait_ms(&json!({"wait_ms":900_000})), SHELL_WAIT_MS_MAX);
assert_eq!(
shell_wait_ms(&json!({"wait_ms":900_000})),
SHELL_WAIT_MS_MAX
);
assert_eq!(shell_wait_ms(&json!({})), SHELL_WAIT_MS_DEFAULT);
assert_eq!(shell_log_lines(&json!({"log_lines":0})), 1);
}
#[test]
fn only_a_missing_script_falls_back_to_one_shot() {
assert!(shell_script_missing(127, "bash: lazyboy-shell: command not found"));
assert!(!shell_script_missing(127, "bash: whatever: command not found"));
assert!(shell_script_missing(
127,
"bash: lazyboy-shell: command not found"
));
assert!(!shell_script_missing(
127,
"bash: whatever: command not found"
));
assert!(!shell_script_missing(1, "lazyboy-shell: nope"));
}
}

15
crates/api/src/vault.rs
View File

@ -331,8 +331,14 @@ fn vault_key() -> Result<[u8; 32], String> {
let material = std::env::var("LAZYBOY_VAULT_KEY")
.ok()
.filter(|value| !value.is_empty())
.or_else(|| std::env::var("LAZYBOY_APP_TOKEN").ok().filter(|v| !v.is_empty()))
.ok_or_else(|| "set LAZYBOY_VAULT_KEY or LAZYBOY_APP_TOKEN to encrypt saved passwords".to_string())?;
.or_else(|| {
std::env::var("LAZYBOY_APP_TOKEN")
.ok()
.filter(|v| !v.is_empty())
})
.ok_or_else(|| {
"set LAZYBOY_VAULT_KEY or LAZYBOY_APP_TOKEN to encrypt saved passwords".to_string()
})?;
let digest = Sha256::digest(material.as_bytes());
let mut key = [0u8; 32];
key.copy_from_slice(&digest);
@ -383,7 +389,10 @@ mod tests {
#[test]
fn host_strips_urls() {
assert_eq!(normalize_host("https://mail.google.com/inbox", "Gmail"), "mail.google.com");
assert_eq!(
normalize_host("https://mail.google.com/inbox", "Gmail"),
"mail.google.com"
);
assert_eq!(normalize_host("", "Gmail"), "gmail");
}
}

18
crates/api/src/voice.rs
View File

@ -15,7 +15,10 @@ use crate::voice_call;
pub fn router() -> Router<AppState> {
Router::new()
.route("/api/voice/settings", get(get_settings).patch(update_settings))
.route(
"/api/voice/settings",
get(get_settings).patch(update_settings),
)
.route("/api/sessions/{id}/call", get(voice_call::call_ws))
}
@ -193,7 +196,9 @@ async fn update_settings(
if model_id.is_empty() || voice_id.is_empty() {
return Err(StatusCode::BAD_REQUEST);
}
if catalog_voices(provider).iter().all(|(id, _)| *id != voice_id)
if catalog_voices(provider)
.iter()
.all(|(id, _)| *id != voice_id)
&& provider != VoiceProvider::Scripted
{
return Err(StatusCode::BAD_REQUEST);
@ -210,7 +215,14 @@ async fn update_settings(
};
state
.db
.update_voice_settings(&actor, input.enabled, provider.as_str(), model_id, voice_id, api_key)
.update_voice_settings(
&actor,
input.enabled,
provider.as_str(),
model_id,
voice_id,
api_key,
)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
get_settings(State(state)).await

30
crates/api/src/voice_call.rs
View File

@ -1,11 +1,11 @@
use std::sync::Arc;
use std::time::Duration;
use axum::Json;
use axum::extract::ws::{Message as AxumMessage, WebSocket, WebSocketUpgrade};
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::response::IntoResponse;
use axum::Json;
use futures_util::{SinkExt, StreamExt};
use lazyboy_contracts::SessionAttachment;
use lazyboy_harness::{VoiceConnectRequest, VoiceEvent, VoiceSocket, create_voice};
@ -115,7 +115,9 @@ async fn prepare_call(
.ok_or((StatusCode::NOT_FOUND, "workspace not found".into()))?;
let (provider, resolved) =
resolve_space_voice(&space).map_err(|message| (StatusCode::CONFLICT, message))?;
let history = recent_text_history(state, session_id).await.unwrap_or_default();
let history = recent_text_history(state, session_id)
.await
.unwrap_or_default();
let mut connect = VoiceConnectRequest::from_resolved(
&resolved,
voice_instructions(&bot_name, &bot_instructions),
@ -291,7 +293,8 @@ async fn handle_provider_event(
};
user_partial.clear();
if !body.is_empty() {
let _ = persist_transcript(state, session_id, "user", &body, &prep.call_id).await;
let _ =
persist_transcript(state, session_id, "user", &body, &prep.call_id).await;
send_json(
client_write,
json!({"type":"transcript","role":"user","text":body,"final":true}),
@ -340,8 +343,16 @@ async fn handle_provider_event(
name,
arguments,
} => {
let output = dispatch_voice_tool(state, actor, &prep.bot_id, session_id, &prep.call_id, &name, &arguments)
.await;
let output = dispatch_voice_tool(
state,
actor,
&prep.bot_id,
session_id,
&prep.call_id,
&name,
&arguments,
)
.await;
let _ = provider
.send(VoiceEvent::FunctionCallOutput {
call_id,
@ -543,8 +554,7 @@ async fn watch_computer(
)
.await?;
if let Some(line) = speakable_progress(&previous, &snapshot) {
let urgent = snapshot.get("status").and_then(Value::as_str)
== Some("takeover")
let urgent = snapshot.get("status").and_then(Value::as_str) == Some("takeover")
|| snapshot.get("status").and_then(Value::as_str) == Some("failed");
if urgent || last_spoken_at.elapsed() > Duration::from_secs(6) {
let _ = provider.send(VoiceEvent::SpeakNow { text: line }).await;
@ -652,6 +662,10 @@ mod tests {
);
assert!(speakable_progress(&takeover.to_string(), &takeover).is_none());
let idle = json!({"status":"idle"});
assert!(speakable_progress("{\"status\":\"running\"}", &idle).unwrap().contains("做完"));
assert!(
speakable_progress("{\"status\":\"running\"}", &idle)
.unwrap()
.contains("做完")
);
}
}

6
crates/api/src/web_static.rs
View File

@ -62,10 +62,8 @@ mod tests {
use std::fs;
fn scratch(name: &str) -> PathBuf {
let path = std::env::temp_dir().join(format!(
"lazyboy-web-static-{}-{name}",
std::process::id()
));
let path =
std::env::temp_dir().join(format!("lazyboy-web-static-{}-{name}", std::process::id()));
let _ = fs::remove_dir_all(&path);
fs::create_dir_all(&path).unwrap();
path

3
crates/control/Cargo.toml
View File

@ -16,6 +16,9 @@ hex.workspace = true
chrono.workspace = true
async-trait = "0.1"
image.workspace = true
tokio.workspace = true
tracing.workspace = true
base64.workspace = true
[lints]
workspace = true

5
crates/control/src/a11y.rs
View File

@ -139,7 +139,10 @@ mod tests {
assert!(argv.contains(&"DISPLAY=:2".into()));
assert!(argv.iter().any(|item| item.contains("python3")));
assert!(argv.last().unwrap().contains("\"display\":\":2\""));
assert!(argv.iter().any(|item| item.contains("Atspi") || item.contains("atspi")));
assert!(
argv.iter()
.any(|item| item.contains("Atspi") || item.contains("atspi"))
);
}
#[test]

17
crates/control/src/actions.rs
View File

@ -33,9 +33,12 @@ pub enum ActionError {
/// instead of failing the click.
pub fn element_id(value: Option<&Value>) -> Option<u64> {
match value? {
Value::Number(number) => number
.as_u64()
.or_else(|| number.as_f64().filter(|f| *f >= 0.0).map(|f| f.round() as u64)),
Value::Number(number) => number.as_u64().or_else(|| {
number
.as_f64()
.filter(|f| *f >= 0.0)
.map(|f| f.round() as u64)
}),
Value::String(text) => text
.trim()
.trim_start_matches(['#', '['])
@ -611,9 +614,7 @@ mod tests {
assert!(actions[0].get("x").is_none());
let parsed = parse_computer_actions(&actions).unwrap();
match &parsed[0] {
ComputerAction::Ref {
ref_kind, verb, ..
} => {
ComputerAction::Ref { ref_kind, verb, .. } => {
assert_eq!(ref_kind, "dom");
assert_eq!(*verb, RefVerb::Click);
}
@ -636,9 +637,7 @@ mod tests {
}];
let blocked = browser_gui_block(&json!([{"kind":"click","x":40,"y":80}]), &elements);
assert!(blocked.unwrap().contains("browser"));
assert!(
browser_gui_block(&json!([{"kind":"click","element":1}]), &elements).is_none()
);
assert!(browser_gui_block(&json!([{"kind":"click","element":1}]), &elements).is_none());
}
#[test]

43
crates/control/src/cdp.rs
View File

@ -1,4 +1,5 @@
use lazyboy_contracts::UiElement;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use crate::x11::parse_ui_elements;
@ -52,11 +53,7 @@ pub fn cdp_stdin_command_on(display: &str, profile: Option<&str>) -> Vec<String>
if let Some(profile) = profile.filter(|value| !value.is_empty()) {
env.push(format!("LAZYBOY_BROWSER_PROFILE={profile}"));
}
env.extend([
"python3".into(),
"-c".into(),
CDP_PY.into(),
]);
env.extend(["python3".into(), "-c".into(), CDP_PY.into()]);
env
}
@ -70,6 +67,23 @@ pub fn teach_recorder_output(skill_id: &str) -> String {
format!("/tmp/{}.jsonl", teach_recorder_tag(skill_id))
}
pub fn sanitize_skill_id(skill_id: &str) -> String {
let cleaned: String = skill_id
.chars()
.filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_')
.take(80)
.collect();
if cleaned.is_empty() {
"unknown".into()
} else {
cleaned
}
}
pub fn teach_trajectory_dir(skill_id: &str) -> String {
format!("/tmp/lazyboy/teach-{}", sanitize_skill_id(skill_id))
}
/// Detached, long-running CDP recorder for a human demonstration. The script
/// is handed to `sh` as a positional argument so no shell quoting touches it.
pub fn cdp_record_command_on(display: &str, profile: Option<&str>, skill_id: &str) -> Vec<String> {
@ -87,7 +101,8 @@ pub fn cdp_record_command_on(display: &str, profile: Option<&str>, skill_id: &st
vec![
"sh".into(),
"-c".into(),
"setsid nohup env DISPLAY=\"$0\" python3 -c \"$1\" \"$2\" >/dev/null 2>&1 </dev/null &".into(),
"setsid nohup env DISPLAY=\"$0\" python3 -c \"$1\" \"$2\" >/dev/null 2>&1 </dev/null &"
.into(),
normalize_display(display).to_string(),
CDP_PY.into(),
request.to_string(),
@ -98,16 +113,24 @@ pub fn cdp_record_stop_command(skill_id: &str) -> Vec<String> {
vec!["pkill".into(), "-f".into(), teach_recorder_tag(skill_id)]
}
#[derive(Debug, Clone, PartialEq, Default)]
#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CdpPage {
pub ok: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
#[serde(default)]
pub url: String,
#[serde(default)]
pub title: String,
#[serde(default)]
pub text: String,
#[serde(default)]
pub restarted: bool,
/// Seconds the click waited for a disabled control to become enabled.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub waited_seconds: Option<f64>,
#[serde(default)]
pub elements: Vec<UiElement>,
}
@ -164,6 +187,12 @@ mod tests {
assert_eq!(devtools_port("3"), 9224);
}
#[test]
fn trajectory_dir_strips_path_chars() {
assert_eq!(teach_trajectory_dir("abc/../x"), "/tmp/lazyboy/teach-abcx");
assert_eq!(sanitize_skill_id(""), "unknown");
}
#[test]
fn command_passes_port_profile_and_script() {
let argv = cdp_command_on(

215
crates/control/src/controller.rs Normal file
View File

@ -0,0 +1,215 @@
use std::str::FromStr;
use std::sync::Arc;
use async_trait::async_trait;
use thiserror::Error;
use crate::cua::CuaController;
use crate::legacy::LegacyController;
use crate::{
ActionRequest, ActionResult, BrowserRequest, CdpPage, RecordingRequest, RecordingResult,
RecordingSession,
};
use lazyboy_contracts::ComputerObservation;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ControlContext {
pub display: String,
pub profile_path: Option<String>,
}
impl ControlContext {
pub fn new(display: impl Into<String>, profile_path: Option<String>) -> Self {
Self {
display: display.into(),
profile_path,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ControllerHealth {
pub backend: String,
pub version: Option<String>,
pub healthy: bool,
pub degraded: bool,
pub details: Vec<String>,
}
#[derive(Debug, Error, Clone, PartialEq, Eq)]
pub enum ControlError {
#[error("computer driver is not installed")]
DriverUnavailable,
#[error("computer driver is unhealthy")]
DriverUnhealthy,
#[error("display is unavailable")]
DisplayUnavailable,
#[error("accessibility is unavailable")]
AccessibilityUnavailable,
#[error("browser is unavailable")]
BrowserUnavailable,
#[error("target not found")]
TargetNotFound,
#[error("stale UI reference; take a fresh observation")]
StaleReference,
#[error("permission denied")]
PermissionDenied,
#[error("computer action timed out")]
Timeout,
#[error("unsupported computer action")]
Unsupported,
#[error("computer is busy")]
Busy,
#[error("{0}")]
InvalidAction(String),
#[error("{0}")]
Internal(String),
}
impl ControlError {
pub fn internal(text: impl Into<String>) -> Self {
let text = text.into();
Self::Internal(truncate_error(&text))
}
pub fn is_client_error(&self) -> bool {
matches!(
self,
Self::TargetNotFound
| Self::StaleReference
| Self::Unsupported
| Self::InvalidAction(_)
| Self::PermissionDenied
)
}
}
fn truncate_error(text: &str) -> String {
const LIMIT: usize = 800;
let trimmed = text.trim();
if trimmed.len() <= LIMIT {
trimmed.to_string()
} else {
format!("{}…", &trimmed[..LIMIT])
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ComputerDriver {
Legacy,
Cua,
}
impl ComputerDriver {
pub const ENV: &'static str = "LAZYBOY_COMPUTER_DRIVER";
pub fn from_env() -> Self {
match std::env::var(Self::ENV) {
Ok(value) if value.trim().is_empty() => Self::Cua,
Ok(value) => match value.parse() {
Ok(driver) => driver,
Err(error) => {
tracing::error!("{error}; using cua");
Self::Cua
}
},
Err(_) => Self::Cua,
}
}
pub fn as_str(self) -> &'static str {
match self {
Self::Legacy => "legacy",
Self::Cua => "cua",
}
}
pub fn controller(self) -> Arc<dyn ComputerController> {
match self {
Self::Legacy => Arc::new(LegacyController),
Self::Cua => Arc::new(CuaController::default()),
}
}
}
impl FromStr for ComputerDriver {
type Err = UnknownComputerDriver;
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value.trim().to_ascii_lowercase().as_str() {
"legacy" => Ok(Self::Legacy),
"cua" => Ok(Self::Cua),
other => Err(UnknownComputerDriver(other.to_string())),
}
}
}
#[derive(Debug, Error, Clone, PartialEq, Eq)]
#[error("unknown computer driver {0:?}; expected legacy or cua")]
pub struct UnknownComputerDriver(pub String);
#[async_trait]
pub trait ComputerController: Send + Sync {
fn backend(&self) -> ComputerDriver;
async fn health(&self, ctx: &ControlContext) -> Result<ControllerHealth, ControlError>;
async fn observe(&self, ctx: &ControlContext) -> Result<ComputerObservation, ControlError>;
async fn act(
&self,
request: &ActionRequest,
ctx: &ControlContext,
) -> Result<ActionResult, ControlError>;
async fn browser(
&self,
request: &BrowserRequest,
ctx: &ControlContext,
) -> Result<CdpPage, ControlError>;
async fn start_recording(
&self,
request: &RecordingRequest,
ctx: &ControlContext,
) -> Result<RecordingSession, ControlError>;
async fn stop_recording(
&self,
request: &RecordingRequest,
ctx: &ControlContext,
) -> Result<(), ControlError>;
async fn collect_recording(
&self,
request: &RecordingRequest,
ctx: &ControlContext,
) -> Result<RecordingResult, ControlError>;
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_driver_names() {
assert_eq!(
"legacy".parse::<ComputerDriver>().unwrap(),
ComputerDriver::Legacy
);
assert_eq!(
"CUA".parse::<ComputerDriver>().unwrap(),
ComputerDriver::Cua
);
assert!("xdotool".parse::<ComputerDriver>().is_err());
assert_eq!(ComputerDriver::Cua.as_str(), "cua");
}
#[test]
fn invalid_browser_action_is_a_client_error() {
let error = ControlError::InvalidAction(
"browser navigate only accepts http, https, or about URLs".into(),
);
assert!(error.is_client_error());
}
}

602
crates/control/src/cua/browser.rs Normal file
View File

@ -0,0 +1,602 @@
use lazyboy_contracts::UiElement;
use serde_json::{Value, json};
use tokio::time::{Duration, sleep};
use super::ListedWindow;
use super::client::{CuaClient, first_array_of_objects};
use crate::controller::ControlError;
use crate::process::spawn_detached;
use crate::{BrowserRequest, CdpPage, launch_argv_on};
pub const SESSION: &str = "lazyboy";
#[derive(Debug, Clone)]
pub struct BrowserBind {
pub pid: u64,
pub window_id: u64,
pub target_id: String,
pub tab_id: String,
}
pub fn is_cua_ref(selector: &str) -> bool {
let mut parts = selector.split(':');
matches!(
(parts.next(), parts.next(), parts.next()),
(Some(prefix), Some(index), None)
if prefix.starts_with('p')
&& prefix[1..].chars().all(|ch| ch.is_ascii_digit())
&& !prefix[1..].is_empty()
&& index.chars().all(|ch| ch.is_ascii_digit())
&& !index.is_empty()
)
}
pub fn allowed_navigate_url(url: &str) -> bool {
url.starts_with("http://") || url.starts_with("https://") || url.starts_with("about:")
}
pub fn page_from_semantic(value: &Value) -> CdpPage {
let page = value.get("page").unwrap_or(value);
let url = page
.get("url")
.and_then(Value::as_str)
.unwrap_or("")
.to_string();
let title = page
.get("title")
.and_then(Value::as_str)
.unwrap_or("")
.to_string();
let outline = value.get("outline").and_then(Value::as_str).unwrap_or("");
let mut elements = Vec::new();
for (index, item) in first_array_of_objects(value, "ref").into_iter().enumerate() {
let Some(element) = element_from_ref(index as u32 + 1, item) else {
continue;
};
elements.push(element);
}
let ok = value.get("status").and_then(Value::as_str) != Some("refused")
&& value.get("ok").and_then(Value::as_bool) != Some(false);
CdpPage {
ok,
error: if ok {
None
} else {
value
.get("message")
.or_else(|| value.get("error"))
.and_then(Value::as_str)
.map(str::to_string)
},
url,
title,
text: outline.to_string(),
restarted: false,
waited_seconds: None,
elements,
}
}
fn element_from_ref(id: u32, item: &Value) -> Option<UiElement> {
let selector = item.get("ref").and_then(Value::as_str)?.to_string();
if selector.is_empty() {
return None;
}
let name = item
.get("name")
.or_else(|| item.get("label"))
.and_then(Value::as_str)
.unwrap_or("")
.to_string();
let role = item
.get("role")
.and_then(Value::as_str)
.filter(|role| !role.is_empty())
.map(str::to_string);
let visibility = item
.get("visibility")
.and_then(Value::as_str)
.unwrap_or("in_viewport");
let (x, y, w, h) = match item.get("frame") {
Some(frame) => (
number(frame, "x").unwrap_or(0),
number(frame, "y").unwrap_or(0),
number(frame, "w")
.or_else(|| number(frame, "width"))
.unwrap_or(0),
number(frame, "h")
.or_else(|| number(frame, "height"))
.unwrap_or(0),
),
None if visibility == "in_viewport" => (0, 0, 1, 1),
None => (0, 0, 0, 0),
};
Some(UiElement {
id,
title: if name.is_empty() {
selector.clone()
} else {
name
},
x,
y,
w,
h,
selector: Some(selector),
kind: Some("dom".into()),
role,
})
}
fn number(value: &Value, key: &str) -> Option<u32> {
value
.get(key)
.and_then(Value::as_u64)
.or_else(|| {
value
.get(key)
.and_then(Value::as_f64)
.filter(|n| *n >= 0.0)
.map(|n| n as u64)
})
.map(|n| n as u32)
}
pub fn find_ref<'a>(page: &'a CdpPage, selector: &'a str) -> Option<&'a str> {
if is_cua_ref(selector) {
return page
.elements
.iter()
.find(|element| element.selector.as_deref() == Some(selector))
.and_then(|element| element.selector.as_deref())
.or(Some(selector));
}
if let Ok(id) = selector.parse::<u32>() {
return page
.elements
.iter()
.find(|element| element.id == id)
.and_then(|element| element.selector.as_deref());
}
let needle = selector
.trim_start_matches(['#', '.', '['])
.trim_end_matches(']')
.to_ascii_lowercase();
page.elements.iter().find_map(|element| {
let title = element.title.to_ascii_lowercase();
let role = element
.role
.clone()
.unwrap_or_default()
.to_ascii_lowercase();
if title.contains(&needle) || role == needle {
element.selector.as_deref()
} else {
None
}
})
}
pub fn chromium_window(windows: &[ListedWindow]) -> Option<&ListedWindow> {
windows.iter().find(|window| {
let blob = format!("{} {}", window.title, window.app_name).to_ascii_lowercase();
blob.contains("chrom")
})
}
fn ids_from(value: &Value) -> Option<(String, String)> {
let mut target_id = None;
let mut tab_id = None;
let mut nodes = Vec::new();
super::client::walk(value, &mut nodes);
for node in nodes {
if target_id.is_none() {
target_id = node
.get("target_id")
.and_then(Value::as_str)
.map(str::to_string);
}
if tab_id.is_none() {
tab_id = node
.get("tab_id")
.and_then(Value::as_str)
.map(str::to_string);
}
if let Some(tabs) = node.get("tabs").and_then(Value::as_array)
&& let Some(first) = tabs.first()
&& tab_id.is_none()
{
tab_id = first
.get("tab_id")
.or_else(|| first.get("id"))
.and_then(Value::as_str)
.map(str::to_string);
}
}
Some((target_id?, tab_id?))
}
pub async fn ensure_bind(
client: &CuaClient,
display: &str,
profile: Option<&str>,
ensure: bool,
windows: &[ListedWindow],
) -> Result<BrowserBind, ControlError> {
let mut listed = windows.to_vec();
if chromium_window(&listed).is_none() && ensure {
if let Some(argv) = launch_argv_on(display, profile, "browser", None) {
spawn_detached(&argv)
.await
.map_err(ControlError::internal)?;
}
for _ in 0..24 {
sleep(Duration::from_millis(250)).await;
listed = super::CuaController::list_windows_now(client, display).await?;
if chromium_window(&listed).is_some() {
break;
}
}
}
let window = chromium_window(&listed)
.cloned()
.ok_or(ControlError::BrowserUnavailable)?;
attach(client, display, &window).await
}
async fn attach(
client: &CuaClient,
display: &str,
window: &ListedWindow,
) -> Result<BrowserBind, ControlError> {
let pid = window.pid;
let window_id = window.id;
let prepare = client
.call(
display,
"browser_prepare",
&json!({
"pid": pid,
"window_id": window_id,
"session": SESSION,
"strategy": { "kind": "existing_profile" },
"allow_launch": false,
}),
&[],
)
.await;
if let Err(error) = &prepare {
let text = error.to_string();
if !text.contains("consent") && !text.contains("prepare") && !text.contains("grant") {
tracing::warn!(error = %error, "browser_prepare failed");
}
}
let state = client
.call(
display,
"get_browser_state",
&json!({
"pid": pid,
"window_id": window_id,
"session": SESSION,
"include_screenshot": false,
}),
&[],
)
.await?;
let (target_id, tab_id) = ids_from(&state).ok_or(ControlError::BrowserUnavailable)?;
Ok(BrowserBind {
pid,
window_id,
target_id,
tab_id,
})
}
pub async fn snapshot(
client: &CuaClient,
display: &str,
bind: &BrowserBind,
) -> Result<CdpPage, ControlError> {
let value = client
.call(
display,
"get_browser_state",
&json!({
"target_id": bind.target_id,
"tab_id": bind.tab_id,
"session": SESSION,
"snapshot_format": "semantic_v2",
"include_screenshot": false,
}),
&[],
)
.await?;
Ok(page_from_semantic(&value))
}
pub async fn run(
client: &CuaClient,
display: &str,
profile: Option<&str>,
request: &BrowserRequest,
windows: &[ListedWindow],
bind: &mut Option<BrowserBind>,
) -> Result<CdpPage, ControlError> {
if request.action == "probe" {
return Ok(CdpPage {
ok: chromium_window(windows).is_some(),
..CdpPage::default()
});
}
let attached = match bind.as_ref() {
Some(current) => current.clone(),
None => {
let attached = ensure_bind(client, display, profile, request.ensure, windows).await?;
*bind = Some(attached.clone());
attached
}
};
if request.action == "ensure" {
return Ok(CdpPage {
ok: true,
..CdpPage::default()
});
}
match request.action.as_str() {
"snapshot" => snapshot(client, display, &attached).await,
"wait" => {
let ms = request.ms.unwrap_or(400).min(5000);
sleep(Duration::from_millis(ms)).await;
snapshot(client, display, &attached).await
}
"navigate" => {
let url = request.url.as_deref().unwrap_or("");
if !allowed_navigate_url(url) {
return Err(ControlError::InvalidAction(
"browser navigate only accepts http, https, or about URLs".into(),
));
}
client
.call(
display,
"browser_navigate",
&json!({
"target_id": attached.target_id,
"tab_id": attached.tab_id,
"session": SESSION,
"url": url,
}),
&[],
)
.await?;
sleep(Duration::from_millis(800)).await;
snapshot(client, display, &attached).await
}
"click" => click(client, display, &attached, request).await,
"type" => type_into(client, display, &attached, request).await,
"press" => {
let key = map_press_key(request.key.as_deref().unwrap_or("return"));
client
.call(
display,
"press_key",
&json!({
"key": key,
"pid": attached.pid,
"window_id": attached.window_id,
"session": SESSION,
}),
&[],
)
.await?;
sleep(Duration::from_millis(200)).await;
snapshot(client, display, &attached).await
}
other => Err(ControlError::InvalidAction(format!(
"unsupported browser action {other}"
))),
}
}
async fn click(
client: &CuaClient,
display: &str,
bind: &BrowserBind,
request: &BrowserRequest,
) -> Result<CdpPage, ControlError> {
let selector = request
.selector
.as_deref()
.ok_or_else(|| ControlError::InvalidAction("browser click needs a selector".into()))?;
let wait_ms = request.wait_ms.unwrap_or(45_000).min(120_000);
let deadline = tokio::time::Instant::now() + Duration::from_millis(wait_ms);
let mut waited = 0.0f64;
loop {
let page = snapshot(client, display, bind).await?;
let Some(r#ref) = find_ref(&page, selector) else {
return Ok(CdpPage {
ok: false,
error: Some(
"element gone: the page changed and ids were renumbered. Use the fresh element list in this result."
.into(),
),
url: page.url,
title: page.title,
text: page.text,
elements: page.elements,
..CdpPage::default()
});
};
let r#ref = r#ref.to_string();
match client
.call(
display,
"browser_click",
&json!({
"target_id": bind.target_id,
"tab_id": bind.tab_id,
"session": SESSION,
"ref": r#ref,
"input_route": "dom_event",
}),
&[],
)
.await
{
Ok(_) => {
sleep(Duration::from_millis(250)).await;
let mut after = snapshot(client, display, bind).await?;
if waited >= 1.0 {
after.waited_seconds = Some((waited * 10.0).round() / 10.0);
}
return Ok(after);
}
Err(error) if tokio::time::Instant::now() < deadline => {
waited += 0.5;
tracing::info!(error = %error, "browser click retrying");
sleep(Duration::from_millis(500)).await;
}
Err(error) => return Err(error),
}
}
}
async fn type_into(
client: &CuaClient,
display: &str,
bind: &BrowserBind,
request: &BrowserRequest,
) -> Result<CdpPage, ControlError> {
let text = request.text.clone().unwrap_or_default();
tracing::info!(backend = "cua", tool = "browser_type", length = text.len());
if let Some(selector) = request.selector.as_deref() {
let page = snapshot(client, display, bind).await?;
let Some(r#ref) = find_ref(&page, selector) else {
return Ok(CdpPage {
ok: false,
error: Some("target field is unavailable; no text inserted".into()),
url: page.url,
title: page.title,
text: page.text,
elements: page.elements,
..CdpPage::default()
});
};
let r#ref = r#ref.to_string();
client
.call(
display,
"browser_type",
&json!({
"target_id": bind.target_id,
"tab_id": bind.tab_id,
"session": SESSION,
"ref": r#ref,
"text": text,
"replace": true,
}),
&[],
)
.await?;
} else if !text.is_empty() {
client
.call(
display,
"type_text",
&json!({
"text": text,
"pid": bind.pid,
"window_id": bind.window_id,
"session": SESSION,
}),
&[],
)
.await?;
}
sleep(Duration::from_millis(200)).await;
snapshot(client, display, bind).await
}
fn map_press_key(key: &str) -> String {
match key.to_ascii_lowercase().as_str() {
"enter" | "return" => "return".into(),
"esc" | "escape" => "escape".into(),
other => other.to_ascii_lowercase(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn detects_snapshot_scoped_refs() {
assert!(is_cua_ref("p1:1"));
assert!(is_cua_ref("p28:12"));
assert!(!is_cua_ref("#submit"));
assert!(!is_cua_ref("button.primary"));
assert!(!is_cua_ref("p:1"));
}
#[test]
fn semantic_snapshot_becomes_cdp_page() {
let raw = json!({
"status": "ok",
"outline": "- button \"Smoke Click\"\n- textbox \"Smoke Entry\"",
"page": { "title": "LazyBoy Cua Smoke", "url": "http://127.0.0.1:8765/cua-smoke.html" },
"refs": [
{ "name": "Smoke Click", "ref": "p1:1", "role": "button", "visibility": "in_viewport" },
{ "name": "Smoke Entry", "ref": "p1:2", "role": "textbox", "visibility": "in_viewport" }
]
});
let page = page_from_semantic(&raw);
assert!(page.ok);
assert_eq!(page.title, "LazyBoy Cua Smoke");
assert_eq!(page.elements.len(), 2);
assert_eq!(page.elements[0].id, 1);
assert_eq!(page.elements[0].selector.as_deref(), Some("p1:1"));
assert_eq!(page.elements[0].kind.as_deref(), Some("dom"));
assert_eq!(find_ref(&page, "1"), Some("p1:1"));
assert_eq!(find_ref(&page, "p1:1"), Some("p1:1"));
assert_eq!(find_ref(&page, "Smoke Entry"), Some("p1:2"));
}
#[test]
fn navigate_accepts_http_https_about_only() {
assert!(allowed_navigate_url("https://example.com"));
assert!(allowed_navigate_url("http://127.0.0.1:8765/cua-smoke.html"));
assert!(allowed_navigate_url("about:blank"));
assert!(!allowed_navigate_url("file:///tmp/x.html"));
assert!(!allowed_navigate_url("javascript:alert(1)"));
assert!(!allowed_navigate_url(""));
}
#[test]
fn chromium_window_matches_title_or_app() {
let chrome = ListedWindow {
id: 9,
pid: 334,
title: "LazyBoy Cua Smoke - Chromium".into(),
app_name: "Chromium".into(),
x: 0,
y: 0,
w: 1280,
h: 800,
z: 2,
};
let terminal = ListedWindow {
id: 3,
pid: 20,
title: "終端機".into(),
app_name: "xfce4-terminal".into(),
x: 10,
y: 10,
w: 400,
h: 300,
z: 1,
};
assert!(chromium_window(&[terminal.clone(), chrome.clone()]).is_some());
assert!(chromium_window(&[terminal]).is_none());
}
}

238
crates/control/src/cua/client.rs Normal file
View File

@ -0,0 +1,238 @@
use std::path::{Path, PathBuf};
use std::time::Instant;
use serde_json::Value;
use tokio::process::Command;
use crate::controller::ControlError;
use crate::screen::normalize_display;
pub const PRIMARY_SOCKET: &str = "/tmp/lazyboy/cua.sock";
#[derive(Debug, Clone)]
pub struct CuaClient {
bin: PathBuf,
}
impl Default for CuaClient {
fn default() -> Self {
Self {
bin: PathBuf::from("cua-driver"),
}
}
}
impl CuaClient {
pub fn socket_for_display(display: &str) -> PathBuf {
let number = normalize_display(display)
.trim_start_matches(':')
.to_string();
if number == "1" {
PathBuf::from(PRIMARY_SOCKET)
} else {
PathBuf::from(format!("/tmp/lazyboy/cua-{number}.sock"))
}
}
pub fn dbus_file(display: &str) -> PathBuf {
let number = normalize_display(display)
.trim_start_matches(':')
.to_string();
PathBuf::from(format!("/tmp/lazyboy/screen-{number}.dbus"))
}
pub async fn version(&self) -> Result<String, ControlError> {
let output = Command::new(&self.bin)
.arg("--version")
.output()
.await
.map_err(|_| ControlError::DriverUnavailable)?;
if !output.status.success() {
return Err(ControlError::DriverUnavailable);
}
Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
}
pub async fn status(&self, display: &str) -> Result<String, ControlError> {
let socket = Self::socket_for_display(display);
if !socket.exists() {
return Err(ControlError::DriverUnavailable);
}
let output = Command::new(&self.bin)
.args(["status", "--socket", &socket.to_string_lossy()])
.output()
.await
.map_err(|_| ControlError::DriverUnavailable)?;
let text = format!(
"{}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
if !output.status.success() || text.to_ascii_lowercase().contains("not running") {
return Err(ControlError::DriverUnhealthy);
}
Ok(text)
}
pub async fn call(
&self,
screen: &str,
tool: &str,
payload: &Value,
extra: &[&str],
) -> Result<Value, ControlError> {
let socket = Self::socket_for_display(screen);
if !socket.exists() {
return Err(ControlError::DriverUnavailable);
}
let mut command = Command::new(&self.bin);
command
.env("DISPLAY", normalize_display(screen))
.env(
"CUA_DRIVER_RS_HOME",
std::env::var("CUA_DRIVER_RS_HOME")
.unwrap_or_else(|_| "/tmp/lazyboy/cua-home".into()),
)
.args(["call", "--socket", &socket.to_string_lossy()]);
command.args(extra);
command.arg(tool);
command.arg(payload.to_string());
apply_desktop_bus(&mut command, screen);
let started = Instant::now();
let output = command
.output()
.await
.map_err(|error| ControlError::internal(error.to_string()))?;
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
let combined = format!("{stdout}\n{stderr}");
tracing::info!(
backend = "cua",
tool,
screen,
duration_ms = started.elapsed().as_millis() as u64,
success = output.status.success() && !combined.contains('❌')
);
if !output.status.success() || combined.contains('❌') {
return Err(classify_cua_failure(&combined));
}
Ok(parse_jsonish(&stdout).unwrap_or(Value::String(stdout.trim().to_string())))
}
}
fn apply_desktop_bus(command: &mut Command, display: &str) {
let dbus = CuaClient::dbus_file(display);
if let Ok(address) = std::fs::read_to_string(&dbus) {
let address = address.trim();
if !address.is_empty() {
command.env("DBUS_SESSION_BUS_ADDRESS", address);
}
}
let number = normalize_display(display)
.trim_start_matches(':')
.to_string();
let runtime = PathBuf::from(format!("/tmp/lazyboy/screen-{number}.runtime"));
if let Ok(value) = std::fs::read_to_string(runtime) {
let value = value.trim();
if !value.is_empty() {
command.env("XDG_RUNTIME_DIR", value);
}
}
}
pub fn parse_jsonish(text: &str) -> Option<Value> {
let text = text.trim();
if text.is_empty() {
return None;
}
if let Ok(value) = serde_json::from_str::<Value>(text) {
return Some(value);
}
let start = text.find('{')?;
let end = text.rfind('}')?;
if end > start {
serde_json::from_str(&text[start..=end]).ok()
} else {
None
}
}
pub fn walk<'a>(value: &'a Value, out: &mut Vec<&'a Value>) {
out.push(value);
match value {
Value::Object(map) => {
for item in map.values() {
walk(item, out);
}
}
Value::Array(items) => {
for item in items {
walk(item, out);
}
}
_ => {}
}
}
pub fn first_array_of_objects<'a>(value: &'a Value, required: &str) -> Vec<&'a Value> {
let mut nodes = Vec::new();
walk(value, &mut nodes);
for node in nodes {
if let Some(items) = node.as_array()
&& items.iter().any(|item| item.get(required).is_some())
{
return items.iter().collect();
}
if let Some(items) = node.get(required).and_then(Value::as_array)
&& items.iter().any(Value::is_object)
{
return items.iter().collect();
}
}
Vec::new()
}
fn classify_cua_failure(text: &str) -> ControlError {
let lower = text.to_ascii_lowercase();
if lower.contains("stale") {
ControlError::StaleReference
} else if lower.contains("not_found") || lower.contains("not found") {
ControlError::TargetNotFound
} else if lower.contains("timeout") {
ControlError::Timeout
} else if lower.contains("permission") || lower.contains("consent") {
ControlError::PermissionDenied
} else if lower.contains("unsupported") {
ControlError::Unsupported
} else if Path::new(PRIMARY_SOCKET)
.parent()
.is_some_and(|dir| !dir.exists())
{
ControlError::DriverUnavailable
} else {
ControlError::internal(text)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn primary_display_uses_well_known_socket() {
assert_eq!(
CuaClient::socket_for_display(":1"),
PathBuf::from(PRIMARY_SOCKET)
);
assert_eq!(
CuaClient::socket_for_display(":2"),
PathBuf::from("/tmp/lazyboy/cua-2.sock")
);
}
#[test]
fn extracts_json_object_from_noisy_stdout() {
let parsed = parse_jsonish("✅ ok\n{\"status\":\"ok\",\"x\":1}\n").unwrap();
assert_eq!(parsed["status"], "ok");
}
}

532
crates/control/src/cua/mod.rs Normal file
View File

@ -0,0 +1,532 @@
mod browser;
mod client;
mod record;
mod translate;
use std::collections::HashMap;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use async_trait::async_trait;
use lazyboy_contracts::{ActiveWindow, ComputerObservation, CursorPosition, UiElement};
use serde_json::{Value, json};
use tokio::time::{Duration, sleep};
use crate::controller::{
ComputerController, ComputerDriver, ControlContext, ControlError, ControllerHealth,
};
use crate::legacy::apply_action as legacy_apply_action;
use crate::process::spawn_detached;
use crate::{
ActionRequest, ActionResult, BrowserRequest, CdpPage, RecordingRequest, RecordingResult,
RecordingSession, action_pause_ms, normalize_display, observation_from_png,
observation_with_elements, teach_trajectory_dir,
};
use client::first_array_of_objects;
pub use client::CuaClient;
pub use translate::{TranslatedAction, translate_action};
#[derive(Debug, Default)]
pub struct CuaController {
client: CuaClient,
browser: tokio::sync::Mutex<HashMap<String, browser::BrowserBind>>,
}
#[async_trait]
impl ComputerController for CuaController {
fn backend(&self) -> ComputerDriver {
ComputerDriver::Cua
}
async fn health(&self, ctx: &ControlContext) -> Result<ControllerHealth, ControlError> {
let version = self.client.version().await.ok();
match self.client.status(&ctx.display).await {
Ok(status) => Ok(ControllerHealth {
backend: ComputerDriver::Cua.as_str().to_string(),
version,
healthy: true,
degraded: false,
details: vec![status.trim().to_string()],
}),
Err(error) => Ok(ControllerHealth {
backend: ComputerDriver::Cua.as_str().to_string(),
version,
healthy: false,
degraded: true,
details: vec![error.to_string()],
}),
}
}
async fn observe(&self, ctx: &ControlContext) -> Result<ComputerObservation, ControlError> {
self.browser
.lock()
.await
.remove(normalize_display(&ctx.display));
self.observe_display(&ctx.display).await
}
async fn act(
&self,
request: &ActionRequest,
ctx: &ControlContext,
) -> Result<ActionResult, ControlError> {
let display = ctx.display.as_str();
let profile = ctx.profile_path.as_deref();
let mut completed = 0usize;
for action in &request.actions {
match translate_action(action, display, profile) {
Ok(translated) => self.dispatch(display, translated).await?,
Err(ControlError::Unsupported) => {
legacy_apply_action(display, profile, action).await?;
}
Err(error) => return Err(error),
}
let pause = action_pause_ms(action);
if pause > 0 {
sleep(Duration::from_millis(pause)).await;
}
completed += 1;
}
if request.settle_ms > 0 {
sleep(Duration::from_millis(u64::from(request.settle_ms))).await;
}
let observation = if request.observe {
Some(self.observe_display(display).await?)
} else {
None
};
Ok(ActionResult {
completed,
observation,
})
}
async fn browser(
&self,
request: &BrowserRequest,
ctx: &ControlContext,
) -> Result<CdpPage, ControlError> {
let key = normalize_display(&ctx.display).to_string();
let mut windows = self.windows(&ctx.display).await.unwrap_or_default();
let mut cache = self.browser.lock().await;
for attempt in 0..2 {
let mut bind = cache.remove(&key);
let had_bind = bind.is_some();
match browser::run(
&self.client,
&ctx.display,
ctx.profile_path.as_deref(),
request,
&windows,
&mut bind,
)
.await
{
Err(ControlError::BrowserUnavailable) if !had_bind => {
return Ok(CdpPage {
ok: false,
error: Some("cdp unavailable".into()),
..CdpPage::default()
});
}
Err(error)
if attempt == 0
&& had_bind
&& matches!(
error,
ControlError::BrowserUnavailable
| ControlError::StaleReference
| ControlError::TargetNotFound
) =>
{
windows = self.windows(&ctx.display).await.unwrap_or_default();
}
other => {
if let Some(current) = bind {
cache.insert(key, current);
}
return map_browser_unavailable(other);
}
}
}
map_browser_unavailable(Err(ControlError::BrowserUnavailable))
}
async fn start_recording(
&self,
request: &RecordingRequest,
ctx: &ControlContext,
) -> Result<RecordingSession, ControlError> {
if request.skill_id.trim().is_empty() {
return Err(ControlError::InvalidAction(
"recording needs a skill id".into(),
));
}
let output_dir = teach_trajectory_dir(&request.skill_id);
let _ = tokio::fs::create_dir_all(&output_dir).await;
let _ = self
.client
.call(&ctx.display, "stop_recording", &json!({}), &[])
.await;
if let Err(error) = self
.client
.call(
&ctx.display,
"start_recording",
&json!({ "output_dir": output_dir, "record_video": false }),
&[],
)
.await
{
tracing::warn!(error = %error, "cua start_recording failed");
}
if let Err(error) = crate::legacy::start_cdp_recorder(
&ctx.display,
ctx.profile_path.as_deref(),
&request.skill_id,
)
.await
{
tracing::warn!(error = %error, "cdp recorder start failed");
}
Ok(RecordingSession {
skill_id: request.skill_id.clone(),
output_dir,
})
}
async fn stop_recording(
&self,
request: &RecordingRequest,
ctx: &ControlContext,
) -> Result<(), ControlError> {
let _ = self
.client
.call(&ctx.display, "stop_recording", &json!({}), &[])
.await;
crate::legacy::stop_cdp_recorder(&request.skill_id).await
}
async fn collect_recording(
&self,
request: &RecordingRequest,
_ctx: &ControlContext,
) -> Result<RecordingResult, ControlError> {
let mut events = crate::legacy::collect_cdp_events(&request.skill_id).await;
let dir = teach_trajectory_dir(&request.skill_id);
events.extend(record::events_from_dir(std::path::Path::new(&dir)));
events.sort_by_key(|event| event.get("at").and_then(Value::as_i64).unwrap_or(0));
let _ = tokio::fs::remove_dir_all(&dir).await;
Ok(RecordingResult { events })
}
}
impl CuaController {
async fn observe_display(&self, display: &str) -> Result<ComputerObservation, ControlError> {
let png_path = observe_png_path(display);
let _ = tokio::fs::remove_file(&png_path).await;
self.client
.call(
display,
"get_desktop_state",
&json!({ "screenshot_out_file": png_path.to_string_lossy() }),
&["--screenshot-out-file", &png_path.to_string_lossy()],
)
.await?;
let png = tokio::fs::read(&png_path)
.await
.map_err(|_| ControlError::Internal("screenshot failed".into()))?;
let _ = tokio::fs::remove_file(&png_path).await;
if png.is_empty() {
return Err(ControlError::Internal("screenshot failed".into()));
}
let (width, height) = png_dimensions(&png);
let cursor = self.cursor(display).await;
let windows = self.windows(display).await.unwrap_or_default();
let active = windows
.iter()
.max_by_key(|window| window.z)
.map(|window| ActiveWindow {
id: window.id.to_string(),
title: Some(window.title.clone()).filter(|title| !title.is_empty()),
});
let elements: Vec<UiElement> = windows
.into_iter()
.enumerate()
.map(|(index, window)| UiElement {
id: (index + 1) as u32,
title: window.title.chars().take(80).collect(),
x: window.x,
y: window.y,
w: window.w,
h: window.h,
selector: None,
kind: Some("window".into()),
role: None,
})
.collect();
Ok(observation_with_elements(
observation_from_png(png, width, height, cursor, active),
elements,
))
}
async fn cursor(&self, display: &str) -> Option<CursorPosition> {
let value = self
.client
.call(display, "get_cursor_position", &json!({}), &[])
.await
.ok()?;
cursor_from_value(&value)
}
async fn windows(&self, display: &str) -> Result<Vec<ListedWindow>, ControlError> {
Self::list_windows_now(&self.client, display).await
}
pub(crate) async fn list_windows_now(
client: &CuaClient,
display: &str,
) -> Result<Vec<ListedWindow>, ControlError> {
let value = client
.call(
display,
"list_windows",
&json!({ "on_screen_only": true }),
&[],
)
.await?;
Ok(parse_listed_windows(&value))
}
async fn dispatch(
&self,
display: &str,
translated: TranslatedAction,
) -> Result<(), ControlError> {
match translated {
TranslatedAction::Sleep { ms } => {
sleep(Duration::from_millis(ms)).await;
Ok(())
}
TranslatedAction::LegacyArgv { argv, detached } => {
if detached {
spawn_detached(&argv).await.map_err(ControlError::internal)
} else {
crate::process::run_output(&argv)
.await
.map_err(ControlError::internal)
.and_then(|output| {
if output.status.success() {
Ok(())
} else {
Err(ControlError::internal(String::from_utf8_lossy(
&output.stderr,
)))
}
})
}
}
TranslatedAction::FocusTitle { title } => self.focus_title(display, &title).await,
TranslatedAction::Cua { tool, payload } => {
let payload = self.with_window_target(display, tool, payload).await?;
self.client.call(display, tool, &payload, &[]).await?;
Ok(())
}
}
}
async fn with_window_target(
&self,
display: &str,
tool: &str,
mut payload: Value,
) -> Result<Value, ControlError> {
if tool != "mouse_button_down" && tool != "mouse_button_up" {
return Ok(payload);
}
if payload.get("pid").is_some() && payload.get("window_id").is_some() {
return Ok(payload);
}
let window = self
.windows(display)
.await?
.into_iter()
.max_by_key(|window| window.z)
.ok_or(ControlError::TargetNotFound)?;
if let Some(object) = payload.as_object_mut() {
object.insert("pid".into(), json!(window.pid));
object.insert("window_id".into(), json!(window.id));
}
Ok(payload)
}
async fn focus_title(&self, display: &str, title: &str) -> Result<(), ControlError> {
let needle = title.to_ascii_lowercase();
let windows = self.windows(display).await?;
let window = windows
.into_iter()
.find(|window| window.title.to_ascii_lowercase().contains(&needle))
.ok_or(ControlError::TargetNotFound)?;
self.client
.call(
display,
"bring_to_front",
&json!({ "pid": window.pid, "window_id": window.id }),
&[],
)
.await?;
Ok(())
}
}
#[derive(Debug, Clone)]
pub(crate) struct ListedWindow {
pub(crate) id: u64,
pub(crate) pid: u64,
pub(crate) title: String,
pub(crate) app_name: String,
pub(crate) x: u32,
pub(crate) y: u32,
pub(crate) w: u32,
pub(crate) h: u32,
pub(crate) z: i64,
}
fn map_browser_unavailable(result: Result<CdpPage, ControlError>) -> Result<CdpPage, ControlError> {
match result {
Err(ControlError::BrowserUnavailable) => Ok(CdpPage {
ok: false,
error: Some("cdp unavailable".into()),
..CdpPage::default()
}),
other => other,
}
}
fn cursor_from_value(value: &Value) -> Option<CursorPosition> {
let mut nodes = Vec::new();
client::walk(value, &mut nodes);
for node in nodes {
if let (Some(x), Some(y)) = (
node.get("x").and_then(Value::as_i64),
node.get("y").and_then(Value::as_i64),
) {
return Some(CursorPosition {
x: x as i32,
y: y as i32,
});
}
}
None
}
fn parse_listed_windows(value: &Value) -> Vec<ListedWindow> {
first_array_of_objects(value, "window_id")
.into_iter()
.filter_map(listed_window)
.filter(|window| !ignored_window(window))
.collect()
}
fn listed_window(value: &Value) -> Option<ListedWindow> {
let bounds = value.get("bounds");
let x = number(value, "x").or_else(|| bounds.and_then(|bounds| number(bounds, "x")))?;
let y = number(value, "y").or_else(|| bounds.and_then(|bounds| number(bounds, "y")))?;
let w = number(value, "width")
.or_else(|| number(value, "w"))
.or_else(|| bounds.and_then(|bounds| number(bounds, "width")))?;
let h = number(value, "height")
.or_else(|| number(value, "h"))
.or_else(|| bounds.and_then(|bounds| number(bounds, "height")))?;
if w < 32 || h < 16 {
return None;
}
Some(ListedWindow {
id: number(value, "window_id")?,
pid: number(value, "pid").unwrap_or(0),
title: value
.get("title")
.and_then(Value::as_str)
.unwrap_or("")
.to_string(),
app_name: value
.get("app_name")
.and_then(Value::as_str)
.unwrap_or("")
.to_string(),
x: x as u32,
y: y as u32,
w: w as u32,
h: h as u32,
z: number(value, "z_index").unwrap_or(0) as i64,
})
}
fn ignored_window(window: &ListedWindow) -> bool {
let title = window.title.to_ascii_lowercase();
title.is_empty() || title == "desktop" || title == "xfce4-panel" || title.contains("xfdesktop")
}
fn number(value: &Value, key: &str) -> Option<u64> {
value.get(key).and_then(Value::as_u64).or_else(|| {
value
.get(key)
.and_then(Value::as_f64)
.filter(|n| *n >= 0.0)
.map(|n| n as u64)
})
}
fn png_dimensions(bytes: &[u8]) -> (u32, u32) {
image::load_from_memory(bytes)
.map(|image| (image.width(), image.height()))
.unwrap_or((1280, 800))
}
fn observe_png_path(display: &str) -> PathBuf {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or(0);
PathBuf::from(format!(
"/tmp/lazyboy/cua-obs-{}-{nanos}.png",
display.trim_start_matches(':')
))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn window_list_skips_panel_and_numbers_from_one() {
let raw = json!([
{
"window_id": 1,
"pid": 8,
"title": "xfce4-panel",
"x": 0,
"y": 759,
"width": 1280,
"height": 41,
"z_index": 3
},
{
"window_id": 9,
"pid": 20,
"title": "終端機",
"app_name": "xfce4-terminal",
"bounds": { "x": 53, "y": 55, "width": 753, "height": 699 },
"z_index": 1
}
]);
let windows = parse_listed_windows(&raw);
assert_eq!(windows.len(), 1);
assert_eq!(windows[0].title, "終端機");
assert_eq!(windows[0].app_name, "xfce4-terminal");
assert_eq!(windows[0].w, 753);
}
}

283
crates/control/src/cua/record.rs Normal file
View File

@ -0,0 +1,283 @@
use std::fs;
use std::path::{Path, PathBuf};
use std::time::UNIX_EPOCH;
use chrono::DateTime;
use serde_json::{Value, json};
pub fn events_from_dir(dir: &Path) -> Vec<Value> {
let mut turns = match fs::read_dir(dir) {
Ok(entries) => entries
.filter_map(Result::ok)
.map(|entry| entry.path())
.filter(|path| path.join("action.json").is_file())
.collect::<Vec<PathBuf>>(),
Err(_) => return Vec::new(),
};
turns.sort();
turns
.into_iter()
.filter_map(|path| event_from_turn(&path))
.collect()
}
fn event_from_turn(dir: &Path) -> Option<Value> {
let raw = fs::read_to_string(dir.join("action.json")).ok()?;
let value: Value = serde_json::from_str(&raw).ok()?;
let tool = tool_name(&value);
if tool.is_empty() {
return None;
}
let args = arguments(&value);
let at = timestamp_ms(&value).unwrap_or_else(|| file_time_ms(&dir.join("action.json")));
let el = element_from_args(args);
let mut event = match tool.as_str() {
"click" | "right_click" | "double_click" | "browser_click" => {
json!({ "t": "click", "el": el, "at": at })
}
"type_text" | "set_value" | "browser_type" => {
let name = element_label(&el);
let role = el.get("role").and_then(Value::as_str).unwrap_or("");
let raw_text = args
.get("text")
.or_else(|| args.get("value"))
.and_then(Value::as_str)
.unwrap_or("");
let value = if looks_secret(&name) || looks_secret(role) {
"[已遮罩]"
} else {
raw_text
};
json!({ "t": "input", "el": el, "value": value, "at": at })
}
"press_key" | "hotkey" => {
let key = args
.get("key")
.and_then(Value::as_str)
.map(str::to_string)
.or_else(|| {
args.get("keys").and_then(Value::as_array).map(|keys| {
keys.iter()
.filter_map(Value::as_str)
.collect::<Vec<_>>()
.join("+")
})
})
.unwrap_or_default();
json!({ "t": "key", "key": key, "el": el, "at": at })
}
"scroll" => json!({
"t": "scroll",
"y": args.get("amount").and_then(Value::as_i64).unwrap_or(0),
"at": at
}),
"browser_navigate" => json!({
"t": "navigate",
"url": args.get("url").and_then(Value::as_str).unwrap_or(""),
"at": at
}),
_ => return None,
};
if let Some(title) = window_title(dir) {
event["window"] = json!(title);
}
Some(event)
}
fn tool_name(value: &Value) -> String {
value
.get("tool")
.or_else(|| value.get("name"))
.or_else(|| value.get("action"))
.and_then(Value::as_str)
.unwrap_or("")
.to_ascii_lowercase()
}
fn arguments(value: &Value) -> &Value {
value
.get("arguments")
.or_else(|| value.get("input"))
.or_else(|| value.get("args"))
.unwrap_or(value)
}
fn element_from_args(args: &Value) -> Value {
let name = pick(
args,
&[
"name",
"label",
"ref",
"selector",
"element_token",
"element_index",
],
);
let role = pick(args, &["role"]);
json!({
"name": name,
"label": name,
"role": role,
"text": name,
})
}
fn element_label(el: &Value) -> String {
el.get("name")
.or_else(|| el.get("label"))
.or_else(|| el.get("text"))
.and_then(Value::as_str)
.unwrap_or("")
.to_string()
}
fn pick(value: &Value, keys: &[&str]) -> String {
for key in keys {
if let Some(text) = value.get(*key).and_then(Value::as_str).map(str::trim)
&& !text.is_empty()
{
return text.to_string();
}
if let Some(number) = value.get(*key).and_then(Value::as_i64) {
return number.to_string();
}
}
String::new()
}
fn timestamp_ms(value: &Value) -> Option<i64> {
if let Some(ms) = value.get("at").and_then(Value::as_i64) {
return Some(ms);
}
let stamp = value
.get("timestamp")
.or_else(|| value.get("ts"))
.or_else(|| value.get("time"))
.and_then(Value::as_str)?;
DateTime::parse_from_rfc3339(stamp)
.ok()
.map(|time| time.timestamp_millis())
}
fn file_time_ms(path: &Path) -> i64 {
fs::metadata(path)
.and_then(|meta| meta.modified())
.ok()
.and_then(|time| time.duration_since(UNIX_EPOCH).ok())
.map(|duration| duration.as_millis() as i64)
.unwrap_or(0)
}
fn window_title(dir: &Path) -> Option<String> {
for name in ["after_state.json", "app_state.json", "before_state.json"] {
let Ok(raw) = fs::read_to_string(dir.join(name)) else {
continue;
};
let Ok(value) = serde_json::from_str::<Value>(&raw) else {
continue;
};
if let Some(title) = value
.get("title")
.or_else(|| value.get("window_title"))
.or_else(|| value.pointer("/window/title"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|title| !title.is_empty())
{
return Some(title.to_string());
}
}
None
}
pub fn looks_secret(text: &str) -> bool {
let lower = text.to_lowercase();
[
"pass",
"pwd",
"密碼",
"token",
"otp",
"驗證碼",
"secret",
"cvv",
]
.iter()
.any(|needle| lower.contains(needle))
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::SystemTime;
fn write_turn(root: &Path, name: &str, action: &Value, after: Option<&Value>) {
let dir = root.join(name);
fs::create_dir_all(&dir).unwrap();
fs::write(dir.join("action.json"), action.to_string()).unwrap();
if let Some(state) = after {
fs::write(dir.join("after_state.json"), state.to_string()).unwrap();
}
}
#[test]
fn trajectory_turns_become_skill_events() {
let root = std::env::temp_dir().join(format!(
"lazyboy-traj-{}-{}",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
fs::create_dir_all(&root).unwrap();
write_turn(
&root,
"turn-00001",
&json!({
"tool": "browser_click",
"timestamp": "2026-09-07T00:00:01Z",
"arguments": { "ref": "p1:1", "role": "button", "name": "Smoke Click" }
}),
Some(&json!({ "title": "LazyBoy Cua Smoke" })),
);
write_turn(
&root,
"turn-00002",
&json!({
"tool": "type_text",
"timestamp": "2026-09-07T00:00:02Z",
"arguments": { "text": "hunter2", "name": "Password", "role": "textbox" }
}),
None,
);
write_turn(
&root,
"turn-00003",
&json!({
"tool": "browser_navigate",
"timestamp": "2026-09-07T00:00:03Z",
"arguments": { "url": "https://example.com" }
}),
None,
);
let events = events_from_dir(&root);
let _ = fs::remove_dir_all(&root);
assert_eq!(events.len(), 3);
assert_eq!(events[0]["t"], "click");
assert_eq!(events[0]["el"]["name"], "Smoke Click");
assert_eq!(events[0]["window"], "LazyBoy Cua Smoke");
assert_eq!(events[1]["t"], "input");
assert_eq!(events[1]["value"], "[已遮罩]");
assert_eq!(events[2]["t"], "navigate");
assert_eq!(events[2]["url"], "https://example.com");
}
#[test]
fn secret_labels_are_detected() {
assert!(looks_secret("Password"));
assert!(looks_secret("確認密碼"));
assert!(!looks_secret("Search"));
}
}

219
crates/control/src/cua/translate.rs Normal file
View File

@ -0,0 +1,219 @@
use lazyboy_contracts::{ComputerAction, PointerButton, PointerType, ScrollDirection};
use serde_json::{Value, json};
use crate::controller::ControlError;
use crate::{launch_argv_on, open_argv_on};
#[derive(Debug, Clone, PartialEq)]
pub enum TranslatedAction {
Cua { tool: &'static str, payload: Value },
Sleep { ms: u64 },
LegacyArgv { argv: Vec<String>, detached: bool },
FocusTitle { title: String },
}
pub fn translate_action(
action: &ComputerAction,
display: &str,
profile: Option<&str>,
) -> Result<TranslatedAction, ControlError> {
match action {
ComputerAction::Wait { ms } => Ok(TranslatedAction::Sleep { ms: u64::from(*ms) }),
ComputerAction::Open { path } => Ok(TranslatedAction::LegacyArgv {
argv: open_argv_on(display, profile, path),
detached: true,
}),
ComputerAction::Launch { application, uri } => {
let argv = launch_argv_on(display, profile, application, uri.as_deref())
.ok_or(ControlError::Unsupported)?;
Ok(TranslatedAction::LegacyArgv {
argv,
detached: true,
})
}
ComputerAction::Ref { .. } => Err(ControlError::Unsupported),
ComputerAction::Focus { title } => Ok(TranslatedAction::FocusTitle {
title: title.clone(),
}),
ComputerAction::Pointer {
x,
y,
pointer_type,
button,
} => Ok(translate_pointer(*x, *y, pointer_type, *button)),
ComputerAction::Clipboard { text } => {
tracing::info!(backend = "cua", tool = "type_text", length = text.len());
Ok(TranslatedAction::Cua {
tool: "type_text",
payload: json!({
"text": text,
"scope": "desktop",
"target": { "kind": "desktop", "display_id": "primary" },
}),
})
}
ComputerAction::Key { key, modifiers } => Ok(translate_key(key, modifiers.as_deref())),
ComputerAction::Scroll { direction, amount } => Ok(TranslatedAction::Cua {
tool: "scroll",
payload: json!({
"direction": match direction {
ScrollDirection::Up => "up",
ScrollDirection::Down => "down",
},
"amount": amount.unwrap_or(12),
"by": "line",
"scope": "desktop",
"target": { "kind": "desktop", "display_id": "primary" },
}),
}),
}
}
fn translate_pointer(
x: u32,
y: u32,
pointer_type: &PointerType,
button: Option<PointerButton>,
) -> TranslatedAction {
let button = match button.unwrap_or(PointerButton::Left) {
PointerButton::Left => "left",
PointerButton::Middle => "middle",
PointerButton::Right => "right",
};
match pointer_type {
PointerType::Move => TranslatedAction::Cua {
tool: "move_cursor",
payload: json!({
"x": x,
"y": y,
"scope": "desktop",
"target": { "kind": "desktop", "display_id": "primary" },
}),
},
PointerType::Click => TranslatedAction::Cua {
tool: "click",
payload: json!({
"x": x,
"y": y,
"button": button,
"scope": "desktop",
"target": { "kind": "desktop", "display_id": "primary" },
}),
},
PointerType::Down => TranslatedAction::Cua {
tool: "mouse_button_down",
payload: json!({ "x": x, "y": y, "button": button }),
},
PointerType::Up => TranslatedAction::Cua {
tool: "mouse_button_up",
payload: json!({ "x": x, "y": y, "button": button }),
},
}
}
fn translate_key(key: &str, modifiers: Option<&[String]>) -> TranslatedAction {
let key = map_key(key);
match modifiers {
Some(items) if !items.is_empty() => {
let mut keys: Vec<String> = items.iter().map(|item| map_key(item)).collect();
keys.push(key);
TranslatedAction::Cua {
tool: "hotkey",
payload: json!({
"keys": keys,
"scope": "desktop",
"target": { "kind": "desktop", "display_id": "primary" },
}),
}
}
_ => TranslatedAction::Cua {
tool: "press_key",
payload: json!({
"key": key,
"scope": "desktop",
"target": { "kind": "desktop", "display_id": "primary" },
}),
},
}
}
fn map_key(key: &str) -> String {
match key.to_ascii_lowercase().as_str() {
"enter" | "return" => "return".into(),
"esc" | "escape" => "escape".into(),
"cmd" | "command" | "super" | "meta" | "win" => "ctrl".into(),
"control" | "ctl" => "ctrl".into(),
"option" => "alt".into(),
other => other.to_string(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use lazyboy_contracts::RefVerb;
#[test]
fn pixel_click_is_desktop_cua_click() {
let action = ComputerAction::Pointer {
x: 40,
y: 80,
pointer_type: PointerType::Click,
button: Some(PointerButton::Left),
};
let TranslatedAction::Cua { tool, payload } =
translate_action(&action, ":1", None).unwrap()
else {
panic!("expected cua click");
};
assert_eq!(tool, "click");
assert_eq!(payload["x"], 40);
assert_eq!(payload["y"], 80);
assert_eq!(payload["scope"], "desktop");
assert_eq!(payload["target"]["display_id"], "primary");
}
#[test]
fn typed_text_is_redacted_from_tool_name_only() {
let action = ComputerAction::Clipboard {
text: "secret-password".into(),
};
let TranslatedAction::Cua { tool, payload } =
translate_action(&action, ":1", None).unwrap()
else {
panic!("expected type_text");
};
assert_eq!(tool, "type_text");
assert_eq!(payload["text"], "secret-password");
assert_eq!(payload["scope"], "desktop");
}
#[test]
fn chord_uses_hotkey_and_maps_cmd_to_ctrl() {
let action = ComputerAction::Key {
key: "c".into(),
modifiers: Some(vec!["cmd".into()]),
};
let TranslatedAction::Cua { tool, payload } =
translate_action(&action, ":1", None).unwrap()
else {
panic!("expected hotkey");
};
assert_eq!(tool, "hotkey");
assert_eq!(payload["keys"], json!(["ctrl", "c"]));
}
#[test]
fn semantic_ref_is_not_translated_here() {
let action = ComputerAction::Ref {
verb: RefVerb::Click,
target: "#go".into(),
ref_kind: "dom".into(),
text: None,
};
assert!(matches!(
translate_action(&action, ":1", None),
Err(ControlError::Unsupported)
));
}
}

300
crates/control/src/legacy.rs Normal file
View File

@ -0,0 +1,300 @@
use async_trait::async_trait;
use lazyboy_contracts::{ComputerAction, ComputerObservation, RefVerb};
use tokio::time::{Duration, sleep};
use crate::controller::{
ComputerController, ComputerDriver, ControlContext, ControlError, ControllerHealth,
};
use crate::process::{
capture_stdout, run_output, run_stdout_text, run_stdout_text_timeout, spawn_detached,
};
use crate::{
ActionRequest, ActionResult, BrowserRequest, CdpPage, RecordingRequest, RecordingResult,
RecordingSession, a11y_command_on, action_pause_ms, cdp_command_on, cdp_record_command_on,
cdp_record_stop_command, devtools_port, launch_argv_on, observation_from_png,
observation_with_elements, open_argv_on, parse_a11y_page, parse_cdp_page, parse_pointer_state,
parse_ui_elements, pointer_state_command_on, screenshot_command_on, teach_recorder_output,
teach_trajectory_dir, window_list_command_on, xdotool_argv_on,
};
#[derive(Debug, Default, Clone, Copy)]
pub struct LegacyController;
#[async_trait]
impl ComputerController for LegacyController {
fn backend(&self) -> ComputerDriver {
ComputerDriver::Legacy
}
async fn health(&self, ctx: &ControlContext) -> Result<ControllerHealth, ControlError> {
Ok(ControllerHealth {
backend: ComputerDriver::Legacy.as_str().to_string(),
version: None,
healthy: true,
degraded: false,
details: vec![format!("display {}", ctx.display)],
})
}
async fn observe(&self, ctx: &ControlContext) -> Result<ComputerObservation, ControlError> {
observe_display(&ctx.display).await
}
async fn act(
&self,
request: &ActionRequest,
ctx: &ControlContext,
) -> Result<ActionResult, ControlError> {
let display = ctx.display.as_str();
let profile = ctx.profile_path.as_deref();
let mut completed = 0usize;
for action in &request.actions {
apply_action(display, profile, action).await?;
let pause = action_pause_ms(action);
if pause > 0 {
sleep(Duration::from_millis(pause)).await;
}
completed += 1;
}
if request.settle_ms > 0 {
sleep(Duration::from_millis(u64::from(request.settle_ms))).await;
}
let observation = if request.observe {
Some(observe_display(display).await?)
} else {
None
};
Ok(ActionResult {
completed,
observation,
})
}
async fn browser(
&self,
request: &BrowserRequest,
ctx: &ControlContext,
) -> Result<CdpPage, ControlError> {
legacy_browser(request, ctx).await
}
async fn start_recording(
&self,
request: &RecordingRequest,
ctx: &ControlContext,
) -> Result<RecordingSession, ControlError> {
start_cdp_recorder(&ctx.display, ctx.profile_path.as_deref(), &request.skill_id).await?;
Ok(RecordingSession {
skill_id: request.skill_id.clone(),
output_dir: teach_trajectory_dir(&request.skill_id),
})
}
async fn stop_recording(
&self,
request: &RecordingRequest,
_ctx: &ControlContext,
) -> Result<(), ControlError> {
stop_cdp_recorder(&request.skill_id).await
}
async fn collect_recording(
&self,
request: &RecordingRequest,
_ctx: &ControlContext,
) -> Result<RecordingResult, ControlError> {
Ok(RecordingResult {
events: collect_cdp_events(&request.skill_id).await,
})
}
}
async fn legacy_browser(
request: &BrowserRequest,
ctx: &ControlContext,
) -> Result<CdpPage, ControlError> {
let mut body = serde_json::json!({
"action": request.action,
"ensure": request.ensure,
"display": ctx.display,
"port": devtools_port(&ctx.display),
});
if let Some(profile) = &ctx.profile_path {
body["profile"] = serde_json::json!(profile);
}
if let Some(url) = &request.url {
body["url"] = serde_json::json!(url);
}
if let Some(text) = &request.text {
body["text"] = serde_json::json!(text);
}
if let Some(key) = &request.key {
body["key"] = serde_json::json!(key);
}
if let Some(ms) = request.ms {
body["ms"] = serde_json::json!(ms);
}
if let Some(selector) = &request.selector {
body["selector"] = serde_json::json!(selector);
}
if let Some(wait_ms) = request.wait_ms {
body["waitMs"] = serde_json::json!(wait_ms);
}
let timeout_ms = if request.action == "click" {
request.wait_ms.unwrap_or(45_000).min(120_000) + 25_000
} else {
20_000
};
let argv = cdp_command_on(&ctx.display, ctx.profile_path.as_deref(), &body);
let raw = run_stdout_text_timeout(&argv, timeout_ms)
.await
.map_err(ControlError::internal)?;
Ok(parse_cdp_page(&raw))
}
pub async fn observe_display(display: &str) -> Result<ComputerObservation, ControlError> {
let png = capture_stdout(&screenshot_command_on(display))
.await
.map_err(|_| ControlError::Internal("screenshot failed".into()))?;
let ((cursor, window), elements) =
tokio::join!(run_pointer_state(display), run_window_list(display));
Ok(observation_with_elements(
observation_from_png(png, 1280, 800, cursor, window),
elements,
))
}
async fn run_window_list(display: &str) -> Vec<lazyboy_contracts::UiElement> {
let Ok(raw) = run_stdout_text(&window_list_command_on(display)).await else {
return Vec::new();
};
parse_ui_elements(&raw)
}
async fn run_pointer_state(
display: &str,
) -> (
Option<lazyboy_contracts::CursorPosition>,
Option<lazyboy_contracts::ActiveWindow>,
) {
let Ok(raw) = run_stdout_text(&pointer_state_command_on(display)).await else {
return (None, None);
};
parse_pointer_state(&raw)
}
pub async fn apply_action(
display: &str,
profile: Option<&str>,
action: &ComputerAction,
) -> Result<(), ControlError> {
match action {
ComputerAction::Wait { ms } => {
sleep(Duration::from_millis(u64::from(*ms))).await;
Ok(())
}
ComputerAction::Open { path } => spawn_detached(&open_argv_on(display, profile, path))
.await
.map_err(ControlError::internal),
ComputerAction::Focus { .. } => run_xdotool(display, action).await,
ComputerAction::Launch { application, uri } => {
let argv = launch_argv_on(display, profile, application, uri.as_deref())
.ok_or(ControlError::Unsupported)?;
spawn_detached(&argv).await.map_err(ControlError::internal)
}
ComputerAction::Ref {
verb,
target,
ref_kind,
text,
} => apply_ref(display, profile, *verb, target, ref_kind, text.as_deref()).await,
other => run_xdotool(display, other).await,
}
}
async fn run_xdotool(display: &str, action: &ComputerAction) -> Result<(), ControlError> {
let argv = xdotool_argv_on(display, action).ok_or(ControlError::Unsupported)?;
let output = run_output(&argv).await.map_err(ControlError::internal)?;
if output.status.success() {
Ok(())
} else {
Err(ControlError::internal(String::from_utf8_lossy(
&output.stderr,
)))
}
}
async fn apply_ref(
display: &str,
profile: Option<&str>,
verb: RefVerb,
target: &str,
kind: &str,
text: Option<&str>,
) -> Result<(), ControlError> {
let action = match verb {
RefVerb::Click => "click",
RefVerb::SetValue => "type",
RefVerb::Focus => "focus",
};
let mut request = serde_json::json!({
"action": action,
"selector": target,
"display": display,
"ensure": false,
});
if let Some(text) = text {
request["text"] = serde_json::json!(text);
}
let argv = if kind == "dom" {
cdp_command_on(display, profile, &request)
} else {
a11y_command_on(display, &request)
};
let raw = run_stdout_text(&argv)
.await
.map_err(ControlError::internal)?;
let ok = if kind == "dom" {
parse_cdp_page(&raw).ok
} else {
parse_a11y_page(&raw).ok
};
if ok {
Ok(())
} else {
Err(ControlError::internal(raw))
}
}
pub(crate) async fn start_cdp_recorder(
display: &str,
profile: Option<&str>,
skill_id: &str,
) -> Result<(), ControlError> {
if skill_id.trim().is_empty() {
return Err(ControlError::InvalidAction(
"recording needs a skill id".into(),
));
}
let argv = cdp_record_command_on(display, profile, skill_id);
run_output(&argv).await.map_err(ControlError::internal)?;
Ok(())
}
pub(crate) async fn stop_cdp_recorder(skill_id: &str) -> Result<(), ControlError> {
let argv = cdp_record_stop_command(skill_id);
let _ = run_output(&argv).await;
Ok(())
}
pub(crate) async fn collect_cdp_events(skill_id: &str) -> Vec<serde_json::Value> {
let path = teach_recorder_output(skill_id);
let Ok(text) = tokio::fs::read_to_string(&path).await else {
return Vec::new();
};
let _ = tokio::fs::remove_file(&path).await;
text.lines()
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
.filter(|event| event.get("t").and_then(serde_json::Value::as_str) != Some("recorder"))
.collect()
}

7
crates/control/src/lib.rs
View File

@ -1,10 +1,14 @@
mod a11y;
mod actions;
mod cdp;
mod controller;
mod cua;
mod lease;
mod legacy;
mod observe;
mod overlay;
mod path;
mod process;
mod sandbox;
mod screen;
mod takeover;
@ -13,7 +17,10 @@ mod x11;
pub use a11y::*;
pub use actions::*;
pub use cdp::*;
pub use controller::*;
pub use cua::{CuaClient, CuaController, TranslatedAction, translate_action};
pub use lease::*;
pub use legacy::LegacyController;
pub use observe::*;
pub use overlay::*;
pub use path::*;

38
crates/control/src/observe.rs
View File

@ -1,5 +1,7 @@
use base64::Engine;
use chrono::Utc;
use lazyboy_contracts::{ActiveWindow, ComputerObservation, CursorPosition, UiElement};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
pub fn observation_from_png(
@ -22,6 +24,24 @@ pub fn observation_from_png(
}
}
pub fn observation_to_control_json(observation: &ComputerObservation) -> Value {
let mut body = json!({
"png_base64": base64::engine::general_purpose::STANDARD.encode(&observation.image),
});
if let Some(cursor) = &observation.cursor {
body["cursor"] = json!({ "x": cursor.x, "y": cursor.y });
}
if let Some(window) = &observation.active_window {
body["activeWindow"] = json!({ "id": window.id, "title": window.title });
}
if !observation.elements.is_empty()
&& let Ok(value) = serde_json::to_value(&observation.elements)
{
body["elements"] = value;
}
body
}
pub fn observation_with_elements(
mut observation: ComputerObservation,
elements: Vec<UiElement>,
@ -62,7 +82,11 @@ const SIGNATURE_H: u32 = 18;
pub fn frame_signature(image: &[u8]) -> Option<Vec<u8>> {
let dynamic = image::load_from_memory(image).ok()?;
let thumb = dynamic
.resize_exact(SIGNATURE_W, SIGNATURE_H, image::imageops::FilterType::Triangle)
.resize_exact(
SIGNATURE_W,
SIGNATURE_H,
image::imageops::FilterType::Triangle,
)
.to_luma8();
Some(thumb.into_raw())
}
@ -108,12 +132,20 @@ mod tests {
let base = frame_signature(&png(|_, _| Rgb([240, 240, 240]))).unwrap();
// A panel clock flipping digits touches a couple of pixels only.
let clock = frame_signature(&png(|x, y| {
if x < 6 && y < 6 { Rgb([0, 0, 0]) } else { Rgb([240, 240, 240]) }
if x < 6 && y < 6 {
Rgb([0, 0, 0])
} else {
Rgb([240, 240, 240])
}
}))
.unwrap();
// A dialog covering a quarter of the screen.
let dialog = frame_signature(&png(|x, y| {
if x < 160 && y < 90 { Rgb([20, 20, 20]) } else { Rgb([240, 240, 240]) }
if x < 160 && y < 90 {
Rgb([20, 20, 20])
} else {
Rgb([240, 240, 240])
}
}))
.unwrap();
assert!(signatures_similar(&base, &clock));

71
crates/control/src/process.rs Normal file
View File

@ -0,0 +1,71 @@
use std::process::Stdio;
use std::time::Duration;
use tokio::io::AsyncReadExt;
use tokio::process::Command;
pub async fn run_output(argv: &[String]) -> Result<std::process::Output, String> {
if argv.is_empty() {
return Err("empty command".into());
}
Command::new(&argv[0])
.args(&argv[1..])
.output()
.await
.map_err(|error| error.to_string())
}
pub async fn run_stdout_text(argv: &[String]) -> Result<String, String> {
let output = run_output(argv).await?;
Ok(if output.stdout.is_empty() {
String::from_utf8_lossy(&output.stderr).into_owned()
} else {
String::from_utf8_lossy(&output.stdout).into_owned()
})
}
pub async fn run_stdout_text_timeout(argv: &[String], timeout_ms: u64) -> Result<String, String> {
tokio::time::timeout(
Duration::from_millis(timeout_ms.max(100)),
run_stdout_text(argv),
)
.await
.map_err(|_| "timed out".to_string())?
}
pub async fn capture_stdout(argv: &[String]) -> Result<Vec<u8>, String> {
if argv.is_empty() {
return Err("empty command".into());
}
let mut child = Command::new(&argv[0])
.args(&argv[1..])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|error| error.to_string())?;
let mut stdout = Vec::new();
if let Some(mut pipe) = child.stdout.take() {
pipe.read_to_end(&mut stdout)
.await
.map_err(|error| error.to_string())?;
}
let status = child.wait().await.map_err(|error| error.to_string())?;
if !status.success() || stdout.is_empty() {
return Err("command failed".into());
}
Ok(stdout)
}
pub async fn spawn_detached(argv: &[String]) -> Result<(), String> {
if argv.is_empty() {
return Err("empty command".into());
}
Command::new(&argv[0])
.args(&argv[1..])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.map_err(|error| error.to_string())?;
Ok(())
}

119
crates/control/src/sandbox.rs
View File

@ -77,6 +77,53 @@ impl ActionRequest {
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct BrowserRequest {
pub action: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub key: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ms: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub wait_ms: Option<u64>,
#[serde(default)]
pub ensure: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub display: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub profile_path: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct RecordingRequest {
pub skill_id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub display: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub profile_path: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct RecordingSession {
pub skill_id: String,
pub output_dir: String,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct RecordingResult {
pub events: Vec<serde_json::Value>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct EnsureScreenRequest {
pub slot: u32,
@ -208,6 +255,46 @@ pub trait SandboxProvider: Send + Sync {
context: &AdapterContext,
) -> Result<ActionResult, SandboxError>;
async fn browser(
&self,
computer: &ComputerRef,
request: BrowserRequest,
context: &AdapterContext,
) -> Result<crate::CdpPage, SandboxError> {
let _ = (computer, request, context);
Err(SandboxError::message("browser is unavailable"))
}
async fn start_recording(
&self,
computer: &ComputerRef,
request: RecordingRequest,
context: &AdapterContext,
) -> Result<RecordingSession, SandboxError> {
let _ = (computer, request, context);
Err(SandboxError::message("recording is unavailable"))
}
async fn stop_recording(
&self,
computer: &ComputerRef,
request: RecordingRequest,
context: &AdapterContext,
) -> Result<(), SandboxError> {
let _ = (computer, request, context);
Err(SandboxError::message("recording is unavailable"))
}
async fn collect_recording(
&self,
computer: &ComputerRef,
request: RecordingRequest,
context: &AdapterContext,
) -> Result<RecordingResult, SandboxError> {
let _ = (computer, request, context);
Err(SandboxError::message("recording is unavailable"))
}
async fn connect_screen(
&self,
computer: &ComputerRef,
@ -249,3 +336,35 @@ pub trait SandboxProvider: Send + Sync {
context: &AdapterContext,
) -> Result<(), SandboxError>;
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn browser_request_reads_camel_case_wait_ms() {
let request: BrowserRequest = serde_json::from_value(json!({
"action": "click",
"waitMs": 12000,
"ensure": true,
"selector": "p1:1"
}))
.unwrap();
assert_eq!(request.action, "click");
assert_eq!(request.wait_ms, Some(12_000));
assert!(request.ensure);
assert_eq!(request.selector.as_deref(), Some("p1:1"));
}
#[test]
fn recording_request_reads_camel_case_skill_id() {
let request: RecordingRequest = serde_json::from_value(json!({
"skillId": "abc-1",
"display": ":2"
}))
.unwrap();
assert_eq!(request.skill_id, "abc-1");
assert_eq!(request.display.as_deref(), Some(":2"));
}
}

2
crates/controld/Cargo.toml
View File

@ -7,14 +7,12 @@ license.workspace = true
publish.workspace = true
[dependencies]
lazyboy-contracts.workspace = true
lazyboy-control.workspace = true
axum.workspace = true
tokio.workspace = true
serde_json.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
base64.workspace = true
[lints]
workspace = true

319
crates/controld/src/main.rs
View File

@ -1,24 +1,19 @@
use std::process::Stdio;
use std::time::Duration;
use std::sync::Arc;
use axum::extract::State;
use axum::http::{HeaderMap, StatusCode};
use axum::routing::{get, post};
use axum::{Json, Router};
use lazyboy_contracts::{ComputerAction, RefVerb};
use lazyboy_control::{
ActionRequest, PRIMARY_DISPLAY, a11y_command_on, action_pause_ms, cdp_command_on,
launch_argv_on, normalize_display, open_argv_on, parse_a11y_page, parse_cdp_page,
parse_pointer_state, parse_ui_elements, pointer_state_command_on, screenshot_command_on,
window_list_command_on, xdotool_argv_on,
ActionRequest, BrowserRequest, ComputerController, ComputerDriver, ControlContext,
ControlError, PRIMARY_DISPLAY, RecordingRequest, normalize_display,
observation_to_control_json,
};
use tokio::io::AsyncReadExt;
use tokio::process::Command;
use tokio::time::sleep;
#[derive(Clone)]
struct App {
token: String,
controller: Arc<dyn ComputerController>,
}
#[tokio::main]
@ -27,11 +22,20 @@ async fn main() {
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.init();
let token = std::env::var("LAZYBOY_CONTROL_TOKEN").unwrap_or_default();
let driver = ComputerDriver::from_env();
tracing::info!(backend = driver.as_str(), "computer controller");
let app = Router::new()
.route("/health", get(|| async { "ok" }))
.route("/observe", post(observe))
.route("/act", post(act))
.with_state(App { token });
.route("/browser", post(browser))
.route("/recording/start", post(recording_start))
.route("/recording/stop", post(recording_stop))
.route("/recording/collect", post(recording_collect))
.with_state(App {
token,
controller: driver.controller(),
});
let listener = tokio::net::TcpListener::bind("127.0.0.1:7070")
.await
.expect("bind control port");
@ -72,6 +76,15 @@ fn profile_of(headers: &HeaderMap, fallback: Option<&str>) -> Option<String> {
.map(str::to_string)
}
fn status_for(error: &ControlError) -> StatusCode {
if error.is_client_error() {
StatusCode::BAD_REQUEST
} else {
tracing::error!(error = %error, "control failed");
StatusCode::INTERNAL_SERVER_ERROR
}
}
async fn observe(
State(app): State<App>,
headers: HeaderMap,
@ -79,11 +92,11 @@ async fn observe(
if !authorized(&headers, &app.token) {
return Err(StatusCode::UNAUTHORIZED);
}
let display = display_of(&headers, None);
let png = run_capture(&display)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
Ok(Json(observation_json(&display, png).await))
let ctx = ControlContext::new(display_of(&headers, None), None);
match app.controller.observe(&ctx).await {
Ok(observation) => Ok(Json(observation_to_control_json(&observation))),
Err(error) => Err(status_for(&error)),
}
}
async fn act(
@ -94,205 +107,105 @@ async fn act(
if !authorized(&headers, &app.token) {
return Err(StatusCode::UNAUTHORIZED);
}
let display = display_of(&headers, request.display.as_deref());
let profile = profile_of(&headers, request.profile_path.as_deref());
let mut completed = 0usize;
for action in &request.actions {
apply_action(&display, profile.as_deref(), action)
.await
.map_err(|_| StatusCode::BAD_REQUEST)?;
let pause = action_pause_ms(action);
if pause > 0 {
sleep(Duration::from_millis(pause)).await;
}
completed += 1;
}
if request.settle_ms > 0 {
sleep(Duration::from_millis(request.settle_ms as u64)).await;
}
let mut body = serde_json::json!({ "completed": completed });
if request.observe {
let png = run_capture(&display)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
if let serde_json::Value::Object(map) = observation_json(&display, png).await {
body.as_object_mut().unwrap().extend(map);
}
}
Ok(Json(body))
}
async fn apply_action(
display: &str,
profile: Option<&str>,
action: &ComputerAction,
) -> Result<(), String> {
match action {
ComputerAction::Wait { ms } => {
sleep(Duration::from_millis(*ms as u64)).await;
Ok(())
}
ComputerAction::Open { path } => {
spawn_detached(&open_argv_on(display, profile, path)).await
}
ComputerAction::Focus { .. } => {
let argv =
xdotool_argv_on(display, action).ok_or_else(|| "unsupported action".to_string())?;
let output = Command::new(&argv[0])
.args(&argv[1..])
.output()
.await
.map_err(|error| error.to_string())?;
if output.status.success() {
Ok(())
} else {
Err(String::from_utf8_lossy(&output.stderr).into_owned())
}
}
ComputerAction::Launch { application, uri } => {
let argv = launch_argv_on(display, profile, application, uri.as_deref())
.ok_or_else(|| "unknown application".to_string())?;
spawn_detached(&argv).await
}
ComputerAction::Ref {
verb,
target,
ref_kind,
text,
} => apply_ref(display, profile, *verb, target, ref_kind, text.as_deref()).await,
other => {
let argv =
xdotool_argv_on(display, other).ok_or_else(|| "unsupported action".to_string())?;
let output = Command::new(&argv[0])
.args(&argv[1..])
.output()
.await
.map_err(|error| error.to_string())?;
if output.status.success() {
Ok(())
} else {
Err(String::from_utf8_lossy(&output.stderr).into_owned())
let ctx = ControlContext::new(
display_of(&headers, request.display.as_deref()),
profile_of(&headers, request.profile_path.as_deref()),
);
match app.controller.act(&request, &ctx).await {
Ok(result) => {
let mut body = serde_json::json!({ "completed": result.completed });
if let Some(observation) = result.observation
&& let serde_json::Value::Object(map) = observation_to_control_json(&observation)
{
body.as_object_mut().expect("object").extend(map);
}
Ok(Json(body))
}
Err(error) => Err(status_for(&error)),
}
}
async fn apply_ref(
display: &str,
profile: Option<&str>,
verb: RefVerb,
target: &str,
kind: &str,
text: Option<&str>,
) -> Result<(), String> {
let action = match verb {
RefVerb::Click => "click",
RefVerb::SetValue => "type",
RefVerb::Focus => "focus",
};
let mut request = serde_json::json!({
"action": action,
"selector": target,
"display": display,
"ensure": false,
});
if let Some(text) = text {
request["text"] = serde_json::json!(text);
async fn browser(
State(app): State<App>,
headers: HeaderMap,
Json(request): Json<BrowserRequest>,
) -> Result<Json<serde_json::Value>, StatusCode> {
if !authorized(&headers, &app.token) {
return Err(StatusCode::UNAUTHORIZED);
}
let argv = if kind == "dom" {
cdp_command_on(display, profile, &request)
} else {
a11y_command_on(display, &request)
};
let output = Command::new(&argv[0])
.args(&argv[1..])
.output()
let ctx = ControlContext::new(
display_of(&headers, request.display.as_deref()),
profile_of(&headers, request.profile_path.as_deref()),
);
match app.controller.browser(&request, &ctx).await {
Ok(page) => Ok(Json(
serde_json::to_value(&page).unwrap_or_else(|_| serde_json::json!({"ok": false})),
)),
Err(error) => Err(status_for(&error)),
}
}
fn recording_ctx(headers: &HeaderMap, request: &RecordingRequest) -> ControlContext {
ControlContext::new(
display_of(headers, request.display.as_deref()),
profile_of(headers, request.profile_path.as_deref()),
)
}
async fn recording_start(
State(app): State<App>,
headers: HeaderMap,
Json(request): Json<RecordingRequest>,
) -> Result<Json<serde_json::Value>, StatusCode> {
if !authorized(&headers, &app.token) {
return Err(StatusCode::UNAUTHORIZED);
}
match app
.controller
.start_recording(&request, &recording_ctx(&headers, &request))
.await
.map_err(|error| error.to_string())?;
let raw = if output.stdout.is_empty() {
String::from_utf8_lossy(&output.stderr).into_owned()
} else {
String::from_utf8_lossy(&output.stdout).into_owned()
};
let ok = if kind == "dom" {
parse_cdp_page(&raw).ok
} else {
parse_a11y_page(&raw).ok
};
if ok { Ok(()) } else { Err(raw) }
}
async fn spawn_detached(argv: &[String]) -> Result<(), String> {
Command::new(&argv[0])
.args(&argv[1..])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.map_err(|error| error.to_string())?;
Ok(())
}
async fn observation_json(display: &str, png: Vec<u8>) -> serde_json::Value {
use base64::Engine;
let mut body = serde_json::json!({
"png_base64": base64::engine::general_purpose::STANDARD.encode(png)
});
let ((cursor, window), elements) =
tokio::join!(run_pointer_state(display), run_window_list(display));
if let Some(cursor) = cursor {
body["cursor"] = serde_json::json!({ "x": cursor.x, "y": cursor.y });
{
Ok(session) => Ok(Json(
serde_json::to_value(&session).unwrap_or_else(|_| serde_json::json!({"ok": false})),
)),
Err(error) => Err(status_for(&error)),
}
if let Some(window) = window {
body["activeWindow"] = serde_json::json!({ "id": window.id, "title": window.title });
}
if !elements.is_empty() {
body["elements"] = serde_json::to_value(elements).unwrap_or(serde_json::json!([]));
}
body
}
async fn run_window_list(display: &str) -> Vec<lazyboy_contracts::UiElement> {
let argv = window_list_command_on(display);
let output = Command::new(&argv[0]).args(&argv[1..]).output().await.ok();
let Some(output) = output else {
return Vec::new();
};
parse_ui_elements(&String::from_utf8_lossy(&output.stdout))
async fn recording_stop(
State(app): State<App>,
headers: HeaderMap,
Json(request): Json<RecordingRequest>,
) -> Result<Json<serde_json::Value>, StatusCode> {
if !authorized(&headers, &app.token) {
return Err(StatusCode::UNAUTHORIZED);
}
match app
.controller
.stop_recording(&request, &recording_ctx(&headers, &request))
.await
{
Ok(()) => Ok(Json(serde_json::json!({ "ok": true }))),
Err(error) => Err(status_for(&error)),
}
}
async fn run_pointer_state(
display: &str,
) -> (
Option<lazyboy_contracts::CursorPosition>,
Option<lazyboy_contracts::ActiveWindow>,
) {
let argv = pointer_state_command_on(display);
let output = Command::new(&argv[0]).args(&argv[1..]).output().await.ok();
let Some(output) = output else {
return (None, None);
};
parse_pointer_state(&String::from_utf8_lossy(&output.stdout))
}
async fn run_capture(display: &str) -> Result<Vec<u8>, String> {
let argv = screenshot_command_on(display);
let mut child = Command::new(&argv[0])
.args(&argv[1..])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|error| error.to_string())?;
let mut stdout = Vec::new();
if let Some(mut pipe) = child.stdout.take() {
pipe.read_to_end(&mut stdout)
.await
.map_err(|error| error.to_string())?;
async fn recording_collect(
State(app): State<App>,
headers: HeaderMap,
Json(request): Json<RecordingRequest>,
) -> Result<Json<serde_json::Value>, StatusCode> {
if !authorized(&headers, &app.token) {
return Err(StatusCode::UNAUTHORIZED);
}
let status = child.wait().await.map_err(|error| error.to_string())?;
if !status.success() || stdout.is_empty() {
return Err("screenshot failed".into());
match app
.controller
.collect_recording(&request, &recording_ctx(&headers, &request))
.await
{
Ok(result) => Ok(Json(
serde_json::to_value(&result).unwrap_or_else(|_| serde_json::json!({"events": []})),
)),
Err(error) => Err(status_for(&error)),
}
Ok(stdout)
}

38
crates/harness/src/execution.rs
View File

@ -115,7 +115,10 @@ mod tests {
#[test]
fn a_prose_answer_is_not_a_stop() {
assert_eq!(stop_reason(ExecutionMode::Bounded(40), 3, "done", false, false), None);
assert_eq!(
stop_reason(ExecutionMode::Bounded(40), 3, "done", false, false),
None
);
}
#[test]
@ -132,7 +135,10 @@ mod tests {
stop_reason(mode, 3, "需要密码\n[NEEDS_INPUT]", true, false),
Some(StopReason::MidTaskText)
);
assert_eq!(stop_reason(mode, 3, "我先跳到下一步", true, true), Some(StopReason::MidTaskText));
assert_eq!(
stop_reason(mode, 3, "我先跳到下一步", true, true),
Some(StopReason::MidTaskText)
);
assert_eq!(stop_reason(mode, 3, "看起来好了", true, false), None);
}
@ -143,12 +149,21 @@ mod tests {
Some(StopReason::BudgetExhausted)
);
assert_eq!(
stop_reason(ExecutionMode::Bounded(40), 40, "需要密码\n[NEEDS_INPUT]", true, false),
stop_reason(
ExecutionMode::Bounded(40),
40,
"需要密码\n[NEEDS_INPUT]",
true,
false
),
Some(StopReason::BudgetExhausted)
);
// A goal run that left the loop on its own terms reported its outcome;
// the after-loop guard must not invent a second stop for it.
assert_eq!(stop_reason(ExecutionMode::Goal, 4000, "", true, false), None);
assert_eq!(
stop_reason(ExecutionMode::Goal, 4000, "", true, false),
None
);
assert_eq!(
stop_reason(ExecutionMode::Goal, 4000, "我先跳过这一步", true, true),
Some(StopReason::MidTaskText)
@ -178,8 +193,17 @@ mod tests {
fn progress_and_quoted_markers_do_not_complete_a_goal() {
assert_eq!(goal_outcome("[GOAL_COMPLETE]"), GoalOutcome::Continue);
assert_eq!(goal_outcome("\n[GOAL_BLOCKED]"), GoalOutcome::Continue);
assert_eq!(goal_outcome("Next I will use [GOAL_COMPLETE]."), GoalOutcome::Continue);
assert_eq!(goal_outcome("Verified output.\n[GOAL_COMPLETE]"), GoalOutcome::Complete);
assert_eq!(goal_outcome("Please supply the date.\n[GOAL_BLOCKED]"), GoalOutcome::NeedsInput);
assert_eq!(
goal_outcome("Next I will use [GOAL_COMPLETE]."),
GoalOutcome::Continue
);
assert_eq!(
goal_outcome("Verified output.\n[GOAL_COMPLETE]"),
GoalOutcome::Complete
);
assert_eq!(
goal_outcome("Please supply the date.\n[GOAL_BLOCKED]"),
GoalOutcome::NeedsInput
);
}
}

2
crates/harness/src/lib.rs
View File

@ -1,6 +1,6 @@
mod resolve;
pub mod execution;
pub mod policy;
mod resolve;
mod voice;
pub use resolve::*;

126
crates/harness/src/voice.rs
View File

@ -116,21 +116,36 @@ pub enum VoiceEvent {
AudioPcm(Vec<u8>),
SpeechStarted,
SpeechStopped,
InputTranscript { text: String, final_: bool },
OutputTranscript { text: String, final_: bool },
InputTranscript {
text: String,
final_: bool,
},
OutputTranscript {
text: String,
final_: bool,
},
FunctionCall {
call_id: String,
name: String,
arguments: String,
},
FunctionCallOutput { call_id: String, output: String },
SpeakNow { text: String },
InjectContext { text: String },
FunctionCallOutput {
call_id: String,
output: String,
},
SpeakNow {
text: String,
},
InjectContext {
text: String,
},
ResponseCreate,
ResponseStarted,
ResponseFinished,
CancelResponse,
Error { message: String },
Error {
message: String,
},
}
#[derive(Debug, Clone)]
@ -190,7 +205,9 @@ fn native_roots() -> Result<rustls::RootCertStore, VoiceError> {
let certs = rustls_native_certs::load_native_certs();
roots.add_parsable_certificates(certs.certs);
if roots.is_empty() {
return Err(VoiceError::Message("No trusted TLS certificates available".into()));
return Err(VoiceError::Message(
"No trusted TLS certificates available".into(),
));
}
Ok(roots)
}
@ -222,18 +239,21 @@ impl VoiceRealtime for HostedVoice {
);
// Choose explicitly: the dependency graph enables both ring and aws-lc-rs.
// Rustls's automatic provider selection panics in that configuration.
let tls = rustls::ClientConfig::builder_with_provider(
Arc::new(rustls::crypto::ring::default_provider()),
)
let tls = rustls::ClientConfig::builder_with_provider(Arc::new(
rustls::crypto::ring::default_provider(),
))
.with_safe_default_protocol_versions()
.map_err(|error| VoiceError::Message(error.to_string()))?
.with_root_certificates(native_roots()?)
.with_no_client_auth();
let (stream, _) = tokio_tungstenite::connect_async_tls_with_config(
http_request, None, false, Some(tokio_tungstenite::Connector::Rustls(Arc::new(tls))),
http_request,
None,
false,
Some(tokio_tungstenite::Connector::Rustls(Arc::new(tls))),
)
.await
.map_err(|error| VoiceError::Message(error.to_string()))?;
.await
.map_err(|error| VoiceError::Message(error.to_string()))?;
let (write, read) = stream.split();
let socket = HostedSocket {
provider: self.provider,
@ -241,7 +261,9 @@ impl VoiceRealtime for HostedVoice {
read,
};
socket
.send_raw(Message::Text(session_update_json(self.provider, &request).into()))
.send_raw(Message::Text(
session_update_json(self.provider, &request).into(),
))
.await?;
for (role, text) in &request.history {
if text.trim().is_empty() {
@ -266,10 +288,13 @@ impl VoiceRealtime for HostedVoice {
}
}
type HostedWrite =
futures_util::stream::SplitSink<tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>, Message>;
type HostedRead =
futures_util::stream::SplitStream<tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>>;
type HostedWrite = futures_util::stream::SplitSink<
tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>,
Message,
>;
type HostedRead = futures_util::stream::SplitStream<
tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>,
>;
struct HostedSocket {
provider: VoiceProvider,
@ -447,10 +472,7 @@ pub fn parse_provider_event(text: &str) -> Option<VoiceEvent> {
.and_then(Value::as_str)
.or_else(|| event.get("text").and_then(Value::as_str))?
.to_string();
Some(VoiceEvent::InputTranscript {
text,
final_: true,
})
Some(VoiceEvent::InputTranscript { text, final_: true })
}
"response.output_audio.delta" | "response.audio.delta" => {
let encoded = event.get("delta").and_then(Value::as_str)?;
@ -473,10 +495,7 @@ pub fn parse_provider_event(text: &str) -> Option<VoiceEvent> {
.or_else(|| event.get("text").and_then(Value::as_str))
.unwrap_or("")
.to_string();
Some(VoiceEvent::OutputTranscript {
text,
final_: true,
})
Some(VoiceEvent::OutputTranscript { text, final_: true })
}
"response.function_call_arguments.done" => {
let call_id = event.get("call_id")?.as_str()?.to_string();
@ -574,10 +593,7 @@ impl VoiceSocket for ScriptedSocket {
}
VoiceEvent::SpeakNow { text } | VoiceEvent::InjectContext { text } => {
let tx = self.incoming.lock().await;
let _ = tx.send(VoiceEvent::OutputTranscript {
text,
final_: true,
});
let _ = tx.send(VoiceEvent::OutputTranscript { text, final_: true });
let _ = tx.send(VoiceEvent::AudioPcm(Self::beep()));
}
VoiceEvent::ResponseCreate => {}
@ -597,31 +613,49 @@ mod tests {
#[test]
fn voice_tls_config_uses_an_explicit_provider() {
let config = rustls::ClientConfig::builder_with_provider(
Arc::new(rustls::crypto::ring::default_provider()),
).with_safe_default_protocol_versions().unwrap()
.with_root_certificates(native_roots().unwrap()).with_no_client_auth();
let config = rustls::ClientConfig::builder_with_provider(Arc::new(
rustls::crypto::ring::default_provider(),
))
.with_safe_default_protocol_versions()
.unwrap()
.with_root_certificates(native_roots().unwrap())
.with_no_client_auth();
assert!(!config.crypto_provider().cipher_suites.is_empty());
}
#[test]
fn openai_audio_uses_json_and_nested_session_settings() {
let request = VoiceConnectRequest {
api_key: String::new(), model_id: "gpt-realtime".into(),
voice_id: "marin".into(), instructions: "test".into(),
tools: vec![], history: vec![],
api_key: String::new(),
model_id: "gpt-realtime".into(),
voice_id: "marin".into(),
instructions: "test".into(),
tools: vec![],
history: vec![],
};
let session: Value = serde_json::from_str(&session_update_json(VoiceProvider::Openai, &request)).unwrap();
let session: Value =
serde_json::from_str(&session_update_json(VoiceProvider::Openai, &request)).unwrap();
assert!(session["session"].get("voice").is_none());
assert!(session["session"].get("turn_detection").is_none());
assert_eq!(session.pointer("/session/audio/input/format/rate"), Some(&json!(24000)));
assert_eq!(session.pointer("/session/audio/input/turn_detection/type"), Some(&json!("server_vad")));
assert_eq!(
session.pointer("/session/audio/input/format/rate"),
Some(&json!(24000))
);
assert_eq!(
session.pointer("/session/audio/input/turn_detection/type"),
Some(&json!("server_vad"))
);
let event = VoiceEvent::AudioPcm(vec![0, 1, 2, 3]);
let Some(Message::Text(text)) = encode_provider_event(VoiceProvider::Openai, &event) else { panic!("expected JSON audio") };
let Some(Message::Text(text)) = encode_provider_event(VoiceProvider::Openai, &event) else {
panic!("expected JSON audio")
};
let encoded: Value = serde_json::from_str(&text).unwrap();
assert_eq!(encoded["type"], "input_audio_buffer.append");
assert_eq!(encoded["audio"], "AAECAw==");
assert!(matches!(encode_provider_event(VoiceProvider::Xai, &event), Some(Message::Binary(_))));
assert!(matches!(
encode_provider_event(VoiceProvider::Xai, &event),
Some(Message::Binary(_))
));
}
#[test]
@ -656,9 +690,7 @@ mod tests {
#[test]
fn parses_xai_and_openai_event_aliases() {
let started = parse_provider_event(
r#"{"type":"input_audio_buffer.speech_started"}"#,
);
let started = parse_provider_event(r#"{"type":"input_audio_buffer.speech_started"}"#);
assert!(matches!(started, Some(VoiceEvent::SpeechStarted)));
let transcript = parse_provider_event(
@ -672,9 +704,7 @@ mod tests {
other => panic!("{other:?}"),
}
let old_audio = parse_provider_event(
r#"{"type":"response.audio.delta","delta":"AQID"}"#,
);
let old_audio = parse_provider_event(r#"{"type":"response.audio.delta","delta":"AQID"}"#);
assert!(matches!(old_audio, Some(VoiceEvent::AudioPcm(_))));
let tool = parse_provider_event(

92
crates/sandbox/src/docker.rs
View File

@ -3,8 +3,9 @@ use base64::Engine;
use lazyboy_contracts::ComputerCapabilities;
use lazyboy_contracts::{ActiveWindow, ComputerObservation, CursorPosition, SandboxKind};
use lazyboy_control::{
ActionRequest, ActionResult, AdapterContext, CommandRequest, CommandResult, ComputerRef,
EnsureScreenRequest, EnsureScreenResult, FileEntry, ProvisionRequest, SandboxError,
ActionRequest, ActionResult, AdapterContext, BrowserRequest, CdpPage, CommandRequest,
CommandResult, ComputerRef, EnsureScreenRequest, EnsureScreenResult, FileEntry,
ProvisionRequest, RecordingRequest, RecordingResult, RecordingSession, SandboxError,
SandboxProvider, ScreenSession, observation_from_png, observation_with_elements,
parse_ui_elements,
};
@ -251,6 +252,93 @@ impl SandboxProvider for DockerSandbox {
})
}
async fn browser(
&self,
computer: &ComputerRef,
request: BrowserRequest,
context: &AdapterContext,
) -> Result<CdpPage, SandboxError> {
let response = self
.client
.post(self.url(&format!("/computers/{}/browser", computer.id)))
.headers(self.headers(context))
.json(&request)
.send()
.await
.map_err(|error| SandboxError::message(error.to_string()))?;
let body: Value = response
.json()
.await
.map_err(|error| SandboxError::message(error.to_string()))?;
serde_json::from_value(body).map_err(|error| SandboxError::message(error.to_string()))
}
async fn start_recording(
&self,
computer: &ComputerRef,
request: RecordingRequest,
context: &AdapterContext,
) -> Result<RecordingSession, SandboxError> {
let response = self
.client
.post(self.url(&format!("/computers/{}/recording/start", computer.id)))
.headers(self.headers(context))
.json(&request)
.send()
.await
.map_err(|error| SandboxError::message(error.to_string()))?;
let body: Value = response
.json()
.await
.map_err(|error| SandboxError::message(error.to_string()))?;
serde_json::from_value(body).map_err(|error| SandboxError::message(error.to_string()))
}
async fn stop_recording(
&self,
computer: &ComputerRef,
request: RecordingRequest,
context: &AdapterContext,
) -> Result<(), SandboxError> {
let response = self
.client
.post(self.url(&format!("/computers/{}/recording/stop", computer.id)))
.headers(self.headers(context))
.json(&request)
.send()
.await
.map_err(|error| SandboxError::message(error.to_string()))?;
if response.status().is_success() {
Ok(())
} else {
Err(SandboxError::message(format!(
"stop recording failed: {}",
response.status()
)))
}
}
async fn collect_recording(
&self,
computer: &ComputerRef,
request: RecordingRequest,
context: &AdapterContext,
) -> Result<RecordingResult, SandboxError> {
let response = self
.client
.post(self.url(&format!("/computers/{}/recording/collect", computer.id)))
.headers(self.headers(context))
.json(&request)
.send()
.await
.map_err(|error| SandboxError::message(error.to_string()))?;
let body: Value = response
.json()
.await
.map_err(|error| SandboxError::message(error.to_string()))?;
serde_json::from_value(body).map_err(|error| SandboxError::message(error.to_string()))
}
async fn connect_screen(
&self,
computer: &ComputerRef,

50
crates/sandbox/src/fake.rs
View File

@ -4,9 +4,9 @@ use std::sync::Mutex;
use async_trait::async_trait;
use lazyboy_contracts::{ComputerObservation, SandboxKind};
use lazyboy_control::{
ActionRequest, ActionResult, AdapterContext, CommandRequest, CommandResult, ComputerRef,
FileEntry, ProvisionRequest, SandboxError, SandboxProvider, ScreenSession,
observation_from_png,
ActionRequest, ActionResult, AdapterContext, BrowserRequest, CdpPage, CommandRequest,
CommandResult, ComputerRef, FileEntry, ProvisionRequest, RecordingRequest, RecordingResult,
RecordingSession, SandboxError, SandboxProvider, ScreenSession, observation_from_png,
};
const EMPTY_PNG: &[u8] = &[
@ -103,6 +103,50 @@ impl SandboxProvider for FakeSandbox {
})
}
async fn browser(
&self,
_computer: &ComputerRef,
_request: BrowserRequest,
_context: &AdapterContext,
) -> Result<CdpPage, SandboxError> {
Ok(CdpPage {
ok: true,
url: "about:blank".into(),
title: "fake".into(),
..CdpPage::default()
})
}
async fn start_recording(
&self,
_computer: &ComputerRef,
request: RecordingRequest,
_context: &AdapterContext,
) -> Result<RecordingSession, SandboxError> {
Ok(RecordingSession {
skill_id: request.skill_id,
output_dir: "/tmp/lazyboy/teach-fake".into(),
})
}
async fn stop_recording(
&self,
_computer: &ComputerRef,
_request: RecordingRequest,
_context: &AdapterContext,
) -> Result<(), SandboxError> {
Ok(())
}
async fn collect_recording(
&self,
_computer: &ComputerRef,
_request: RecordingRequest,
_context: &AdapterContext,
) -> Result<RecordingResult, SandboxError> {
Ok(RecordingResult { events: Vec::new() })
}
async fn connect_screen(
&self,
computer: &ComputerRef,

128
crates/supervisor/src/docker.rs
View File

@ -13,9 +13,10 @@ use bollard::models::{EndpointSettings, HostConfig, HostConfigLogConfig, PortBin
use bollard::network::{ConnectNetworkOptions, CreateNetworkOptions};
use futures_util::StreamExt;
use lazyboy_control::{
ActionRequest, CommandRequest, CommandResult, EnsureScreenRequest, EnsureScreenResult, HOME,
ScreenTarget, TEAM_SCREEN_LIMIT, normalize_display, normalize_workspace_path,
pointer_state_command_on, screen_layout, screenshot_command_on, window_list_command_on,
ActionRequest, BrowserRequest, CommandRequest, CommandResult, EnsureScreenRequest,
EnsureScreenResult, HOME, RecordingRequest, ScreenTarget, TEAM_SCREEN_LIMIT, normalize_display,
normalize_workspace_path, pointer_state_command_on, screen_layout, screenshot_command_on,
window_list_command_on,
};
use tokio::time::{Duration, sleep};
@ -187,7 +188,15 @@ impl DockerHost {
),
format!(
"LAZYBOY_COMPUTER_SUDO={}",
if computer_sudo_enabled() { "true" } else { "false" }
if computer_sudo_enabled() {
"true"
} else {
"false"
}
),
format!(
"LAZYBOY_COMPUTER_DRIVER={}",
lazyboy_control::ComputerDriver::from_env().as_str()
),
]),
labels: Some(labels),
@ -384,6 +393,25 @@ impl DockerHost {
self.control_act(id, &request, &target).await
}
pub async fn browser(
&self,
id: &str,
request: BrowserRequest,
target: &ScreenTarget,
) -> Result<serde_json::Value, String> {
self.control_browser(id, &request, target).await
}
pub async fn recording(
&self,
id: &str,
action: &str,
request: RecordingRequest,
target: &ScreenTarget,
) -> Result<serde_json::Value, String> {
self.control_recording(id, action, &request, target).await
}
pub async fn screen_url(&self, id: &str, interactive: bool) -> Result<String, String> {
self.screen_url_for(id, 0, interactive).await
}
@ -421,11 +449,7 @@ impl DockerHost {
.await
.map_err(|error| error.to_string())?;
if info.state.as_ref().and_then(|state| state.running) == Some(true) {
let name = info
.name
.unwrap_or_default()
.trim_matches('/')
.to_string();
let name = info.name.unwrap_or_default().trim_matches('/').to_string();
if name.is_empty() {
return Err("computer container has no name".into());
}
@ -994,6 +1018,87 @@ PY"#,
}
serde_json::from_str(&result.stdout).map_err(|error| error.to_string())
}
async fn control_browser(
&self,
id: &str,
request: &BrowserRequest,
target: &ScreenTarget,
) -> Result<serde_json::Value, String> {
let payload = serde_json::to_string(request).map_err(|error| error.to_string())?;
let token = self.container_control_token(id).await?;
let timeout = if request.action == "click" {
"140"
} else {
"30"
};
let mut argv = vec![
"curl".into(),
"-fsS".into(),
"--max-time".into(),
timeout.into(),
"-H".into(),
format!("Authorization: Bearer {token}"),
"-H".into(),
format!("x-lazyboy-display: {}", target.display),
"-H".into(),
"content-type: application/json".into(),
];
if let Some(profile) = &target.profile_path {
argv.extend(["-H".into(), format!("x-lazyboy-profile: {profile}")]);
}
argv.extend([
"--data-binary".into(),
"@-".into(),
"http://127.0.0.1:7070/browser".into(),
]);
let result = self
.exec_raw_cmd(id, &argv, None, target, Some(payload))
.await?;
if result.code != 0 {
return Err(result.stderr);
}
serde_json::from_str(&result.stdout).map_err(|error| error.to_string())
}
async fn control_recording(
&self,
id: &str,
action: &str,
request: &RecordingRequest,
target: &ScreenTarget,
) -> Result<serde_json::Value, String> {
let payload = serde_json::to_string(request).map_err(|error| error.to_string())?;
let token = self.container_control_token(id).await?;
let timeout = if action == "collect" { "30" } else { "20" };
let mut argv = vec![
"curl".into(),
"-fsS".into(),
"--max-time".into(),
timeout.into(),
"-H".into(),
format!("Authorization: Bearer {token}"),
"-H".into(),
format!("x-lazyboy-display: {}", target.display),
"-H".into(),
"content-type: application/json".into(),
];
if let Some(profile) = &target.profile_path {
argv.extend(["-H".into(), format!("x-lazyboy-profile: {profile}")]);
}
argv.extend([
"--data-binary".into(),
"@-".into(),
format!("http://127.0.0.1:7070/recording/{action}"),
]);
let result = self
.exec_raw_cmd(id, &argv, None, target, Some(payload))
.await?;
if result.code != 0 {
return Err(result.stderr);
}
serde_json::from_str(&result.stdout).map_err(|error| error.to_string())
}
}
fn image_ids_match(wanted: &str, have: &str) -> bool {
@ -1041,7 +1146,10 @@ fn computer_pids_limit() -> i64 {
}
fn computer_sudo_enabled() -> bool {
matches!(std::env::var("LAZYBOY_COMPUTER_SUDO").as_deref(), Ok("1" | "true" | "yes"))
matches!(
std::env::var("LAZYBOY_COMPUTER_SUDO").as_deref(),
Ok("1" | "true" | "yes")
)
}
/// LXCFS supplies cgroup-aware /proc views so tools such as htop and free

101
crates/supervisor/src/main.rs
View File

@ -8,8 +8,8 @@ use axum::routing::{delete, get, post};
use axum::{Json, Router};
use docker::DockerHost;
use lazyboy_control::{
ActionRequest, CommandRequest, EnsureScreenRequest, HOME, ScreenTarget,
normalize_workspace_path,
ActionRequest, BrowserRequest, CommandRequest, EnsureScreenRequest, HOME, RecordingRequest,
ScreenTarget, normalize_workspace_path,
};
use serde::{Deserialize, Serialize};
use tracing_subscriber::EnvFilter;
@ -77,6 +77,10 @@ async fn main() {
.route("/computers/{id}/exec", post(exec))
.route("/computers/{id}/observe", post(observe))
.route("/computers/{id}/act", post(act))
.route("/computers/{id}/browser", post(browser))
.route("/computers/{id}/recording/start", post(recording_start))
.route("/computers/{id}/recording/stop", post(recording_stop))
.route("/computers/{id}/recording/collect", post(recording_collect))
.route("/computers/{id}/screens", post(ensure_screen))
.route("/computers/{id}/screen-mode", post(screen_mode))
.route("/computers/{id}/files", get(list_files).post(write_file))
@ -205,6 +209,99 @@ async fn act(
Ok(Json(result))
}
async fn browser(
State(app): State<App>,
headers: HeaderMap,
Path(id): Path<String>,
Json(body): Json<BrowserRequest>,
) -> Result<Json<serde_json::Value>, StatusCode> {
require_token(&headers, &app.token)?;
let mut body = body;
let target = screen_target(&headers);
if body.display.is_none() {
body.display = Some(target.display.clone());
}
if body.profile_path.is_none() {
body.profile_path = target.profile_path.clone();
}
let result = app
.docker
.browser(&id, body, &target)
.await
.map_err(|error| {
tracing::error!("browser: {error}");
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Json(result))
}
fn with_screen_target(mut body: RecordingRequest, target: &ScreenTarget) -> RecordingRequest {
if body.display.is_none() {
body.display = Some(target.display.clone());
}
if body.profile_path.is_none() {
body.profile_path = target.profile_path.clone();
}
body
}
async fn recording_start(
State(app): State<App>,
headers: HeaderMap,
Path(id): Path<String>,
Json(body): Json<RecordingRequest>,
) -> Result<Json<serde_json::Value>, StatusCode> {
require_token(&headers, &app.token)?;
let target = screen_target(&headers);
let result = app
.docker
.recording(&id, "start", with_screen_target(body, &target), &target)
.await
.map_err(|error| {
tracing::error!("recording start: {error}");
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Json(result))
}
async fn recording_stop(
State(app): State<App>,
headers: HeaderMap,
Path(id): Path<String>,
Json(body): Json<RecordingRequest>,
) -> Result<Json<serde_json::Value>, StatusCode> {
require_token(&headers, &app.token)?;
let target = screen_target(&headers);
let result = app
.docker
.recording(&id, "stop", with_screen_target(body, &target), &target)
.await
.map_err(|error| {
tracing::error!("recording stop: {error}");
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Json(result))
}
async fn recording_collect(
State(app): State<App>,
headers: HeaderMap,
Path(id): Path<String>,
Json(body): Json<RecordingRequest>,
) -> Result<Json<serde_json::Value>, StatusCode> {
require_token(&headers, &app.token)?;
let target = screen_target(&headers);
let result = app
.docker
.recording(&id, "collect", with_screen_target(body, &target), &target)
.await
.map_err(|error| {
tracing::error!("recording collect: {error}");
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Json(result))
}
#[derive(Deserialize)]
struct ScreenModeBody {
interactive: bool,

1
docker-compose.yml
View File

@ -51,6 +51,7 @@ services:
LAZYBOY_COMPUTER_MEMORY_MB: ${LAZYBOY_COMPUTER_MEMORY_MB:-2048}
LAZYBOY_COMPUTER_PIDS: ${LAZYBOY_COMPUTER_PIDS:-2048}
LAZYBOY_COMPUTER_SUDO: ${LAZYBOY_COMPUTER_SUDO:-false}
LAZYBOY_COMPUTER_DRIVER: ${LAZYBOY_COMPUTER_DRIVER:-cua}
LAZYBOY_LXCFS_ROOT: /var/lib/lxcfs
SUPERVISOR_BIND: 0.0.0.0:7091
DATA_DIR: /data

87
docs/cua-compatibility.md Normal file
View File

@ -0,0 +1,87 @@
# Cua Driver compatibility (LazyBoy desktop)
This report answers one question, from a real `make cua-smoke` run on 2026-09-07:
> Can Cua Driver reliably control the existing LazyBoy XFCE + Xvfb desktop container?
**Yes.** Five core actions succeeded 10 consecutive times. Production `computer_observe` / `computer_act` / `browser` still use the legacy controld path; this only proves Cua as a driver behind those abstractions.
## Environment
- Image: `lazyboy/computer:local` (`image/computer/Dockerfile`)
- Distro: Debian bookworm, x86_64
- Display: Xvfb `DISPLAY=:1` at 1280×800, XFCE (`xfwm4` compositor off, `xfce4-panel`, `xfdesktop`)
- Accessibility: AT-SPI 2 per screen (`at-spi-bus-launcher` + `at-spi2-registryd`)
- Browser: Debian `chromium` via `lazyboy-browser` (persistent profile, `--remote-debugging-port=9221+display`, `--force-renderer-accessibility`, `--lang=zh-TW`)
- Cua Driver: **0.23.2** (`cua-driver-rs-v0.23.2` linux-x86_64-binary, SHA256 `01bf8339ec129cc00f4b4b2c6056ef1a7c5b52df39ff83ad17c9b16818aec500`)
- Install path: `/usr/local/lib/cua-driver` + `/usr/local/bin/cua-driver` (not under the persisted `/home/lazyboy` bind)
- Daemon: `cua-driver serve --grant existing-profile --socket /tmp/lazyboy/cua.sock --no-overlay` on the primary display only
- Telemetry: disabled
- How to reproduce: `make cua-smoke` (artifacts in `/tmp/lazyboy-cua-smoke-last/`)
## Doctor
`cua-driver doctor --json` exit 0, `ok: true`.
| Probe | Status | Note |
| --- | --- | --- |
| binary | ok | `cua-driver 0.23.2 (x86_64-linux)` |
| install dir | ok | `/usr/local/lib/cua-driver/cua-driver` |
| telemetry | ok | disabled |
| display server | ok | X11 `DISPLAY=:1` |
| X11 connection | ok | connected, visible top-level windows |
| AT-SPI | **warn** | CLI `doctor` (docker exec) does not always see the XFCE session bus. The **daemon** started from `lazyboy-screen` does: native `get_window_state` + AT-SPI click/type worked 10/10. |
`cua-driver status --socket /tmp/lazyboy/cua.sock`: daemon running, permission mode `standard`. Unix socket rejects uid 0; smoke and future controld calls must run as uid 1000 (`lazyboy`).
## Results (10 consecutive iterations)
Independent application state, not Cua `"ok"`:
| Check | Result |
| --- | --- |
| `cua-driver --version` | `cua-driver 0.23.2` |
| screenshot (`get_desktop_state`) | 10/10 PNG of the XFCE desktop |
| window / accessibility observation | 10/10 `list_windows` + GTK `get_window_state` |
| native click | 10/10 GTK `Smoke Click` wrote `/tmp/lazyboy/cua-smoke-clicked` |
| native type | 10/10 GTK entry + `Smoke Save` wrote `hello-cua` |
| Chromium attach (existing window/profile) | 10/10 `browser_prepare` `attached_existing_profile` |
| browser semantic click / type | 10/10 local `http://127.0.0.1:8765/cua-smoke.html`; DOM became `clicked-ok` then `typed:hello-cua` |
| noVNC `:6080` still up | 10/10 |
| leftover Cua processes | none (only `cua-driver serve`) |
Same Chromium **pid 334** / **window_id 29360131** across all ten iterations. `browser_prepare` side effects were all false: no isolated profile, no copy, no restart, no extra remote-debugging toggle (LazyBoy already exposes loopback CDP).
Element refs are snapshot-scoped (`p1:1`, `p4:1`, … `p28:1`). Reusing an old ref would be wrong; the smoke re-snapshots every action.
## Relevant tools (0.23.2 `list-tools`)
Observation / native input: `get_desktop_state`, `list_windows`, `get_accessibility_tree`, `get_window_state`, `click`, `type_text`, `press_key`, `hotkey`, `scroll`, `drag`, `get_cursor_position`, `get_screen_size`.
Browser (attach only): `browser_prepare` (`strategy.kind=existing_profile`, `allow_launch=false`), `get_browser_state` (`semantic_v2`), `browser_navigate` (http/https/about only), `browser_click`, `browser_type`.
Not used here: recording, isolated `launch_app` browsers, Wayland helpers. These names must not be exposed to the LLM.
## Integration notes for the next PR
- Call Cua as uid 1000 via `cua-driver call --socket /tmp/lazyboy/cua.sock`. Root is rejected (`reject Unix peer uid 0 for runtime owned by uid 1000`).
- `get_browser_state` on a live LazyBoy Chromium first returns `browser_consent_required` / `consumer_profile_endpoint_requires_grant`. Then `browser_prepare` with `existing_profile` attaches. Serve must keep `--grant existing-profile`. Never `allow_launch`.
- Linux Chromium trusted CDP pointer is unavailable; smoke used `browser_click` `input_route=dom_event` and verified the DOM. Production adapter should prefer that route on this platform and treat `browser_input_trust_unavailable` as classified, not a silent xdotool fallback.
- Extra Team screens are extra Xvfb `DISPLAY`s. This POC only runs a daemon on `:1`. Later: one socket per slot.
- `browser_navigate` refuses `file://`; local fixtures need `http://127.0.0.1`.
- `get_window_state` on Linux is `additionalProperties: false` — do not send macOS-only fields such as `include_accessibility_tree`.
## Known limits
- Doctor AT-SPI warn from a non-desktop D-Bus is not a daemon failure.
- Overlay warnings (`X11 channel rejected command`) appeared in the daemon log with `--no-overlay`; they did not block actions.
- Debian Chromium + zh-TW UI: existing-profile attach worked because CDP was already open, so Cua did not need the English setup-checkbox path.
- Multi-screen, pause/resume, takeover, and skill recording were **not** in this POC; later PRs added controller routing, browser attach, takeover re-observe, and dual-source skill recording.
## Conclusion
Cua Driver 0.23.2 **can** control the existing LazyBoy XFCE + Xvfb container: screenshot, window/AT-SPI observation, native click/type, and Chromium semantic click/type, 10/10, without replacing the browser profile or breaking noVNC.
`ComputerController` is in place. Production now defaults to `cua`. Set `LAZYBOY_COMPUTER_DRIVER=legacy` on the supervisor (passed into each desktop container) to roll back to CDP/AT-SPI/xdotool. Recreate desktop containers after changing the flag.
With `cua`: `POST /observe`, `POST /act`, and `POST /browser` go through Cua Driver. The Agent-facing `browser` schema is unchanged (`snapshot` / `click` / `type` / `press` / `navigate` / `wait`); Cua attaches with `existing_profile` and maps `semantic_v2` refs (`pN:M`) onto the existing element list. After human takeover ends, the run forces a fresh `computer_observe` and drops pre-handoff ids/refs. Skill teaching starts Cua `start_recording` (no video) plus the existing CDP DOM recorder so a human noVNC demo still yields semantic click/type/navigate events; Cua trajectory turns are ingested as extra evidence and password-labelled typing is masked. `use_saved_login` still fills via CDP stdin so passwords never appear on argv. `cdp.py` / AT-SPI remain for login fill, human browser recording, and the `legacy` rollback.

7
docs/development.md
View File

@ -38,6 +38,13 @@ python3 tests/control.test.py
python3 tests/log-rotation.test.py
python3 tests/shell-session.test.py # 持久終端機腳本，只需要 tmux
# Cua Driver 能否控制現有 XFCE + Xvfb 桌面（會建 computer image）
make cua-smoke
# 結果摘要見 docs/cua-compatibility.md
# 生產路徑預設是 cua。要回退 CDP/xdotool：
# LAZYBOY_COMPUTER_DRIVER=legacy
# 寫進 .env 後重建 supervisor 與桌面容器。Agent 工具 schema 不變。
# Python 整合測試用 docker compose exec 連進 Postgres，自己建一次性資料庫後清掉
python3 tests/retention.test.py
python3 tests/run-resume.test.py

1
docs/operations.md
View File

@ -57,6 +57,7 @@
| `LAZYBOY_COMPUTER_MEMORY_MB` | 每台電腦記憶體 | `2048` |
| `LAZYBOY_COMPUTER_PIDS` | 每台電腦 PID 上限 | `2048` |
| `LAZYBOY_COMPUTER_SUDO` | 容器內免密碼 sudo；重建桌面容器後生效 | `false` |
| `LAZYBOY_COMPUTER_DRIVER` | 桌面控制後端：`cua`（預設）或 `legacy`（observe / act / browser / 示範錄製）；重建桌面容器後生效 | `cua` |
| `LAZYBOY_MEMORY_ENABLED` | 長期記憶 | `true` |
完整清單與保留政策請見 [`.env.example`](../.env.example)。

22
image/computer/Dockerfile
View File

@ -65,6 +65,9 @@ RUN printf '%s\n' \
libatk-adaptor \
python3-gi \
gir1.2-atspi-2.0 \
gir1.2-gtk-3.0 \
libxi6 \
libxkbcommon0 \
zsh \
&& echo "zh_TW.UTF-8 UTF-8" >> /etc/locale.gen \
&& echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \
@ -136,8 +139,27 @@ COPY --chmod=644 image/computer/xfce/helper-terminal.desktop /usr/share/xfce4/he
COPY --chmod=644 image/computer/xfce/helper-browser.desktop /usr/share/xfce4/helpers/lazyboy-browser.desktop
COPY --chmod=755 image/computer/chromium /usr/local/bin/chromium
RUN sed -i 's|^Exec=/usr/bin/chromium|Exec=/usr/local/bin/lazyboy-browser|' /usr/share/applications/chromium.desktop || true
# Pin Cua Driver outside the persisted /home/lazyboy bind-mount.
# SHA256 is the linux-x86_64-binary tarball from the matching GitHub release.
ARG CUA_DRIVER_RS_VERSION=0.23.2
ARG CUA_DRIVER_RS_SHA256=01bf8339ec129cc00f4b4b2c6056ef1a7c5b52df39ff83ad17c9b16818aec500
RUN curl -fsSL -o /tmp/cua-driver.tar.gz \
"https://github.com/trycua/cua/releases/download/cua-driver-rs-v${CUA_DRIVER_RS_VERSION}/cua-driver-rs-${CUA_DRIVER_RS_VERSION}-linux-x86_64-binary.tar.gz" \
&& echo "${CUA_DRIVER_RS_SHA256} /tmp/cua-driver.tar.gz" | sha256sum -c \
&& mkdir -p /usr/local/lib/cua-driver \
&& tar -xzf /tmp/cua-driver.tar.gz -C /usr/local/lib/cua-driver \
&& chmod 755 /usr/local/lib/cua-driver/cua-driver \
&& ln -sf /usr/local/lib/cua-driver/cua-driver /usr/local/bin/cua-driver \
&& rm -f /tmp/cua-driver.tar.gz \
&& cua-driver --version
COPY --chmod=755 image/computer/start.sh /usr/local/bin/lazyboy-computer
COPY --chmod=755 image/computer/entrypoint.sh /usr/local/bin/lazyboy-entrypoint
COPY --chmod=644 image/computer/cua-smoke.html /usr/share/lazyboy/cua-smoke.html
COPY --chmod=755 image/computer/cua-smoke-gtk.py /usr/local/bin/lazyboy-cua-smoke-gtk
COPY --chmod=755 scripts/cua-smoke-inner.py /usr/local/bin/lazyboy-cua-smoke
COPY --chmod=755 scripts/cua-smoke-test.sh /usr/local/bin/lazyboy-cua-smoke-host
USER root
ENV HOME=/home/lazyboy DISPLAY=:1 SHELL=/bin/zsh TERM=xterm-256color \

64
image/computer/cua-smoke-gtk.py Executable file
View File

@ -0,0 +1,64 @@
#!/usr/bin/env python3
"""Tiny GTK window used by the Cua smoke test to verify native click/type."""
from pathlib import Path
import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk
ROOT = Path("/tmp/lazyboy")
CLICKED = ROOT / "cua-smoke-clicked"
TYPED = ROOT / "cua-smoke-typed"
class SmokeWindow(Gtk.Window):
def __init__(self) -> None:
super().__init__(title="LazyBoy Cua Smoke")
self.set_default_size(480, 240)
box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=8)
box.set_margin_top(16)
box.set_margin_bottom(16)
box.set_margin_start(16)
box.set_margin_end(16)
self.add(box)
self.status = Gtk.Label(label="ready")
self.entry = Gtk.Entry()
self.entry.set_placeholder_text("type here")
click = Gtk.Button(label="Smoke Click")
save = Gtk.Button(label="Smoke Save")
click.connect("clicked", self.on_click)
save.connect("clicked", self.on_save)
box.pack_start(self.status, False, False, 0)
box.pack_start(click, False, False, 0)
box.pack_start(self.entry, False, False, 0)
box.pack_start(save, False, False, 0)
click.get_accessible().set_name("Smoke Click")
self.entry.get_accessible().set_name("Smoke Entry")
save.get_accessible().set_name("Smoke Save")
self.status.get_accessible().set_name("Smoke Status")
self.connect("destroy", Gtk.main_quit)
def on_click(self, _button: Gtk.Button) -> None:
ROOT.mkdir(parents=True, exist_ok=True)
CLICKED.write_text("clicked\n", encoding="utf-8")
self.status.set_text("clicked-ok")
def on_save(self, _button: Gtk.Button) -> None:
ROOT.mkdir(parents=True, exist_ok=True)
TYPED.write_text(self.entry.get_text(), encoding="utf-8")
self.status.set_text("typed-ok")
def main() -> None:
ROOT.mkdir(parents=True, exist_ok=True)
window = SmokeWindow()
window.show_all()
Gtk.main()
if __name__ == "__main__":
main()

31
image/computer/cua-smoke.html Normal file
View File

@ -0,0 +1,31 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>LazyBoy Cua Smoke</title>
<style>
body { font-family: sans-serif; margin: 32px; }
#result { font-size: 20px; }
</style>
</head>
<body>
<h1>LazyBoy Cua Smoke</h1>
<p id="result">ready</p>
<p>
<button id="click" type="button">Smoke Click</button>
</p>
<p>
<input id="name" type="text" placeholder="type here" aria-label="Smoke Entry">
<button id="save" type="button">Smoke Save</button>
</p>
<script>
document.getElementById("click").onclick = function () {
document.getElementById("result").textContent = "clicked-ok";
};
document.getElementById("save").onclick = function () {
var value = document.getElementById("name").value;
document.getElementById("result").textContent = "typed:" + value;
};
</script>
</body>
</html>

1
image/computer/lazyboy-browser
View File

@ -54,6 +54,7 @@ exec /usr/bin/chromium \
--disable-session-crashed-bubble \
--hide-crash-restore-bubble \
--disable-infobars \
--force-renderer-accessibility \
--password-store=basic \
--lang=zh-TW \
--accept-lang=zh-TW,zh,en-US,en \

43
image/computer/lazyboy-screen
View File

@ -211,6 +211,48 @@ start_vnc() {
wait_port "$view_port"
}
start_cua_driver() {
local display="$1"
local log="$2"
local number="${display#:}"
if ! command -v cua-driver >/dev/null 2>&1; then
return 0
fi
if alive_pidfile "${log}-cua.pid"; then
return 0
fi
export CUA_DRIVER_RS_HOME="${CUA_DRIVER_RS_HOME:-$ROOT/cua-home}"
mkdir -p "$CUA_DRIVER_RS_HOME"
cua-driver telemetry disable >>"${log}-cua.log" 2>&1 || true
local sock="$ROOT/cua-${number}.sock"
if [[ "$number" == "1" ]]; then
sock="$ROOT/cua.sock"
fi
rm -f "$sock"
DISPLAY="$display" cua-driver serve \
--grant existing-profile \
--socket "$sock" \
--pid-file "${log}-cua.pid" \
--no-overlay \
>>"${log}-cua.log" 2>&1 &
local bg=$!
local n
for n in $(seq 1 50); do
if [[ -S "$sock" ]]; then
return 0
fi
if ! kill -0 "$bg" 2>/dev/null; then
echo "cua-driver serve exited" >&2
cat "${log}-cua.log" >&2 || true
return 1
fi
sleep 0.1
done
echo "cua-driver socket was not ready" >&2
cat "${log}-cua.log" >&2 || true
return 1
}
start_xterm() {
local display="$1"
local log="$2"
@ -286,6 +328,7 @@ ensure_slot() {
exit 1
}
start_desktop "$display" "$xfce_home" "$log"
start_cua_driver "$display" "$log" || true
start_xterm "$display" "$log"
start_vnc "$display" "$vnc_port" "$view_port" "$log" || exit 1
if [[ -n "$profile" ]]; then

785
scripts/cua-smoke-inner.py Executable file
View File

@ -0,0 +1,785 @@
#!/usr/bin/env python3
"""In-container Cua Driver smoke test for the LazyBoy XFCE + Xvfb desktop."""
from __future__ import annotations
import argparse
import json
import os
import signal
import subprocess
import sys
import time
from pathlib import Path
from typing import Any
ROOT = Path("/tmp/lazyboy")
SOCKET = ROOT / "cua.sock"
REPORT = ROOT / "cua-smoke-report"
CLICKED = ROOT / "cua-smoke-clicked"
TYPED = ROOT / "cua-smoke-typed"
GTK_SCRIPT = Path("/usr/local/bin/lazyboy-cua-smoke-gtk")
HTML = Path("/usr/share/lazyboy/cua-smoke.html")
FIXTURE_PORT = 8765
FIXTURE_URL = f"http://127.0.0.1:{FIXTURE_PORT}/cua-smoke.html"
TYPED_TEXT = "hello-cua"
class SmokeError(RuntimeError):
pass
def log(message: str) -> None:
print(message, flush=True)
def write_json(path: Path, value: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value, indent=2, default=str) + "\n", encoding="utf-8")
def run(
argv: list[str],
timeout: int = 60,
env: dict[str, str] | None = None,
input_text: str | None = None,
) -> subprocess.CompletedProcess[str]:
return subprocess.run(
argv,
check=False,
capture_output=True,
text=True,
timeout=timeout,
env=env,
input=input_text,
)
def parse_jsonish(text: str) -> Any:
text = text.strip()
if not text:
return None
try:
return json.loads(text)
except json.JSONDecodeError:
pass
start = text.find("{")
end = text.rfind("}")
if start >= 0 and end > start:
try:
return json.loads(text[start : end + 1])
except json.JSONDecodeError:
return None
start = text.find("[")
end = text.rfind("]")
if start >= 0 and end > start:
try:
return json.loads(text[start : end + 1])
except json.JSONDecodeError:
return None
return None
def walk(value: Any) -> list[Any]:
found = [value]
if isinstance(value, dict):
for item in value.values():
found.extend(walk(item))
elif isinstance(value, list):
for item in value:
found.extend(walk(item))
return found
def first_list_of_dicts(value: Any, required_key: str) -> list[dict[str, Any]]:
for node in walk(value):
if isinstance(node, list) and node and all(isinstance(item, dict) for item in node):
if any(required_key in item for item in node):
return node
if isinstance(node, dict) and required_key in node and isinstance(node[required_key], list):
items = node[required_key]
if items and all(isinstance(item, dict) for item in items):
return items
return []
def cua_env() -> dict[str, str]:
env = os.environ.copy()
env.setdefault("DISPLAY", ":1")
env.setdefault("CUA_DRIVER_RS_HOME", str(ROOT / "cua-home"))
env["PATH"] = "/usr/local/bin:/usr/local/lib/cua-driver:" + env.get("PATH", "")
dbus = ROOT / "screen-1.dbus"
if dbus.exists() and "DBUS_SESSION_BUS_ADDRESS" not in env:
env["DBUS_SESSION_BUS_ADDRESS"] = dbus.read_text(encoding="utf-8").strip()
runtime = ROOT / "screen-1.runtime"
if runtime.exists() and "XDG_RUNTIME_DIR" not in env:
env["XDG_RUNTIME_DIR"] = runtime.read_text(encoding="utf-8").strip()
env.setdefault("GTK_MODULES", "atk-bridge")
env.setdefault("GTK_A11Y", "atspi")
env.setdefault("NO_AT_BRIDGE", "0")
return env
def cua_bin(args: list[str], timeout: int = 60) -> subprocess.CompletedProcess[str]:
return run(["cua-driver", *args], timeout=timeout, env=cua_env())
def cua_call(tool: str, payload: dict[str, Any] | None = None, extra: list[str] | None = None, timeout: int = 90) -> dict[str, Any]:
argv = ["call", "--socket", str(SOCKET), tool]
if extra:
argv.extend(extra)
argv.append(json.dumps(payload or {}))
proc = cua_bin(argv, timeout=timeout)
parsed = parse_jsonish(proc.stdout) or parse_jsonish(proc.stderr)
result = {
"tool": tool,
"code": proc.returncode,
"stdout": proc.stdout,
"stderr": proc.stderr,
"parsed": parsed,
}
if proc.returncode != 0:
raise SmokeError(f"{tool} failed ({proc.returncode}): {proc.stderr or proc.stdout}")
combined = f"{proc.stdout or ''}\n{proc.stderr or ''}"
if "❌" in combined:
raise SmokeError(f"{tool} reported an error: {combined[-2000:]}")
return result
def wait_ready(timeout: int) -> None:
deadline = time.time() + timeout
ready = ROOT / "ready"
while time.time() < deadline:
if ready.exists() and SOCKET.is_socket():
status = cua_bin(["status", "--socket", str(SOCKET)], timeout=15)
text = (status.stdout or "") + (status.stderr or "")
if status.returncode == 0 and "not running" not in text.lower():
return
time.sleep(0.4)
raise SmokeError("desktop or cua-driver socket was not ready")
def png_ok(path: Path) -> None:
data = path.read_bytes()
if len(data) < 32 or data[:8] != b"\x89PNG\r\n\x1a\n":
raise SmokeError(f"{path} is not a PNG ({len(data)} bytes)")
def kill_matching(pattern: str) -> None:
run(["pkill", "-f", pattern], timeout=10)
def launch_gtk() -> subprocess.Popen[bytes]:
kill_matching("lazyboy-cua-smoke-gtk")
for path in (CLICKED, TYPED):
if path.exists():
path.unlink()
log_path = REPORT / "gtk.log"
log_file = log_path.open("w", encoding="utf-8")
proc = subprocess.Popen(
[str(GTK_SCRIPT)],
env=cua_env(),
stdout=log_file,
stderr=log_file,
start_new_session=True,
)
deadline = time.time() + 15
while time.time() < deadline:
if proc.poll() is not None:
log_file.close()
detail = log_path.read_text(encoding="utf-8")[-1500:]
raise SmokeError(f"GTK smoke window exited immediately: {detail}")
windows = list_windows()
if find_gtk_window(windows):
return proc
time.sleep(0.3)
log_file.close()
raise SmokeError(f"GTK smoke window did not appear; windows={list_windows()}")
def start_fixture_server() -> subprocess.Popen[bytes]:
proc = subprocess.Popen(
[
"python3",
"-m",
"http.server",
str(FIXTURE_PORT),
"--bind",
"127.0.0.1",
"--directory",
str(HTML.parent),
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
start_new_session=True,
)
deadline = time.time() + 8
while time.time() < deadline:
check = run(
[
"python3",
"-c",
f"import urllib.request; urllib.request.urlopen('{FIXTURE_URL}', timeout=1).read()",
],
timeout=5,
)
if check.returncode == 0:
return proc
if proc.poll() is not None:
raise SmokeError("fixture HTTP server exited")
time.sleep(0.2)
raise SmokeError("fixture HTTP server did not become ready")
def launch_browser() -> None:
env = cua_env()
subprocess.Popen(
["lazyboy-browser", FIXTURE_URL],
env=env,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
start_new_session=True,
)
deadline = time.time() + 25
while time.time() < deadline:
windows = list_windows()
if find_browser_window(windows):
return
time.sleep(0.4)
raise SmokeError("Chromium window did not appear")
def list_windows() -> list[dict[str, Any]]:
result = cua_call("list_windows", {"on_screen_only": True})
windows = first_list_of_dicts(result["parsed"], "window_id")
if not windows:
windows = first_list_of_dicts(result["parsed"], "title")
return windows
def window_title(window: dict[str, Any]) -> str:
return str(window.get("title") or window.get("app_name") or "")
def find_window(windows: list[dict[str, Any]], title_part: str) -> dict[str, Any] | None:
needle = title_part.lower()
for window in windows:
if needle in window_title(window).lower():
return window
return None
def find_gtk_window(windows: list[dict[str, Any]]) -> dict[str, Any] | None:
for window in windows:
title = window_title(window)
app = str(window.get("app_name") or "")
if title == "LazyBoy Cua Smoke" and "chrom" not in app.lower():
return window
return None
def find_browser_window(windows: list[dict[str, Any]]) -> dict[str, Any] | None:
for window in windows:
blob = " ".join(
str(window.get(key) or "")
for key in ("title", "app_name", "application", "wm_class")
).lower()
if "chrom" in blob or "lazyboy cua smoke" in blob:
return window
return None
def snapshot(pid: int, window_id: int, name: str) -> dict[str, Any]:
out = REPORT / f"{name}.png"
result = cua_call(
"get_window_state",
{
"pid": pid,
"window_id": window_id,
"include_screenshot": True,
"screenshot_out_file": str(out),
},
timeout=120,
)
parsed = result["parsed"] or {}
elements = first_list_of_dicts(parsed, "element_index") or first_list_of_dicts(
parsed, "element_token"
)
snapshot_id = None
for node in walk(parsed):
if isinstance(node, dict) and node.get("snapshot_id"):
snapshot_id = node["snapshot_id"]
break
return {
"result": result,
"elements": elements,
"snapshot_id": snapshot_id,
"png": out if out.exists() else None,
}
def element_by_label(elements: list[dict[str, Any]], label: str) -> dict[str, Any]:
needle = label.lower()
for element in elements:
hay = " ".join(
str(element.get(key) or "")
for key in ("label", "name", "title", "role", "value")
).lower()
if needle in hay:
return element
raise SmokeError(f"no accessibility element matching {label!r}")
def click_element(pid: int, window_id: int, element: dict[str, Any], snapshot_id: Any) -> None:
payload: dict[str, Any] = {"pid": pid, "window_id": window_id}
if element.get("element_token"):
payload["element_token"] = element["element_token"]
elif element.get("element_index") is not None and snapshot_id:
payload["element_index"] = element["element_index"]
payload["snapshot_id"] = snapshot_id
else:
raise SmokeError(f"element has no token/index: {element}")
cua_call("click", payload)
def type_element(pid: int, window_id: int, element: dict[str, Any], snapshot_id: Any, text: str) -> None:
payload: dict[str, Any] = {"pid": pid, "window_id": window_id, "text": text}
if element.get("element_token"):
payload["element_token"] = element["element_token"]
elif element.get("element_index") is not None and snapshot_id:
payload["element_index"] = element["element_index"]
payload["snapshot_id"] = snapshot_id
cua_call("type_text", payload)
def wait_file(path: Path, expect: str | None, timeout: float = 8.0) -> str:
deadline = time.time() + timeout
while time.time() < deadline:
if path.exists():
text = path.read_text(encoding="utf-8").strip()
if expect is None or expect in text:
return text
time.sleep(0.2)
raise SmokeError(f"{path} did not contain {expect!r}")
def desktop_screenshot() -> None:
out = REPORT / "desktop.png"
cua_call(
"get_desktop_state",
{"screenshot_out_file": str(out)},
extra=["--screenshot-out-file", str(out)],
)
if not out.exists():
# Some builds only honour the JSON field or only the flag.
time.sleep(0.2)
if not out.exists():
raise SmokeError("get_desktop_state did not write a screenshot")
png_ok(out)
def collect_diagnostics() -> dict[str, Any]:
version = cua_bin(["--version"])
doctor = cua_bin(["doctor", "--json"], timeout=90)
if doctor.returncode != 0:
doctor = cua_bin(["doctor"], timeout=90)
tools = cua_bin(["list-tools"], timeout=30)
status = cua_bin(["status", "--socket", str(SOCKET)], timeout=15)
report = {
"version": (version.stdout or version.stderr).strip(),
"doctor_code": doctor.returncode,
"doctor_stdout": doctor.stdout,
"doctor_stderr": doctor.stderr,
"doctor_json": parse_jsonish(doctor.stdout),
"list_tools": tools.stdout,
"status": (status.stdout or "") + (status.stderr or ""),
"display": os.environ.get("DISPLAY", ":1"),
"socket": str(SOCKET),
}
write_json(REPORT / "diagnostics.json", report)
(REPORT / "list-tools.txt").write_text(tools.stdout or "", encoding="utf-8")
(REPORT / "doctor.txt").write_text(
(doctor.stdout or "") + (doctor.stderr or ""), encoding="utf-8"
)
if version.returncode != 0:
raise SmokeError("cua-driver --version failed")
return report
def native_round(iteration: int) -> None:
for path in (CLICKED, TYPED):
if path.exists():
path.unlink()
gtk = launch_gtk()
try:
windows = list_windows()
window = find_gtk_window(windows)
if not window:
raise SmokeError(f"GTK window missing: {windows}")
pid = int(window["pid"])
window_id = int(window["window_id"])
state = snapshot(pid, window_id, f"native-{iteration}")
click_el = element_by_label(state["elements"], "Smoke Click")
click_element(pid, window_id, click_el, state["snapshot_id"])
wait_file(CLICKED, "clicked")
state = snapshot(pid, window_id, f"native-type-{iteration}")
entry = element_by_label(state["elements"], "Smoke Entry")
type_element(pid, window_id, entry, state["snapshot_id"], TYPED_TEXT)
state = snapshot(pid, window_id, f"native-save-{iteration}")
save = element_by_label(state["elements"], "Smoke Save")
click_element(pid, window_id, save, state["snapshot_id"])
wait_file(TYPED, TYPED_TEXT)
finally:
gtk.send_signal(signal.SIGTERM)
try:
gtk.wait(timeout=5)
except subprocess.TimeoutExpired:
gtk.kill()
def extract_text(value: Any) -> str:
chunks: list[str] = []
for node in walk(value):
if isinstance(node, str) and 0 < len(node) < 400:
chunks.append(node)
elif isinstance(node, dict):
for key in ("text", "value", "label", "name", "url", "title", "result", "ref"):
item = node.get(key)
if isinstance(item, str) and 0 < len(item) < 400:
chunks.append(item)
return "\n".join(chunks)
def strip_heavy(value: Any) -> Any:
if isinstance(value, dict):
out = {}
for key, item in value.items():
if isinstance(item, str) and len(item) > 400:
out[key] = f"<{len(item)} chars>"
else:
out[key] = strip_heavy(item)
return out
if isinstance(value, list):
return [strip_heavy(item) for item in value]
return value
def find_browser_ref(parsed: Any, needle: str) -> str | None:
needle = needle.lower()
for node in walk(parsed):
if not isinstance(node, dict):
continue
ref = node.get("ref")
if not isinstance(ref, str) or not ref:
continue
blob = " ".join(
str(node.get(key) or "")
for key in (
"name",
"label",
"text",
"role",
"description",
"accessible_name",
"selector",
"value",
)
).lower()
if needle in blob or needle in json.dumps(strip_heavy(node), default=str).lower():
return ref
return None
def browser_ids(parsed: Any) -> tuple[Any, Any]:
target_id = None
tab_id = None
for node in walk(parsed):
if not isinstance(node, dict):
continue
target_id = target_id or node.get("target_id")
tab_id = tab_id or node.get("tab_id")
tabs = node.get("tabs")
if isinstance(tabs, list) and tabs and isinstance(tabs[0], dict):
tab_id = tab_id or tabs[0].get("tab_id") or tabs[0].get("id")
return target_id, tab_id
def browser_prepare(pid: int, window_id: int) -> dict[str, Any]:
try:
return cua_call(
"browser_prepare",
{
"pid": pid,
"window_id": window_id,
"session": "lazyboy-smoke",
"strategy": {"kind": "existing_profile"},
"allow_launch": False,
},
timeout=180,
)
except SmokeError as error:
return {"error": str(error)}
def browser_round(iteration: int) -> dict[str, Any]:
windows = list_windows()
window = find_browser_window(windows)
if not window:
launch_browser()
windows = list_windows()
window = find_browser_window(windows)
if not window:
raise SmokeError(f"no Chromium window: {windows}")
pid = int(window["pid"])
window_id = int(window["window_id"])
bind_payload = {
"pid": pid,
"window_id": window_id,
"include_screenshot": False,
"session": "lazyboy-smoke",
}
state = cua_call("get_browser_state", bind_payload, timeout=120)
note = extract_text(state["parsed"]) + state["stdout"]
prepared = None
if any(
marker in note
for marker in (
"browser_requires_setup",
"browser_consent_required",
"requires_grant",
"existing_profile",
)
):
prepared = browser_prepare(pid, window_id)
write_json(REPORT / "browser-prepare.json", prepared)
prep_text = extract_text(prepared.get("parsed")) + str(prepared.get("stdout") or "") + str(prepared.get("error") or "")
if prepared.get("error") or "❌" in prep_text or "refused" in prep_text:
raise SmokeError(f"browser_prepare failed: {prep_text[-2000:]}")
state = cua_call("get_browser_state", bind_payload, timeout=120)
note = extract_text(state["parsed"]) + state["stdout"]
target_id, tab_id = browser_ids(state["parsed"])
if not target_id or not tab_id:
raise SmokeError(f"browser bind missing target/tab: {note[-2000:]}")
cua_call(
"browser_navigate",
{
"target_id": target_id,
"tab_id": tab_id,
"url": FIXTURE_URL,
"session": "lazyboy-smoke",
},
timeout=60,
)
time.sleep(0.8)
snap = cua_call(
"get_browser_state",
{
"target_id": target_id,
"tab_id": tab_id,
"include_screenshot": False,
"session": "lazyboy-smoke",
"snapshot_format": "semantic_v2",
"query": "Smoke",
},
timeout=90,
)
write_json(REPORT / f"browser-snap-{iteration}.json", strip_heavy(snap["parsed"]))
click_ref = find_browser_ref(snap["parsed"], "smoke click")
type_ref = find_browser_ref(snap["parsed"], "smoke entry")
save_ref = find_browser_ref(snap["parsed"], "smoke save")
if not type_ref:
type_ref = find_browser_ref(snap["parsed"], "textbox") or find_browser_ref(
snap["parsed"], "input"
)
if not click_ref:
raise SmokeError(f"browser snapshot has no Smoke Click ref: {extract_text(snap['parsed'])[-1500:]}")
click_result = cua_call(
"browser_click",
{
"target_id": target_id,
"tab_id": tab_id,
"ref": click_ref,
"session": "lazyboy-smoke",
"input_route": "dom_event",
},
timeout=60,
)
write_json(REPORT / f"browser-click-{iteration}.json", strip_heavy(click_result["parsed"] or click_result["stdout"]))
time.sleep(0.4)
after_click = cua_call(
"get_browser_state",
{
"target_id": target_id,
"tab_id": tab_id,
"session": "lazyboy-smoke",
"snapshot_format": "semantic_v2",
"include_screenshot": False,
},
timeout=90,
)
write_json(REPORT / f"browser-after-click-{iteration}.json", strip_heavy(after_click["parsed"]))
click_text = json.dumps(strip_heavy(after_click["parsed"]), default=str)
if "clicked-ok" not in click_text:
raise SmokeError(f"browser click did not change page text to clicked-ok: {click_text[-1500:]}")
type_ref = find_browser_ref(after_click["parsed"], "smoke entry") or find_browser_ref(
after_click["parsed"], "textbox"
)
save_ref = find_browser_ref(after_click["parsed"], "smoke save")
if not type_ref:
raise SmokeError("browser snapshot has no Smoke Entry ref")
cua_call(
"browser_type",
{
"target_id": target_id,
"tab_id": tab_id,
"ref": type_ref,
"text": TYPED_TEXT,
"replace": True,
"session": "lazyboy-smoke",
},
timeout=60,
)
if not save_ref:
raise SmokeError("browser snapshot has no Smoke Save ref")
cua_call(
"browser_click",
{
"target_id": target_id,
"tab_id": tab_id,
"ref": save_ref,
"session": "lazyboy-smoke",
"input_route": "dom_event",
},
timeout=60,
)
time.sleep(0.4)
after_type = cua_call(
"get_browser_state",
{
"target_id": target_id,
"tab_id": tab_id,
"session": "lazyboy-smoke",
"snapshot_format": "semantic_v2",
"include_screenshot": False,
},
timeout=90,
)
write_json(REPORT / f"browser-after-type-{iteration}.json", strip_heavy(after_type["parsed"]))
type_text = json.dumps(strip_heavy(after_type["parsed"]), default=str)
if f"typed:{TYPED_TEXT}" not in type_text:
raise SmokeError(f"browser type did not land in the page: {type_text[-1500:]}")
return {
"pid": pid,
"window_id": window_id,
"target_id": target_id,
"tab_id": tab_id,
"prepared": prepared,
"click_ref": click_ref,
}
def novnc_ok() -> None:
proc = run(
[
"python3",
"-c",
"import socket; s=socket.create_connection(('127.0.0.1',6080),2); s.close()",
],
timeout=10,
)
if proc.returncode != 0:
raise SmokeError("noVNC port 6080 is not accepting connections")
def extra_cua_processes() -> list[str]:
proc = run(["ps", "-eo", "pid,cmd"], timeout=10)
lines = []
for line in (proc.stdout or "").splitlines():
if "cua-driver" in line and "serve" not in line and "cua-smoke" not in line:
if "cua-driver serve" in line:
continue
if str(os.getpid()) in line.split()[:1]:
continue
lines.append(line.strip())
return lines
def one_iteration(iteration: int) -> dict[str, Any]:
log(f"iteration {iteration}: screenshot")
desktop_screenshot()
log(f"iteration {iteration}: windows")
windows = list_windows()
if not windows:
raise SmokeError("list_windows returned no windows")
write_json(REPORT / f"windows-{iteration}.json", windows)
log(f"iteration {iteration}: native click/type")
native_round(iteration)
log(f"iteration {iteration}: browser")
browser = browser_round(iteration)
log(f"iteration {iteration}: noVNC")
novnc_ok()
return {"windows": len(windows), "browser": browser}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repeat", type=int, default=10)
parser.add_argument("--ready-timeout", type=int, default=90)
args = parser.parse_args()
REPORT.mkdir(parents=True, exist_ok=True)
summary: dict[str, Any] = {
"repeat": args.repeat,
"passed": 0,
"failed": 0,
"iterations": [],
"ok": False,
}
fixture: subprocess.Popen[bytes] | None = None
try:
wait_ready(args.ready_timeout)
fixture = start_fixture_server()
diagnostics = collect_diagnostics()
summary["diagnostics"] = {
"version": diagnostics["version"],
"doctor_code": diagnostics["doctor_code"],
"status": diagnostics["status"],
}
for iteration in range(1, args.repeat + 1):
try:
detail = one_iteration(iteration)
summary["passed"] += 1
summary["iterations"].append({"n": iteration, "ok": True, **detail})
log(f"iteration {iteration}: ok")
except Exception as error: # noqa: BLE001 — smoke must record any failure
summary["failed"] += 1
summary["iterations"].append(
{"n": iteration, "ok": False, "error": str(error)}
)
summary["error"] = str(error)
log(f"iteration {iteration}: FAIL {error}")
raise
leftovers = extra_cua_processes()
summary["leftover_cua_processes"] = leftovers
if leftovers:
raise SmokeError(f"extra cua-driver processes: {leftovers}")
summary["ok"] = True
log(f"passed {args.repeat}/{args.repeat}")
return 0
except subprocess.TimeoutExpired as error:
summary["error"] = f"timeout: {error}"
log(summary["error"])
return 1
except SmokeError as error:
summary["error"] = str(error)
log(f"FAIL {error}")
return 1
finally:
write_json(REPORT / "result.json", summary)
kill_matching("lazyboy-cua-smoke-gtk")
if fixture is not None:
fixture.kill()
kill_matching(f"http.server {FIXTURE_PORT}")
if __name__ == "__main__":
sys.exit(main())

92
scripts/cua-smoke-test.sh Executable file
View File

@ -0,0 +1,92 @@
#!/usr/bin/env bash
# Prove Cua Driver can control the LazyBoy XFCE + Xvfb desktop.
# Host usage: scripts/cua-smoke-test.sh --docker [--repeat 10]
# In-container: lazyboy-cua-smoke --repeat 10
set -euo pipefail
repeat=10
ready_timeout=120
image="${COMPUTER_IMAGE:-lazyboy/computer:local}"
name="lazyboy-cua-smoke"
docker_mode=0
usage() {
echo "usage: $0 [--docker] [--repeat N] [--image NAME]" >&2
exit 2
}
while [[ $# -gt 0 ]]; do
case "$1" in
--docker) docker_mode=1; shift ;;
--repeat) repeat="${2:-}"; shift 2 ;;
--image) image="${2:-}"; shift 2 ;;
--ready-timeout) ready_timeout="${2:-}"; shift 2 ;;
-h|--help) usage ;;
*) usage ;;
esac
done
if [[ "$docker_mode" -eq 0 ]] && [[ -x /usr/local/bin/lazyboy-cua-smoke ]] && [[ -S /tmp/lazyboy/cua.sock || -f /tmp/lazyboy/ready ]]; then
exec /usr/local/bin/lazyboy-cua-smoke --repeat "$repeat" --ready-timeout "$ready_timeout"
fi
if [[ "$docker_mode" -eq 0 ]]; then
echo "not inside a LazyBoy desktop; passing --docker to run the computer image" >&2
docker_mode=1
fi
if ! command -v docker >/dev/null 2>&1; then
echo "docker is required" >&2
exit 1
fi
if ! docker image inspect "$image" >/dev/null 2>&1; then
echo "building $image (first desktop image build is slow)" >&2
root="$(cd "$(dirname "$0")/.." && pwd)"
docker build -f "$root/image/computer/Dockerfile" -t "$image" "$root"
fi
cleanup() {
docker rm -f "$name" >/dev/null 2>&1 || true
}
trap cleanup EXIT
cleanup
docker run -d --name "$name" --shm-size=512m \
-e DISPLAY=:1 \
"$image" >/dev/null
echo "waiting for desktop + cua-driver in $name"
for _ in $(seq 1 "$ready_timeout"); do
if docker exec -u 1000:1000 "$name" test -f /tmp/lazyboy/ready \
&& docker exec -u 1000:1000 "$name" test -S /tmp/lazyboy/cua.sock; then
break
fi
if ! docker inspect -f '{{.State.Running}}' "$name" 2>/dev/null | grep -q true; then
echo "computer container exited" >&2
docker logs "$name" >&2 || true
exit 1
fi
sleep 1
done
if ! docker exec -u 1000:1000 "$name" test -f /tmp/lazyboy/ready; then
echo "desktop did not become ready" >&2
docker exec -u 1000:1000 "$name" sh -c 'ls -la /tmp/lazyboy; tail -n 80 /tmp/lazyboy/*.log 2>/dev/null' >&2 || true
exit 1
fi
echo "running $repeat Cua smoke iterations"
set +e
docker exec -u 1000:1000 "$name" /usr/local/bin/lazyboy-cua-smoke --repeat "$repeat" --ready-timeout "$ready_timeout"
code=$?
set -e
out="${CUA_SMOKE_OUT:-/tmp/lazyboy-cua-smoke-last}"
mkdir -p "$out"
docker cp "$name:/tmp/lazyboy/cua-smoke-report/." "$out/" 2>/dev/null || true
docker exec -u 1000:1000 "$name" sh -c 'tail -n 80 /tmp/lazyboy/screen-1-cua.log 2>/dev/null || true' \
>"$out/cua-driver.log" || true
echo "smoke artifacts copied to $out"
exit "$code"