daniel.w
/
lazyBoy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat/cua-driver-poc #7

Merged
daniel.w merged 9 commits from feat/cua-driver-poc into main 2026-09-08 16:57:24 +00:00
Conversation 0 Commits 9 Files Changed 159 +2143 -3721
65 changed files with 2143 additions and 3721 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 17926f1092 - Show all commits

5
.env.example
View File

@ -22,9 +22,8 @@ LAZYBOY_COMPUTER_PIDS=2048
# Only affects the Agent desktop container. Disabled by default.
LAZYBOY_COMPUTER_SUDO=false
# Computer-control backend inside each desktop container. Rebuild/recreate
# computers after changing. `cua` is the opt-in Cua Driver backend; `legacy` (default) is
# CDP/AT-SPI/xdotool rollback.
LAZYBOY_COMPUTER_DRIVER=legacy
# computers after upgrading. Cua is the only supported computer controller.
LAZYBOY_COMPUTER_DRIVER=cua
# Linux only (optional): point this at the host's LXCFS root to make htop/free
# report the per-Agent cgroup quota. Leave the default empty directory on macOS.
LAZYBOY_LXCFS_ROOT=./data/lxcfs

7
apps/web/src/App.tsx
View File

File diff suppressed because one or more lines are too long

4
apps/web/src/chat.css
View File

@ -73,3 +73,7 @@
.slash-suggestions button[aria-selected="true"]{background:rgba(255,255,255,.08)}
.slash-suggestions small{flex-shrink:0;color:var(--muted);font-size:11px}
.message{padding-bottom:21px}
.message-time{position:absolute;bottom:0;left:0;font-size:11px;line-height:17px;color:var(--muted);font-variant-numeric:tabular-nums}
.message.user .message-time{left:auto;right:28px}

75
crates/api/src/computer.rs
View File

@ -111,6 +111,71 @@ pub struct BoundScreen {
pub gui_block: Option<String>,
}
/// Refresh presentation on an existing running screen without opening apps,
/// changing the Cua session, or booting a stopped computer.
pub async fn refresh_cursor_color(
state: &AppState,
actor: &Actor,
bot_id: &str,
) -> Result<(), String> {
let Some(bot) = state
.db
.get_bot(actor, bot_id)
.await
.map_err(|e| e.to_string())?
else {
return Ok(());
};
let Some(computer_id) = bot.computer_id else {
return Ok(());
};
let Some(computer) = state
.db
.get_computer(&computer_id)
.await
.map_err(|e| e.to_string())?
else {
return Ok(());
};
if computer.state != "running" {
return Ok(());
}
let Some(target) = computer_ref(&computer) else {
return Ok(());
};
let Some(screen) = state
.db
.get_screen(&computer_id, bot_id)
.await
.map_err(|e| e.to_string())?
else {
return Ok(());
};
let result = state
.sandbox
.execute(
&target,
CommandRequest {
argv: vec![
"/usr/local/bin/lazyboy-screen".into(),
"color".into(),
screen.slot.to_string(),
bot.avatar_color,
],
cwd: None,
timeout_ms: Some(5_000),
stdin: None,
},
&adapter_context_for(actor, bot_id, "cursor-color", Some(&screen), None),
)
.await
.map_err(|e| e.to_string())?;
if result.code != 0 {
return Err("cursor color refresh failed".into());
}
Ok(())
}
pub async fn ensure_bot_screen(
state: &AppState,
actor: &Actor,
@ -210,10 +275,17 @@ pub async fn ensure_bot_screen(
}
};
let ctx = adapter_context_for(actor, bot_id, "screen", Some(&row), run_id);
let bot = state
.db
.get_bot(actor, bot_id)
.await
.map_err(|error| error.to_string())?;
let request = EnsureScreenRequest {
slot: row.slot as u32,
profile_path: row.profile_path.clone(),
bot_id: bot_id.to_string(),
bot_name: bot.as_ref().map(|bot| bot.name.clone()).unwrap_or_default(),
bot_color: bot.map(|bot| bot.avatar_color).unwrap_or_default(),
};
let mut last_error = None;
for attempt in 0..8 {
@ -263,6 +335,7 @@ async fn restore_computer_screens(
continue;
}
let ctx = adapter_context_for(actor, &screen.bot_id, "screen", Some(&screen), None);
let bot = state.db.get_bot(actor, &screen.bot_id).await.ok().flatten();
let _ = state
.sandbox
.ensure_screen(
@ -271,6 +344,8 @@ async fn restore_computer_screens(
slot: screen.slot as u32,
profile_path: screen.profile_path.clone(),
bot_id: screen.bot_id.clone(),
bot_name: bot.as_ref().map(|bot| bot.name.clone()).unwrap_or_default(),
bot_color: bot.map(|bot| bot.avatar_color).unwrap_or_default(),
},
&ctx,
)

48
crates/api/src/routes.rs
View File

@ -314,6 +314,11 @@ async fn update_bot(
Json(json!({"message":"bot not found"})),
));
}
// The saved setting remains authoritative if the desktop is disconnected;
// ensure/restore will apply it again before the next run.
if let Err(error) = computer::refresh_cursor_color(&state, &actor, &id).await {
tracing::warn!(bot_id = %id, %error, "could not refresh cursor color");
}
Ok(Json(json!({"ok":true})))
}
@ -732,40 +737,15 @@ async fn input(
return Err(StatusCode::CONFLICT);
}
let computer_ref = computer::computer_ref(&computer).ok_or(StatusCode::BAD_REQUEST)?;
if body.kind == "clipboard" || body.kind == "copy" {
let text = body.text.unwrap_or_default();
if text.len() > 1024 * 1024 {
return Err(StatusCode::PAYLOAD_TOO_LARGE);
}
let context =
computer::adapter_context_for(&actor, &id, "clipboard", screen.as_ref(), None);
let mut argv =
lazyboy_control::paste_command_on(context.display.as_deref().unwrap_or(":1"));
if body.kind == "copy" {
argv.push("copy".into());
}
let result = state
.sandbox
.execute(
&computer_ref,
lazyboy_control::CommandRequest {
argv,
cwd: None,
timeout_ms: Some(10_000),
stdin: Some(text),
},
&context,
)
.await
.map_err(|_| StatusCode::BAD_GATEWAY)?;
if result.code != 0 {
return Err(StatusCode::BAD_GATEWAY);
}
return Ok(Json(
json!({"ok":true,"text":if body.kind=="copy" {Some(result.stdout)} else {None}}),
));
if body
.text
.as_ref()
.is_some_and(|text| text.len() > 1024 * 1024)
{
return Err(StatusCode::PAYLOAD_TOO_LARGE);
}
let action = match body.kind.as_str() {
"copy" => lazyboy_contracts::ComputerAction::CopySelection,
"key" => lazyboy_contracts::ComputerAction::Key {
key: body.key.unwrap_or_default(),
modifiers: None,
@ -780,7 +760,7 @@ async fn input(
button: Some(lazyboy_contracts::PointerButton::Left),
},
};
state
let result = state
.sandbox
.act(
&computer_ref,
@ -795,7 +775,7 @@ async fn input(
)
.await
.map_err(|_| StatusCode::BAD_GATEWAY)?;
Ok(Json(json!({ "ok": true })))
Ok(Json(json!({ "ok": true, "text": result.clipboard_text })))
}
fn _mode(mode: ComputerMode) {

14
crates/api/src/runs.rs
View File

@ -39,13 +39,13 @@ const SYSTEM: &str = "You are this bot's assistant. You have a Linux desktop you
Reply in text — no tools — for greetings, small talk, questions you can answer from knowledge, planning, or explaining. Do not call computer_observe, computer_act, browser, launch_app, open_path, wait, list_files, or shell just to check the screen or because a desktop exists. A hello does not need a screenshot or a file listing.
Use tools only when the user wants something done on the computer: open a site, click through a UI, run a command, read/write workspace files, or follow a taught skill. Route directly by task:
1) Website, video, search, email, or anything in Chromium: use browser first. Navigate directly, then snapshot/click/type/press by element id. Do not use shell/curl or pixel clicks to inspect a web page.
1) Website, video, search, email, or anything in Chromium: use browser first. Navigate directly, then snapshot/click/type/press by element id. Do not use shell/curl to inspect a web page. For canvas or controls the browser tool cannot operate, call computer_observe and use Cua computer_act coordinates from that fresh screenshot.
2) Workspace files or commands: use list_files, read_file, write_file, or shell.
3) Connected services: use an MCP tool when it directly matches the task.
4) Opening a local file or non-browser app: use open_path or launch_app.
5) Native GUI with no DOM (dialogs, file manager, XFCE): use computer_act by element id. Those ids are AT-SPI controls, not window boxes.
The shell is one real terminal that stays open between calls: same directory, same exports, same background jobs. cd where the work is and stay there. A long-running job (server, build, download) comes back as status=running and keeps going — read it with log_lines, stop it with keys \"C-c\", and never type a second command into a terminal that is still busy.
The shell is a visible Cua-controlled terminal on the shared VNC screen. The same session keeps its directory, exports and background jobs. Results are screenshots, not hidden stdout: inspect the prompt to decide whether a command finished. A timeout does not stop the job. Omit command to inspect it again, use keys \"C-c\" to interrupt, and never type a second command while busy. File tools also work through this visible terminal; read_file supports start_line and lines, and you can scroll to inspect longer output. These computer tools require a vision model.
When you ARE using the desktop: the human can interact with the same live screen while you work; this does not pause your task. Prefer browser/native element actions over moving the shared pointer. If the screen changes unexpectedly, observe again and continue from the current state; do not undo human changes or replay an uncertain click. Request human assistance only when the task needs it. Only the latest screenshot you received is current; they may have interacted since. Call computer_observe before coordinate clicks, after navigation, when the outcome is uncertain, and before describing what is on screen. Never guess the screen state from files, history or memory. Never kill or restart the browser, display, or desktop processes; if the browser tool reports it is unavailable, use computer_observe / computer_act on the existing window instead.
@ -2837,7 +2837,11 @@ fn tool_needs_sandbox(name: &str) -> bool {
fn tool_needs_gui(name: &str) -> bool {
matches!(
name,
"computer_observe"
"shell"
| "list_files"
| "read_file"
| "write_file"
| "computer_observe"
| "computer_act"
| "browser"
| "connection_check"
@ -3260,8 +3264,8 @@ mod tests {
assert!(tool_needs_sandbox("shell"));
assert!(tool_needs_gui("computer_observe"));
assert!(tool_needs_gui("browser"));
assert!(!tool_needs_gui("shell"));
assert!(!tool_needs_gui("list_files"));
assert!(tool_needs_gui("shell"));
assert!(tool_needs_gui("list_files"));
}
#[test]

35
crates/api/src/skills.rs
View File

@ -1,14 +1,10 @@
//! Skills taught by demonstration.
//!
//! Flow: the human presses "teach it a task", takes control of the bot's
//! desktop and does the task once. Meanwhile a CDP recorder inside the
//! desktop logs *semantic* browser events (which control was clicked, what
//! text went into which field, which URL loaded) and a poller keeps window
//! titles plus a few keyframes. When the human stops, a model distils the
//! trace into an intent-level playbook (goal, inputs, steps described by
//! meaning, how to verify). Later runs get the playbook and execute it with the
//! ordinary tools, locating controls on the live screen instead of replaying
//! coordinates.
//! The human demonstrates on the shared desktop. Cua observations supply
//! window changes and keyframes; Cua trajectory recording adds actions invoked
//! through the driver. Trajectories do not record raw human VNC input. A model
//! distils the available visual evidence into an intent-level playbook, which
//! later runs execute using controls located on the current screen.
use std::time::Duration;
@ -304,7 +300,7 @@ async fn start_skill(
&skill_id,
&bot_id,
&format!(
"開始學習：{goal}\n畫面交給你了，請直接在上面示範一次。我會記錄你點了哪些控制項、輸入了什麼、去了哪些頁面（密碼欄位不會記錄）。做完請按「完成示範」。"
"開始學習：{goal}\n畫面交給你了，請直接在上面示範一次。我會擷取示範中的畫面與視窗變化，整理成可供你確認的流程草稿。每個步驟完成後請稍停一下，做完請按「完成示範」。"
),
)
.await;
@ -1051,10 +1047,17 @@ async fn finalize(
.await;
if let Some(thread_id) = &row.thread_id {
let mut body = format!("我學會了「{name}」。\n{}", summarize_playbook(&playbook));
if error.is_some() {
body.push_str("\n\n（模型整理失敗，這是依事件直接列出的版本，建議先修改再儲存。）");
}
let mut body = if error.is_some() {
format!(
"已保留「{name}」的示範，但模型整理失敗，流程草稿尚未完成。請補齊並確認步驟後再儲存。\n{}",
summarize_playbook(&playbook)
)
} else {
format!(
"已整理「{name}」的流程草稿，請先確認內容。\n{}",
summarize_playbook(&playbook)
)
};
body.push_str("\n\n確認名稱後按「儲存」，之後跟我說「執行");
body.push_str(&name);
body.push_str("」就會照這個流程做；或先「試跑」看看。");
@ -1237,9 +1240,9 @@ fn pick_frames(frames: &[Value]) -> Vec<Value> {
picked
}
const DISTILL_SYSTEM: &str = "You turn a human's one-time screen demonstration into a reusable skill for a computer-use agent that controls the same Linux desktop (Chromium via a DOM snapshot/click/type tool, plus screenshots and xdotool for native windows).
const DISTILL_SYSTEM: &str = "You turn a human's one-time screen demonstration into a reusable skill for a computer-use agent that controls the same Linux desktop (Cua browser and native element tools, plus shared-desktop screenshots).
You receive: the human's stated goal, a timeline of what they did (semantic browser events: which control was clicked by its label/text, what text was typed into which field, which URLs loaded, active window titles) and a few screenshots taken along the way. The trace is noisy: ignore mis-clicks, corrections, tab switches and anything unrelated to the goal.
You receive the human's stated goal, window changes and screenshots captured during the demonstration. Driver-invoked actions may also appear, but raw human clicks and keystrokes are not recorded as semantic events. Infer steps only when the visual evidence supports them; mark missing or ambiguous steps for user review instead of inventing an action. Ignore corrections, tab switches and anything unrelated to the goal.
Produce a playbook that captures INTENT and PROCESS, never pixel positions:
- Describe each step by what it achieves and which control to use, named by its visible label/role/page (e.g. \"在 Wikipedia 首頁的搜尋框輸入 <主題> 並按 Enter\"), so the agent can find it on a slightly different layout.

927
crates/api/src/tools.rs
View File

File diff suppressed because it is too large Load Diff

7
crates/contracts/src/action.rs
View File

@ -44,7 +44,7 @@ pub enum ComputerAction {
#[serde(default)]
button: Option<PointerButton>,
},
/// Semantic target (DOM selector or AT-SPI path). Execute via CDP/a11y, not xdotool.
/// Snapshot-scoped semantic target resolved by Cua.
Ref {
verb: RefVerb,
target: String,
@ -53,6 +53,7 @@ pub enum ComputerAction {
#[serde(default)]
text: Option<String>,
},
CopySelection,
Clipboard {
text: String,
},
@ -103,7 +104,7 @@ pub struct UiElement {
pub y: u32,
pub w: u32,
pub h: u32,
/// CSS selector (DOM) or AT-SPI path (a11y). Native windows leave this empty.
/// Opaque Cua browser or native element reference. Windows leave this empty.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// "dom" for page controls, "a11y" for AT-SPI widgets, "window" for native windows.
@ -137,7 +138,7 @@ impl UiElement {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ComputerObservation {
/// The controller already populated native semantics; skip legacy enrichment.
/// The controller already populated native semantics; skip duplicate enrichment.
#[serde(default)]
pub native_observation_complete: bool,
pub frame_id: String,

353
crates/control/src/a11y.py
View File

@ -1,353 +0,0 @@
import json, os, sys, time
def fail(msg):
print(json.dumps({"ok": False, "error": msg, "elements": []}))
sys.exit(0)
def load_session(display):
display = display or os.environ.get("DISPLAY") or ":1"
if not display.startswith(":"):
display = ":" + display
os.environ["DISPLAY"] = display
number = display.lstrip(":")
dbus_file = "/tmp/lazyboy/screen-%s.dbus" % number
runtime_file = "/tmp/lazyboy/screen-%s.runtime" % number
try:
addr = open(dbus_file).read().strip()
if addr:
os.environ["DBUS_SESSION_BUS_ADDRESS"] = addr
except Exception:
pass
try:
runtime = open(runtime_file).read().strip()
if runtime:
os.environ["XDG_RUNTIME_DIR"] = runtime
except Exception:
if number == "1":
candidate = "/tmp/xfce-home/runtime"
else:
candidate = "/tmp/xfce-home-%s/runtime" % number
if os.path.isdir(candidate):
os.environ["XDG_RUNTIME_DIR"] = candidate
def atspi():
import gi
gi.require_version("Atspi", "2.0")
from gi.repository import Atspi
try:
Atspi.init()
except Exception:
pass
try:
Atspi.set_timeout(200, 200)
except Exception:
pass
return Atspi
INTERACTIVE = {
"push button", "toggle button", "check box", "radio button",
"combo box", "text", "password text", "menu item", "check menu item",
"radio menu item", "tab", "page tab", "slider", "spin button",
"link", "tree item", "entry", "password", "button", "menu",
"list item", "column header", "toggle",
}
BROWSER_APPS = ("chromium", "chrome", "google-chrome", "chromium-browser")
def is_browser_name(name):
n = (name or "").lower()
return any(token in n for token in BROWSER_APPS)
def child_count(acc):
try:
return int(acc.get_child_count())
except Exception:
return 0
def child_at(acc, i):
try:
return acc.get_child_at_index(i)
except Exception:
return None
def role_name(acc):
try:
return (acc.get_role_name() or "").lower()
except Exception:
return ""
def acc_name(acc):
try:
text = (acc.get_name() or "").strip()
if text:
return text
except Exception:
pass
try:
return (acc.get_description() or "").strip()
except Exception:
return ""
def state_names(Atspi, acc):
out = []
try:
ss = acc.get_state_set()
except Exception:
return out
for name in ("showing", "visible", "enabled", "sensitive", "checked",
"selected", "focused", "editable", "defunct", "expandable",
"expanded"):
try:
st = getattr(Atspi.StateType, name.upper())
if ss.contains(st):
out.append(name)
except Exception:
pass
return out
def extents(Atspi, acc):
try:
ext = acc.get_extents(Atspi.CoordType.SCREEN)
return int(ext.x), int(ext.y), int(ext.width), int(ext.height)
except Exception:
pass
try:
comp = acc.get_component_iface()
if comp is None:
return None
ext = comp.get_extents(Atspi.CoordType.SCREEN)
return int(ext.x), int(ext.y), int(ext.width), int(ext.height)
except Exception:
return None
def action_iface(acc):
for getter in ("get_action_iface", "queryAction", "get_action"):
fn = getattr(acc, getter, None)
if not fn:
continue
try:
iface = fn()
if iface is not None:
return iface
except Exception:
pass
if hasattr(acc, "get_n_actions") and hasattr(acc, "do_action"):
return acc
return None
def text_ifaces(acc):
edit = None
text = None
for getter in ("get_editable_text_iface", "queryEditableText"):
fn = getattr(acc, getter, None)
if not fn:
continue
try:
edit = fn()
if edit is not None:
break
except Exception:
pass
for getter in ("get_text_iface", "queryText"):
fn = getattr(acc, getter, None)
if not fn:
continue
try:
text = fn()
if text is not None:
break
except Exception:
pass
if edit is None and hasattr(acc, "insert_text"):
edit = acc
if text is None and hasattr(acc, "get_character_count"):
text = acc
return edit, text
def resolve(Atspi, path):
desktop = Atspi.get_desktop(0)
node = desktop
for part in str(path).split("/"):
if part == "":
continue
node = child_at(node, int(part))
if node is None:
return None
return node
def grab_focus(acc):
for getter in ("grab_focus",):
fn = getattr(acc, getter, None)
if fn:
try:
fn()
return True
except Exception:
pass
try:
comp = acc.get_component_iface()
if comp is not None:
comp.grab_focus()
return True
except Exception:
pass
return False
def do_click(acc):
action = action_iface(acc)
if action is None:
return grab_focus(acc) and False
try:
n = int(action.get_n_actions())
except Exception:
n = 0
idx = 0
prefer = ("click", "press", "activate", "jump", "open", "toggle", "select")
for i in range(n):
try:
name = (action.get_action_name(i) or "").lower()
except Exception:
name = ""
if name in prefer:
idx = i
break
if n <= 0:
return False
try:
return bool(action.do_action(idx))
except Exception:
return False
def set_text(acc, value):
grab_focus(acc)
edit, text = text_ifaces(acc)
if edit is None:
return False
n = 0
if text is not None:
try:
n = int(text.get_character_count())
except Exception:
n = 0
try:
edit.delete_text(0, n)
except Exception:
pass
try:
edit.insert_text(0, value, len(value))
return True
except Exception:
return False
def snapshot(Atspi, include_browser):
deadline = time.time() + 2.8
desktop = Atspi.get_desktop(0)
found = []
visited = 0
apps = child_count(desktop)
for app_i in range(apps):
if time.time() > deadline or len(found) >= 50:
break
app = child_at(desktop, app_i)
if app is None:
continue
app_label = acc_name(app) or role_name(app)
if not include_browser and is_browser_name(app_label):
continue
stack = [(app, str(app_i), 0)]
while stack:
if time.time() > deadline or len(found) >= 50 or visited > 400:
break
acc, path, depth = stack.pop()
visited += 1
states = state_names(Atspi, acc)
if "defunct" in states:
continue
role = role_name(acc)
name = acc_name(acc)
showing = ("showing" in states) or ("visible" in states) or not states
if role in INTERACTIVE and showing and name:
box = extents(Atspi, acc)
if box and box[2] >= 2 and box[3] >= 2:
x, y, w, h = box
if x + w > 0 and y + h > 0:
title = "%s %s" % (role, name.replace("\n", " ").strip())
if "checked" in states:
title += " (checked)"
if "expanded" in states:
title += " (expanded)"
if "enabled" in states and "sensitive" in states:
pass
elif states and "enabled" not in states:
title += " [disabled]"
title = title[:80]
found.append({
"id": len(found) + 1,
"title": title,
"role": role,
"kind": "a11y",
"selector": path,
"x": max(0, x),
"y": max(0, y),
"w": w,
"h": h,
})
if depth >= 12:
continue
n = child_count(acc)
# Walk children in reverse so index 0 is processed first with pop().
for i in range(n - 1, -1, -1):
child = child_at(acc, i)
if child is None:
continue
stack.append((child, "%s/%d" % (path, i), depth + 1))
return found
def main():
req = json.loads(sys.argv[1]) if len(sys.argv) > 1 else {}
action = req.get("action") or "snapshot"
display = req.get("display") or ":1"
load_session(display)
try:
Atspi = atspi()
except Exception as error:
fail("atspi unavailable: %s" % error)
return
if action == "snapshot":
include_browser = bool(req.get("includeBrowser"))
try:
elements = snapshot(Atspi, include_browser)
except Exception as error:
fail("atspi snapshot failed: %s" % error)
return
print(json.dumps({"ok": True, "elements": elements}))
return
selector = req.get("selector") or ""
if not selector:
fail("a11y action needs selector")
return
try:
acc = resolve(Atspi, selector)
except Exception as error:
fail("a11y resolve failed: %s" % error)
return
if acc is None:
fail("a11y element gone")
return
ok = False
if action == "click":
ok = do_click(acc)
elif action == "type":
ok = set_text(acc, req.get("text") or "")
elif action == "focus":
ok = grab_focus(acc)
else:
fail("unsupported a11y action")
return
print(json.dumps({"ok": bool(ok), "error": None if ok else "a11y %s failed" % action}))
if __name__ == "__main__":
try:
main()
except Exception as error:
fail(str(error))

83
crates/control/src/a11y.rs
View File

@ -1,55 +1,5 @@
use crate::is_browser_title;
use lazyboy_contracts::UiElement;
use serde_json::{Value, json};
use crate::normalize_display;
use crate::x11::{is_browser_title, parse_ui_elements};
const A11Y_PY: &str = include_str!("a11y.py");
#[derive(Debug, Clone, PartialEq, Default)]
pub struct A11yPage {
pub ok: bool,
pub error: Option<String>,
pub elements: Vec<UiElement>,
}
pub fn a11y_command_on(display: &str, request: &Value) -> Vec<String> {
let mut body = request.clone();
if let Some(object) = body.as_object_mut() {
object
.entry("display")
.or_insert_with(|| json!(normalize_display(display)));
}
vec![
"env".into(),
format!("DISPLAY={}", normalize_display(display)),
"python3".into(),
"-c".into(),
A11Y_PY.into(),
body.to_string(),
]
}
pub fn parse_a11y_page(raw: &str) -> A11yPage {
let value: Value = serde_json::from_str(raw.trim()).unwrap_or(Value::Null);
let ok = value.get("ok").and_then(Value::as_bool) == Some(true);
A11yPage {
ok,
error: if ok {
None
} else {
value
.get("error")
.and_then(Value::as_str)
.map(str::to_string)
.or_else(|| Some("atspi unavailable".into()))
},
elements: value
.get("elements")
.map(|items| parse_ui_elements(&items.to_string()))
.unwrap_or_default(),
}
}
/// Merge DOM (keep ids), then a11y, then native windows that are not already
/// covered by a control tree. Chromium's whole window is dropped when the
@ -129,30 +79,6 @@ mod tests {
}
}
#[test]
fn command_injects_display_and_script() {
let argv = a11y_command_on(":2", &json!({"action": "snapshot"}));
assert!(argv.contains(&"DISPLAY=:2".into()));
assert!(argv.iter().any(|item| item.contains("python3")));
assert!(argv.last().unwrap().contains("\"display\":\":2\""));
assert!(
argv.iter()
.any(|item| item.contains("Atspi") || item.contains("atspi"))
);
}
#[test]
fn parses_snapshot_elements() {
let page = parse_a11y_page(
r#"{"ok":true,"elements":[{"id":1,"title":"push button Open","role":"push button","kind":"a11y","selector":"0/2/1","x":10,"y":20,"w":80,"h":24}]}"#,
);
assert!(page.ok);
assert_eq!(page.elements.len(), 1);
assert_eq!(page.elements[0].kind.as_deref(), Some("a11y"));
assert_eq!(page.elements[0].selector.as_deref(), Some("0/2/1"));
assert_eq!(page.elements[0].role.as_deref(), Some("push button"));
}
#[test]
fn merge_keeps_dom_ids_and_replaces_covered_windows() {
let windows = vec![
@ -189,11 +115,4 @@ mod tests {
assert_eq!(merged.len(), 1);
assert_eq!(merged[0].title, "Thunar");
}
#[test]
fn unavailable_tree_is_not_ok() {
let page = parse_a11y_page(r#"{"ok":false,"error":"atspi unavailable"}"#);
assert!(!page.ok);
assert_eq!(page.error.as_deref(), Some("atspi unavailable"));
}
}

100
crates/control/src/actions.rs
View File

@ -1,4 +1,3 @@
use crate::is_browser_title;
use lazyboy_contracts::{
ComputerAction, PointerButton, PointerType, RefVerb, ScrollDirection, UiElement,
};
@ -125,62 +124,6 @@ pub fn should_block_stale_click(miss_streak: u32, last: Option<&str>, next: Opti
miss_streak >= 2 && next.is_some() && next == last
}
/// When a CDP page snapshot is live, refuse pixel-clicking the Chromium window.
pub fn browser_gui_block(actions: &Value, elements: &[UiElement]) -> Option<String> {
if !elements
.iter()
.any(|element| element.kind.as_deref() == Some("dom"))
{
return None;
}
let items = actions.as_array()?;
for raw in items {
let Some(action) = raw.as_object() else {
continue;
};
let kind = action
.get("kind")
.and_then(Value::as_str)
.or_else(|| action.get("type").and_then(Value::as_str))
.unwrap_or("");
if !matches!(kind, "click" | "move" | "down" | "up" | "hover" | "drag") {
continue;
}
if let Some(id) = element_id(action.get("element")) {
match elements.iter().find(|element| u64::from(element.id) == id) {
Some(element) if element.has_ref() => continue,
Some(element)
if element.kind.as_deref() == Some("window")
&& is_browser_title(&element.title) =>
{
return Some(browser_block_message(elements));
}
_ => continue,
}
} else {
return Some(browser_block_message(elements));
}
}
None
}
fn browser_block_message(elements: &[UiElement]) -> String {
let known: Vec<String> = elements
.iter()
.filter(|element| element.kind.as_deref() == Some("dom"))
.take(12)
.map(|element| format!("[{}] {}", element.id, element.title))
.collect();
format!(
"Chromium is in front: use the browser tool (snapshot / click element N) instead of computer_act pixel clicks. Known page elements: {}",
if known.is_empty() {
"call browser snapshot first".to_string()
} else {
known.join(", ")
}
)
}
pub fn parse_computer_actions(value: &Value) -> Result<Vec<ComputerAction>, ActionError> {
let Value::Array(items) = value else {
return Err(ActionError::Empty);
@ -622,49 +565,6 @@ mod tests {
}
}
#[test]
fn pixel_clicks_are_blocked_when_dom_is_live() {
let elements = vec![UiElement {
id: 1,
title: "Submit".into(),
selector: Some("[data-lazyboy=\"1\"]".into()),
kind: Some("dom".into()),
x: 10,
y: 20,
w: 80,
h: 24,
..UiElement::default()
}];
let blocked = browser_gui_block(&json!([{"kind":"click","x":40,"y":80}]), &elements);
assert!(blocked.unwrap().contains("browser"));
assert!(browser_gui_block(&json!([{"kind":"click","element":1}]), &elements).is_none());
}
#[test]
fn chromium_window_clicks_are_blocked_when_dom_is_live() {
let elements = vec![
UiElement {
id: 1,
title: "Submit".into(),
selector: Some("[data-lazyboy=\"1\"]".into()),
kind: Some("dom".into()),
..UiElement::default()
},
UiElement {
id: 2,
title: "Chromium".into(),
kind: Some("window".into()),
x: 0,
y: 0,
w: 1280,
h: 800,
..UiElement::default()
},
];
let blocked = browser_gui_block(&json!([{"kind":"click","element":2}]), &elements);
assert!(blocked.unwrap().contains("browser"));
}
#[test]
fn stale_repeat_blocks_the_third_same_click() {
assert!(!should_block_stale_click(1, Some("e3"), Some("e3")));

40
crates/control/src/browser_page.rs Normal file
View File

@ -0,0 +1,40 @@
use lazyboy_contracts::UiElement;
use serde::{Deserialize, Serialize};
pub fn sanitize_skill_id(skill_id: &str) -> String {
let cleaned: String = skill_id
.chars()
.filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_')
.take(80)
.collect();
if cleaned.is_empty() {
"unknown".into()
} else {
cleaned
}
}
pub fn teach_trajectory_dir(skill_id: &str) -> String {
format!("/tmp/lazyboy/teach-{}", sanitize_skill_id(skill_id))
}
#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct BrowserPage {
pub ok: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
#[serde(default)]
pub url: String,
#[serde(default)]
pub title: String,
#[serde(default)]
pub text: String,
#[serde(default)]
pub restarted: bool,
/// Seconds the click waited for a disabled control to become enabled.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub waited_seconds: Option<f64>,
#[serde(default)]
pub elements: Vec<UiElement>,
}

673
crates/control/src/cdp.py
View File

@ -1,673 +0,0 @@
import json, os, sys, time, socket, base64, struct, subprocess, urllib.request
def fail(msg):
print(json.dumps({"ok": False, "error": msg}))
sys.exit(0)
def http_json(url, timeout=2):
try:
with urllib.request.urlopen(url, timeout=timeout) as r:
return json.loads(r.read().decode())
except Exception:
return None
class Ws:
"""Use a maintained RFC6455 transport (fragmentation, ping/pong, handshake)."""
def __init__(self, url):
import websocket
self.sock = websocket.create_connection(url, timeout=5, suppress_origin=True,
http_no_proxy=["127.0.0.1", "localhost"])
self.n = 0
def recv_json(self):
return json.loads(self.sock.recv())
def call(self, method, params=None):
self.n += 1
self.sock.send(json.dumps({"id": self.n, "method": method, "params": params or {}}))
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
self.sock.settimeout(max(.1, deadline-time.monotonic()))
obj = self.recv_json()
if obj.get("id") == self.n:
if "error" in obj:
raise RuntimeError(str(obj["error"]))
return obj.get("result") or {}
raise TimeoutError("CDP response deadline exceeded")
def close(self):
self.sock.close()
def probe(port):
return http_json("http://127.0.0.1:%s/json/version" % port) is not None
def profile_alive(profile):
if not profile:
return False
try:
out = subprocess.check_output(["pgrep", "-af", "chromium"], text=True, stderr=subprocess.DEVNULL)
except Exception:
return False
for line in out.splitlines():
if ("--user-data-dir=%s" % profile) in line and "--type=" not in line:
return True
return False
def active_port(profile):
# Chromium writes the DevTools port it actually bound here. Trust it over
# our expected port so we attach to the window the human already sees.
if not profile:
return None
try:
with open(os.path.join(profile, "DevToolsActivePort")) as f:
return int(f.readline().strip())
except Exception:
return None
def spawn_browser(display, profile, port):
env = os.environ.copy()
env["DISPLAY"] = display
if profile:
env["LAZYBOY_BROWSER_PROFILE"] = profile
subprocess.Popen(
["lazyboy-browser", "--remote-debugging-port=%s" % port],
env=env,
start_new_session=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
def connect(port):
tabs = http_json("http://127.0.0.1:%s/json/list" % port) or []
pages = [t for t in tabs if t.get("type") == "page" and t.get("webSocketDebuggerUrl")]
if not pages:
fail("no browser tab")
# /json/list is ordered by last activity, so pages[0] is the tab the human
# is looking at. Bring it to the front anyway so what the model reads and
# clicks is always the tab shown on the live screen.
pages.sort(key=lambda t: (t.get("url") or "").startswith("chrome://"), reverse=False)
page = pages[0]
if page.get("id"):
http_json("http://127.0.0.1:%s/json/activate/%s" % (port, page["id"]))
ws = Ws(page["webSocketDebuggerUrl"])
ws.call("Runtime.enable")
ws.call("Page.enable")
return ws
SNAP_JS = r"""
(() => {
const sels = 'a, button, input, textarea, select, summary, label, [role="button"], [role="link"], [role="textbox"], [role="checkbox"], [role="menuitem"], [contenteditable="true"]';
const chromeH = Math.max(0, (window.outerHeight || 0) - (window.innerHeight || 0));
const chromeW = Math.max(0, (window.outerWidth || 0) - (window.innerWidth || 0));
const sx0 = (window.screenX || 0) + Math.floor(chromeW / 2);
const sy0 = (window.screenY || 0) + chromeH;
document.querySelectorAll('[data-lazyboy]').forEach(el => el.removeAttribute('data-lazyboy'));
const generation = crypto.randomUUID();
const seen = new Set();
const inView = [];
const offView = [];
for (const el of document.querySelectorAll(sels)) {
const r = el.getBoundingClientRect();
if (r.width < 4 || r.height < 4) continue;
const st = getComputedStyle(el);
if (st.visibility === "hidden" || st.display === "none" || Number(st.opacity) === 0) continue;
const type = (el.getAttribute("type") || "").toLowerCase();
let text;
if (type === "password" || /password|secret|token|one-time-code/i.test([el.name, el.id, el.autocomplete].join(" "))) {
text = "[protected input]";
} else if (el.tagName === "INPUT" && (type === "radio" || type === "checkbox")) {
// Quiz answers: the value is usually "on"; the label next to it is what
// the model must read to pick the right option.
const owner = (el.labels && el.labels[0]) || el.closest("label") || el.parentElement;
const label = (owner && owner.innerText || el.getAttribute("aria-label") || el.value || "").replace(/\s+/g, " ").trim().slice(0, 70);
text = type + " " + label + (el.checked ? " (checked)" : "");
} else {
text = (el.innerText || el.value || el.getAttribute("aria-label") || el.getAttribute("placeholder") || el.getAttribute("name") || el.tagName)
.replace(/\s+/g, " ").trim().slice(0, 80);
}
if (!text.trim()) continue;
const key = [el.tagName, text, Math.round(r.x), Math.round(r.y)].join("|");
if (seen.has(key)) continue;
seen.add(key);
if (el.disabled || el.getAttribute("aria-disabled") === "true") text += " [disabled]";
const visible = !(r.bottom < 0 || r.right < 0 || r.top > innerHeight || r.left > innerWidth);
if (visible) {
inView.push({el, text, x: Math.max(0, Math.round(sx0 + r.x)), y: Math.max(0, Math.round(sy0 + r.y)), w: Math.round(r.width), h: Math.round(r.height)});
} else {
// Controls outside the viewport are still clickable by id: the click
// handler scrolls them into view. Zero size tells the desktop side not
// to paint or pixel-click them.
const where = r.top > innerHeight ? "below" : r.bottom < 0 ? "above" : "beside";
// Buttons and form controls (Next, Submit, radios) matter more than the
// hundredth body link, so they win the limited off-screen slots.
const link = el.tagName === "A" || el.getAttribute("role") === "link";
offView.push({el, text: text + " [" + where + " viewport]", x: 0, y: 0, w: 0, h: 0, rank: (link ? 1 : 0), dist: Math.abs(r.top > innerHeight ? r.top - innerHeight : r.bottom)});
}
}
offView.sort((a, b) => a.rank - b.rank || a.dist - b.dist);
const out = [];
let n = 1;
for (const item of inView.slice(0, 50).concat(offView.slice(0, 20))) {
item.el.setAttribute("data-lazyboy", generation + "-" + n);
out.push({
id: n,
title: item.text,
tag: item.el.tagName.toLowerCase(),
selector: '[data-lazyboy="' + generation + "-" + n + '"]',
kind: "dom",
x: item.x,
y: item.y,
w: item.w,
h: item.h
});
n += 1;
}
const body = (document.body && document.body.innerText || "").replace(/\s+/g, " ").trim().slice(0, 3000);
return {url: location.href, title: document.title || "", text: body, elements: out};
})()
"""
CLICK_JS = r"""
(sel) => {
const el = document.querySelector(sel);
if (!el) return {ok: false, error: "element gone"};
el.scrollIntoView({block: "center", inline: "nearest"});
const r = el.getBoundingClientRect();
const style = getComputedStyle(el);
const hit = document.elementFromPoint(r.x+r.width/2, r.y+r.height/2);
if (el.disabled || el.getAttribute("aria-disabled") === "true" || r.width <= 0 || r.height <= 0 || style.visibility === "hidden" || style.display === "none" || !hit || !(hit === el || el.contains(hit))) {
return {ok:false,error:"element is disabled, hidden, or covered; observe again"};
}
el.focus();
el.click();
const chromeH = Math.max(0, (window.outerHeight || 0) - (window.innerHeight || 0));
const chromeW = Math.max(0, (window.outerWidth || 0) - (window.innerWidth || 0));
const sx = (window.screenX || 0) + Math.floor(chromeW / 2) + r.x + r.width / 2;
const sy = (window.screenY || 0) + chromeH + r.y + r.height / 2;
return {ok: true, x: Math.round(sx), y: Math.round(sy)};
}
"""
STATE_JS = r"""
(sel) => {
const el = document.querySelector(sel);
if (!el) return {found: false};
const disabled = !!el.disabled || el.getAttribute("aria-disabled") === "true";
// Training sites explain the lock next to the button ("Please watch the
// video", a countdown); surface that text so the model can decide how
// long to wait.
let hint = "";
const near = el.parentElement && el.parentElement.parentElement;
if (disabled && near) hint = (near.innerText || "").replace(/\s+/g, " ").trim().slice(0, 120);
return {found: true, disabled, hint};
}
"""
# Pages often lock Next for a few seconds (stay timers) or until a video ends.
# A human just waits and clicks; do the same instead of making the model plan
# a wait/observe/click loop it tends to abandon.
CLICK_WAIT_MS = 45000
def wait_until_enabled(ws, sel, wait_ms=None):
budget = CLICK_WAIT_MS if wait_ms is None else max(0, min(int(wait_ms), 120000))
started = time.time()
while True:
state = evaluate(ws, STATE_JS, sel) or {}
if not state.get("found"):
return None
if not state.get("disabled"):
return time.time() - started
if (time.time() - started) * 1000 >= budget:
return None
time.sleep(0.5)
def evaluate(ws, expression, args=None):
params = {"expression": expression, "returnByValue": True, "awaitPromise": True}
if args is not None:
params = {
"expression": "(%s)(%s)" % (expression, json.dumps(args)),
"returnByValue": True,
"awaitPromise": True,
}
result = ws.call("Runtime.evaluate", params)
val = (result.get("result") or {}).get("value")
if result.get("exceptionDetails"):
raise RuntimeError(str(result["exceptionDetails"]))
return val
def wait_for_visual_update(ws):
# CDP input and DOM clicks can complete before Chromium commits the next
# painted frame. The caller captures X11 immediately after this process
# exits, so wait for two animation frames to keep that screenshot aligned
# with the framebuffer streamed by VNC.
try:
evaluate(ws, "new Promise(resolve => {setTimeout(resolve, 250); requestAnimationFrame(() => requestAnimationFrame(resolve));})")
except Exception:
pass
def snapshot(ws):
val = evaluate(ws, SNAP_JS) or {}
return {
"ok": True,
"action": "snapshot",
"url": val.get("url") or "",
"title": val.get("title") or "",
"text": val.get("text") or "",
"elements": val.get("elements") or [],
}
def pointer(display, x, y):
try:
subprocess.check_call(
["env", "DISPLAY=%s" % display, "xdotool", "mousemove", "--sync", "--", str(int(x)), str(int(y))],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
except Exception:
pass
KEYS = {
"Return": (13, "Enter", "Enter"),
"Enter": (13, "Enter", "Enter"),
"Tab": (9, "Tab", "Tab"),
"BackSpace": (8, "Backspace", "Backspace"),
"Backspace": (8, "Backspace", "Backspace"),
"Escape": (27, "Escape", "Escape"),
"Esc": (27, "Escape", "Escape"),
"Space": (32, " ", "Space"),
}
def press(ws, key):
spec = KEYS.get(key) or KEYS.get(key.title())
if spec:
code, name, key_id = spec
for typ in ("keyDown", "keyUp"):
ws.call("Input.dispatchKeyEvent", {
"type": typ,
"windowsVirtualKeyCode": code,
"key": name,
"code": key_id,
})
return
ws.call("Input.dispatchKeyEvent", {"type": "keyDown", "text": key[:1]})
ws.call("Input.dispatchKeyEvent", {"type": "keyUp", "text": key[:1]})
# Injected into every page while a human demonstrates a task. It reports what
# the person did in terms of page semantics (which control, what text, which
# URL) rather than pixels, so the distilled skill can generalise. Secrets are
# masked before they leave the page.
RECORD_JS = r"""
(() => {
if (window.__lbTeachInstalled) return;
window.__lbTeachInstalled = true;
const send = (ev) => { try { ev.at = Date.now(); ev.url = location.href; window.__lbTeach(JSON.stringify(ev)); } catch (e) {} };
const clean = (s) => (s || "").replace(/\s+/g, " ").trim().slice(0, 120);
const secretRe = /pass|pwd|secret|token|otp|cvv|card|pin\b/i;
const isSecret = (el) => !el ? false : (el.type === "password" || secretRe.test(el.name || "") || secretRe.test(el.id || "") || secretRe.test(el.autocomplete || "") || secretRe.test(el.getAttribute && el.getAttribute("aria-label") || ""));
const labelFor = (el) => {
if (!el) return "";
if (el.labels && el.labels.length) return clean(el.labels[0].innerText);
const id = el.id && document.querySelector('label[for="' + el.id + '"]');
if (id) return clean(id.innerText);
return clean(el.getAttribute("aria-label") || el.placeholder || el.title || el.name || "");
};
const describe = (el) => {
if (!el || el.nodeType !== 1) return null;
const tag = el.tagName.toLowerCase();
const d = { tag, role: el.getAttribute("role") || "", text: clean(el.innerText || el.value || el.alt || el.getAttribute("aria-label") || el.title || el.placeholder || ""), label: labelFor(el) };
if (el.id) d.id = el.id;
if (el.name) d.name = el.name;
if (tag === "a" && el.href) d.href = el.href.slice(0, 200);
if (tag === "input") d.type = el.type || "text";
return d;
};
const actionable = (node) => {
let el = node;
for (let i = 0; el && i < 6; i++) {
if (el.nodeType === 1) {
const t = el.tagName.toLowerCase();
if (["a","button","input","select","textarea","summary","label","option"].includes(t) || el.getAttribute("role") || el.onclick || el.getAttribute("tabindex") !== null || el.isContentEditable) return el;
}
el = el.parentNode;
}
return node && node.nodeType === 1 ? node : null;
};
send({ t: "page", title: document.title });
document.addEventListener("click", (e) => {
const el = actionable(e.target);
const d = describe(el);
if (d) send({ t: "click", el: d, x: Math.round(e.clientX), y: Math.round(e.clientY) });
}, true);
const pending = new Map();
const flush = (el) => {
pending.delete(el);
const d = describe(el);
if (!d) return;
let value = el.isContentEditable ? el.innerText : (el.value || "");
if (el.tagName === "SELECT" && el.selectedOptions && el.selectedOptions[0]) value = el.selectedOptions[0].text;
if (el.type === "checkbox" || el.type === "radio") value = el.checked ? "checked" : "unchecked";
send({ t: "input", el: d, value: isSecret(el) ? "[redacted]" : clean(value) });
};
document.addEventListener("input", (e) => {
const el = e.target;
if (!el || !(el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable)) return;
clearTimeout(pending.get(el));
pending.set(el, setTimeout(() => flush(el), 900));
}, true);
document.addEventListener("change", (e) => { const el = e.target; if (el && el.nodeType === 1) { clearTimeout(pending.get(el)); flush(el); } }, true);
document.addEventListener("keydown", (e) => {
const special = ["Enter","Escape","Tab"].includes(e.key) || e.ctrlKey || e.metaKey || e.altKey;
if (!special || e.key === "Control" || e.key === "Meta" || e.key === "Alt" || e.key === "Shift") return;
const el = document.activeElement;
if (el && pending.has(el)) { clearTimeout(pending.get(el)); flush(el); }
const combo = [e.ctrlKey ? "Ctrl" : "", e.metaKey ? "Meta" : "", e.altKey ? "Alt" : "", e.shiftKey ? "Shift" : "", e.key].filter(Boolean).join("+");
send({ t: "key", key: combo, el: describe(el) });
}, true);
document.addEventListener("submit", (e) => { const f = e.target; send({ t: "submit", form: { action: (f && f.action || "").slice(0, 200), name: f && (f.name || f.id) || "" } }); }, true);
let lastScroll = 0;
window.addEventListener("scroll", () => { const now = Date.now(); if (now - lastScroll > 2000) { lastScroll = now; send({ t: "scroll", y: Math.round(window.scrollY) }); } }, true);
})()
"""
class Recorder:
"""Browser-level CDP session with flattened page sessions. Events from
every tab are appended to a JSONL file until the process is killed."""
def __init__(self, port, out):
info = http_json("http://127.0.0.1:%s/json/version" % port) or {}
url = info.get("webSocketDebuggerUrl")
if not url:
raise RuntimeError("browser has no DevTools endpoint")
self.ws = Ws(url)
self.out = open(out, "a", buffering=1)
self.sessions = {}
self.pending = []
self.last = (None, 0)
def emit(self, ev):
ev.setdefault("at", int(time.time() * 1000))
# Two sessions on one page (auto-attach + explicit) deliver the same
# binding call twice; a key repeat is never that fast either.
key = json.dumps({k: v for k, v in ev.items() if k != "at"}, sort_keys=True)
if key == self.last[0] and ev["at"] - self.last[1] < 800:
return
self.last = (key, ev["at"])
self.out.write(json.dumps(ev, ensure_ascii=False) + "\n")
def call(self, method, params=None, session=None):
self.ws.n += 1
msg = {"id": self.ws.n, "method": method}
if params:
msg["params"] = params
if session:
msg["sessionId"] = session
self.ws.sock.send(json.dumps(msg))
while True:
obj = self.ws.recv_json()
if obj.get("id") == self.ws.n:
if "error" in obj:
raise RuntimeError(str(obj["error"]))
return obj.get("result") or {}
self.pending.append(obj)
def attach(self, session, target):
if target.get("type") != "page" or not session:
return
target_id = target.get("targetId")
if session in self.sessions:
return
if target_id in self.sessions.values():
try:
self.call("Target.detachFromTarget", {"sessionId": session})
except Exception:
pass
return
self.sessions[session] = target_id
for method, params in (
("Runtime.enable", None),
("Page.enable", None),
("Runtime.addBinding", {"name": "__lbTeach"}),
("Page.addScriptToEvaluateOnNewDocument", {"source": RECORD_JS}),
("Runtime.evaluate", {"expression": RECORD_JS}),
):
try:
self.call(method, params, session)
except Exception:
pass
def handle(self, obj):
method = obj.get("method")
params = obj.get("params") or {}
if method == "Target.attachedToTarget":
self.attach(params.get("sessionId"), params.get("targetInfo") or {})
elif method == "Target.detachedFromTarget":
self.sessions.pop(params.get("sessionId"), None)
elif method == "Runtime.bindingCalled" and params.get("name") == "__lbTeach":
try:
self.emit(json.loads(params.get("payload") or "{}"))
except Exception:
pass
elif method == "Page.frameNavigated":
frame = params.get("frame") or {}
if not frame.get("parentId"):
self.emit({"t": "navigate", "url": frame.get("url") or ""})
elif method == "Target.targetInfoChanged":
info = params.get("targetInfo") or {}
if info.get("type") == "page" and info.get("title"):
self.emit({"t": "title", "url": info.get("url") or "", "title": info.get("title")})
def run(self):
self.ws.sock.settimeout(None)
self.call("Target.setDiscoverTargets", {"discover": True})
self.call("Target.setAutoAttach", {"autoAttach": True, "waitForDebuggerOnStart": False, "flatten": True})
for target in (self.call("Target.getTargets") or {}).get("targetInfos", []):
if target.get("type") == "page":
try:
result = self.call("Target.attachToTarget", {"targetId": target["targetId"], "flatten": True})
self.attach(result.get("sessionId"), target)
except Exception:
pass
self.emit({"t": "recorder", "state": "started"})
while True:
while self.pending:
self.handle(self.pending.pop(0))
self.handle(self.ws.recv_json())
FILL_LOGIN_JS = r"""
(creds) => {
if (!creds.expectedHost || location.protocol !== "https:" || location.hostname.toLowerCase() !== creds.expectedHost.toLowerCase()) {
return {ok: false, error: "saved login requires the exact configured HTTPS host"};
}
const user = creds.username || "";
const pass = creds.password || "";
const inputs = Array.from(document.querySelectorAll("input"));
const visible = (el) => {
const s = getComputedStyle(el);
const r = el.getBoundingClientRect();
return s.display !== "none" && s.visibility !== "hidden" && el.type !== "hidden" && r.width > 0 && r.height > 0;
};
const password = inputs.find((el) => el.type === "password" && visible(el) && !el.disabled);
if (!password) return {ok: false, error: "no password field on this page"};
const userish = /user|email|login|account|phone|id/i;
const username = inputs.find((el) => {
if (el === password || !visible(el) || el.disabled) return false;
const type = (el.type || "text").toLowerCase();
if (["email", "tel", "url"].includes(type)) return true;
if (type !== "text" && type !== "search") return false;
const blob = [el.name, el.id, el.placeholder, el.autocomplete, el.getAttribute("aria-label")].join(" ");
return userish.test(blob) || el === inputs[0];
});
function setValue(el, value) {
const proto = HTMLInputElement.prototype;
const desc = Object.getOwnPropertyDescriptor(proto, "value");
if (desc && desc.set) desc.set.call(el, value);
else el.value = value;
el.dispatchEvent(new Event("input", {bubbles: true}));
el.dispatchEvent(new Event("change", {bubbles: true}));
}
if (username) setValue(username, user);
setValue(password, pass);
return {ok: true, filledUsername: Boolean(username), submitted: false};
}
"""
def main():
raw = sys.argv[1] if len(sys.argv) > 1 and sys.argv[1].strip() else sys.stdin.read()
req = json.loads(raw)
action = req.get("action") or "snapshot"
display = req.get("display") or ":1"
profile = req.get("profile") or ""
port = int(req.get("port") or 9222)
ensure = bool(req.get("ensure"))
if action == "probe":
print(json.dumps({"ok": probe(port)}))
return
def bound_port():
if probe(port):
return port
bound = active_port(profile)
if bound and bound != port and probe(bound):
return bound
return None
restarted = False
ready_port = bound_port()
if ready_port is None and profile_alive(profile):
# The window may still be booting (lazyboy-screen just spawned it).
for _ in range(12):
time.sleep(0.25)
ready_port = bound_port()
if ready_port is not None:
break
if ready_port is not None:
port = ready_port
else:
if profile_alive(profile):
# Never kill the window the human is watching. Fall back to the
# screenshot tools, which see exactly what the live screen shows.
fail("browser is open but has no DevTools; use computer_observe/computer_act on it instead. Do not restart the browser.")
if not ensure:
fail("cdp unavailable")
spawn_browser(display, profile, port)
ready = False
for _ in range(24):
time.sleep(0.25)
if probe(port):
ready = True
break
if not ready:
fail("cdp unavailable")
restarted = True
if action == "ensure":
print(json.dumps({"ok": True, "restarted": restarted}))
return
if action == "record":
# Long-running: the API starts this detached and kills it on stop.
out = req.get("out") or "/tmp/lazyboy-teach.jsonl"
Recorder(port, out).run()
return
ws = connect(port)
try:
if action == "snapshot":
body = snapshot(ws)
body["restarted"] = restarted
print(json.dumps(body))
return
if action == "navigate":
url = req.get("url") or ""
if not url:
fail("url required")
ws.call("Page.navigate", {"url": url})
time.sleep(1.2)
body = snapshot(ws)
body["restarted"] = restarted
print(json.dumps(body))
return
if action == "click":
sel = req.get("selector") or ""
if not sel:
fail("selector required")
waited = wait_until_enabled(ws, sel, req.get("waitMs"))
if waited is None:
state = evaluate(ws, STATE_JS, sel) or {}
if not state.get("found"):
# Ids are renumbered whenever the page changes; hand back
# the fresh numbering so the model does not have to ask.
body = snapshot(ws)
body.update({"ok": False, "error": "element gone: the page changed and ids were renumbered. Use the fresh element list in this result."})
print(json.dumps(body))
return
fail("control %s is still disabled after waiting %ss (page says: %s). Use wait for longer if a video or timer must finish, then click again."
% (sel, int(req.get("waitMs") or CLICK_WAIT_MS) // 1000, state.get("hint") or "nothing"))
val = evaluate(ws, CLICK_JS, sel) or {}
if not val.get("ok"):
fail(val.get("error") or "click failed")
pointer(display, val.get("x") or 0, val.get("y") or 0)
wait_for_visual_update(ws)
out = snapshot(ws)
out.update({"action": "click", "selector": sel, "restarted": restarted})
if waited >= 1.0:
out["waitedSeconds"] = round(waited, 1)
print(json.dumps(out))
return
if action == "fill_login":
val = evaluate(ws, FILL_LOGIN_JS, {
"expectedHost": req.get("expectedHost") or "",
"username": req.get("username") or "",
"password": req.get("password") or "",
}) or {}
if not val.get("ok"):
fail(val.get("error") or "could not fill the login form")
wait_for_visual_update(ws)
print(json.dumps({
"ok": True,
"action": "fill_login",
"filledUsername": bool(val.get("filledUsername")),
"submitted": bool(val.get("submitted")),
"restarted": restarted,
}))
return
if action == "type":
sel = req.get("selector") or ""
text = req.get("text") or ""
if sel:
val = evaluate(ws, CLICK_JS, sel) or {}
if not val.get("ok"):
fail(val.get("error") or "target field is unavailable; no text inserted")
pointer(display, val.get("x") or 0, val.get("y") or 0)
if text:
ws.call("Input.insertText", {"text": text})
wait_for_visual_update(ws)
out = snapshot(ws)
out.update({"action": "type", "restarted": restarted})
print(json.dumps(out))
return
if action == "press":
key = req.get("key") or "Return"
press(ws, key)
wait_for_visual_update(ws)
out = snapshot(ws)
out.update({"action": "press", "key": key, "restarted": restarted})
print(json.dumps(out))
return
if action == "wait":
ms = min(max(int(req.get("ms") or 400), 0), 5000)
time.sleep(ms / 1000.0)
out = snapshot(ws)
out.update({"action": "wait", "ms": ms})
print(json.dumps(out))
return
fail("unsupported action")
finally:
ws.close()
if __name__ == "__main__":
try:
main()
except Exception as e:
fail(str(e))

271
crates/control/src/cdp.rs
View File

@ -1,271 +0,0 @@
use lazyboy_contracts::UiElement;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use crate::normalize_display;
use crate::x11::parse_ui_elements;
const CDP_PY: &str = include_str!("cdp.py");
pub fn devtools_port(display: &str) -> u16 {
let number = normalize_display(display)
.trim_start_matches(':')
.parse::<u16>()
.unwrap_or(1)
.max(1);
9221 + number
}
pub fn cdp_command_on(display: &str, profile: Option<&str>, request: &Value) -> Vec<String> {
let mut body = request.clone();
if let Some(object) = body.as_object_mut() {
object
.entry("display")
.or_insert_with(|| json!(normalize_display(display)));
object
.entry("port")
.or_insert_with(|| json!(devtools_port(display)));
if let Some(profile) = profile.filter(|value| !value.is_empty()) {
object.entry("profile").or_insert_with(|| json!(profile));
}
}
vec![
"env".into(),
format!("DISPLAY={}", normalize_display(display)),
"python3".into(),
"-c".into(),
CDP_PY.into(),
body.to_string(),
]
}
/// Same as `cdp_command_on` but the JSON body is meant to arrive on stdin
/// so secrets never appear on the process argv.
pub fn cdp_stdin_command_on(display: &str, profile: Option<&str>) -> Vec<String> {
let mut env = vec![
"env".into(),
format!("DISPLAY={}", normalize_display(display)),
];
if let Some(profile) = profile.filter(|value| !value.is_empty()) {
env.push(format!("LAZYBOY_BROWSER_PROFILE={profile}"));
}
env.extend(["python3".into(), "-c".into(), CDP_PY.into()]);
env
}
/// Marker embedded in the recorder's argv so `pkill -f` can find exactly one
/// teaching session without touching other python processes.
pub fn teach_recorder_tag(skill_id: &str) -> String {
format!("lazyboy-teach-{skill_id}")
}
pub fn teach_recorder_output(skill_id: &str) -> String {
format!("/tmp/{}.jsonl", teach_recorder_tag(skill_id))
}
pub fn sanitize_skill_id(skill_id: &str) -> String {
let cleaned: String = skill_id
.chars()
.filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_')
.take(80)
.collect();
if cleaned.is_empty() {
"unknown".into()
} else {
cleaned
}
}
pub fn teach_trajectory_dir(skill_id: &str) -> String {
format!("/tmp/lazyboy/teach-{}", sanitize_skill_id(skill_id))
}
/// Detached, long-running CDP recorder for a human demonstration. The script
/// is handed to `sh` as a positional argument so no shell quoting touches it.
pub fn cdp_record_command_on(display: &str, profile: Option<&str>, skill_id: &str) -> Vec<String> {
let mut request = json!({
"action": "record",
"ensure": true,
"out": teach_recorder_output(skill_id),
"tag": teach_recorder_tag(skill_id),
"display": normalize_display(display),
"port": devtools_port(display),
});
if let Some(profile) = profile.filter(|value| !value.is_empty()) {
request["profile"] = json!(profile);
}
vec![
"sh".into(),
"-c".into(),
"setsid nohup env DISPLAY=\"$0\" python3 -c \"$1\" \"$2\" >/dev/null 2>&1 </dev/null &"
.into(),
normalize_display(display).to_string(),
CDP_PY.into(),
request.to_string(),
]
}
pub fn cdp_record_stop_command(skill_id: &str) -> Vec<String> {
vec!["pkill".into(), "-f".into(), teach_recorder_tag(skill_id)]
}
#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CdpPage {
pub ok: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
#[serde(default)]
pub url: String,
#[serde(default)]
pub title: String,
#[serde(default)]
pub text: String,
#[serde(default)]
pub restarted: bool,
/// Seconds the click waited for a disabled control to become enabled.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub waited_seconds: Option<f64>,
#[serde(default)]
pub elements: Vec<UiElement>,
}
pub fn parse_cdp_page(raw: &str) -> CdpPage {
let value: Value = serde_json::from_str(raw.trim()).unwrap_or(Value::Null);
let ok = value.get("ok").and_then(Value::as_bool) == Some(true);
CdpPage {
ok,
error: if ok {
None
} else {
value
.get("error")
.and_then(Value::as_str)
.map(str::to_string)
.or_else(|| Some("cdp unavailable".into()))
},
waited_seconds: value.get("waitedSeconds").and_then(Value::as_f64),
url: value
.get("url")
.and_then(Value::as_str)
.unwrap_or("")
.to_string(),
title: value
.get("title")
.and_then(Value::as_str)
.unwrap_or("")
.to_string(),
text: value
.get("text")
.and_then(Value::as_str)
.unwrap_or("")
.to_string(),
restarted: value.get("restarted").and_then(Value::as_bool) == Some(true),
elements: value
.get("elements")
.map(|items| parse_ui_elements(&items.to_string()))
.unwrap_or_default(),
}
}
pub fn merge_page_elements(windows: Vec<UiElement>, page: &[UiElement]) -> Vec<UiElement> {
crate::merge_ui_elements(windows, page, &[])
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn debug_port_follows_the_display() {
assert_eq!(devtools_port(":1"), 9222);
assert_eq!(devtools_port(":2"), 9223);
assert_eq!(devtools_port("3"), 9224);
}
#[test]
fn trajectory_dir_strips_path_chars() {
assert_eq!(teach_trajectory_dir("abc/../x"), "/tmp/lazyboy/teach-abcx");
assert_eq!(sanitize_skill_id(""), "unknown");
}
#[test]
fn command_passes_port_profile_and_script() {
let argv = cdp_command_on(
":2",
Some("/home/lazyboy/.browser-profiles/bots/a"),
&json!({"action": "snapshot", "ensure": true}),
);
assert!(argv.contains(&"DISPLAY=:2".into()));
assert!(argv.iter().any(|item| item.contains("python3")));
let payload = argv.last().unwrap();
assert!(payload.contains("\"port\":9223"));
assert!(payload.contains("bots/a"));
assert!(payload.contains("snapshot"));
}
#[test]
fn parses_snapshot_elements() {
let page = parse_cdp_page(
r#"{"ok":true,"url":"https://example.com","title":"Example","text":"Hello","elements":[{"id":1,"title":"Submit","selector":"[data-lazyboy=\"1\"]","kind":"dom","x":10,"y":20,"w":80,"h":24}]}"#,
);
assert!(page.ok);
assert_eq!(page.url, "https://example.com");
assert_eq!(page.elements.len(), 1);
assert_eq!(page.elements[0].title, "Submit");
assert_eq!(
page.elements[0].selector.as_deref(),
Some("[data-lazyboy=\"1\"]")
);
assert_eq!(page.elements[0].center(), (50, 32));
}
#[test]
fn merge_keeps_page_controls_and_native_dialogs() {
let windows = vec![
UiElement {
id: 1,
title: "Chromium".into(),
x: 0,
y: 0,
w: 1280,
h: 800,
kind: Some("window".into()),
..UiElement::default()
},
UiElement {
id: 2,
title: "Open File".into(),
x: 100,
y: 100,
w: 400,
h: 300,
kind: Some("window".into()),
..UiElement::default()
},
];
let page = vec![UiElement {
id: 1,
title: "Login".into(),
selector: Some("[data-lazyboy=\"1\"]".into()),
kind: Some("dom".into()),
x: 40,
y: 80,
w: 60,
h: 20,
..UiElement::default()
}];
let merged = merge_page_elements(windows, &page);
assert_eq!(merged.len(), 2);
assert_eq!(merged[0].title, "Login");
assert_eq!(merged[1].id, 2);
assert_eq!(merged[1].title, "Open File");
}
#[test]
fn unavailable_page_is_not_ok() {
let page = parse_cdp_page(r#"{"ok":false,"error":"cdp unavailable"}"#);
assert!(!page.ok);
assert_eq!(page.error.as_deref(), Some("cdp unavailable"));
}
}

51
crates/control/src/clipboard.py
View File

@ -1,51 +0,0 @@
"""Set X11 clipboard, confirm ownership/content, then paste into the active app."""
import subprocess
import sys
import time
def run(argv, **kwargs):
return subprocess.run(argv, check=True, timeout=2, **kwargs)
def paste(text):
if not text:
return
raw = text.encode('utf-8')
# xclip forks after reading stdin; detached descriptors avoid pipe hangs.
run(['xclip', '-selection', 'clipboard', '-in'], input=raw,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
deadline = time.monotonic() + 2
while True:
actual = run(['xclip', '-selection', 'clipboard', '-out'], stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL).stdout
if actual == raw:
break
if time.monotonic() >= deadline:
raise RuntimeError('clipboard synchronization timed out; nothing pasted')
time.sleep(.02)
key_for_active_app('v')
def key_for_active_app(key):
window = run(['xdotool', 'getactivewindow'], stdout=subprocess.PIPE).stdout.decode().strip()
wmclass = run(['xprop', '-id', window, 'WM_CLASS'], stdout=subprocess.PIPE).stdout.decode().lower()
terminal = any(name in wmclass for name in ('terminal', 'xterm', 'kitty', 'alacritty', 'konsole'))
run(['xdotool', 'key', '--clearmodifiers', ('ctrl+shift+' if terminal else 'ctrl+') + key],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def copy_selection():
key_for_active_app('c')
# Wait for the application to process the shortcut before reading selection.
time.sleep(.1)
return run(['xclip', '-selection', 'clipboard', '-out'], stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL).stdout.decode('utf-8')
if __name__ == '__main__':
if len(sys.argv)>1 and sys.argv[1]=='copy':
sys.stdout.write(copy_selection())
else:
paste(sys.stdin.read())

34
crates/control/src/controller.rs
View File

@ -5,9 +5,8 @@ use async_trait::async_trait;
use thiserror::Error;
use crate::cua::CuaController;
use crate::legacy::LegacyController;
use crate::{
ActionRequest, ActionResult, BrowserRequest, CdpPage, RecordingRequest, RecordingResult,
ActionRequest, ActionResult, BrowserPage, BrowserRequest, RecordingRequest, RecordingResult,
RecordingSession,
};
use lazyboy_contracts::ComputerObservation;
@ -96,7 +95,6 @@ fn truncate_error(text: &str) -> String {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ComputerDriver {
Legacy,
Cua,
}
@ -104,29 +102,25 @@ impl ComputerDriver {
pub const ENV: &'static str = "LAZYBOY_COMPUTER_DRIVER";
pub fn from_env() -> Self {
match std::env::var(Self::ENV) {
Ok(value) if value.trim().is_empty() => Self::Legacy,
Ok(value) => match value.parse() {
Ok(driver) => driver,
Err(error) => {
tracing::error!("{error}; using legacy");
Self::Legacy
}
},
Err(_) => Self::Legacy,
if let Ok(value) = std::env::var(Self::ENV)
&& !value.trim().is_empty()
&& value.parse::<Self>().is_err()
{
tracing::warn!(
"Only the Cua computer driver is supported; ignoring obsolete driver setting"
);
}
Self::Cua
}
pub fn as_str(self) -> &'static str {
match self {
Self::Legacy => "legacy",
Self::Cua => "cua",
}
}
pub fn controller(self) -> Arc<dyn ComputerController> {
match self {
Self::Legacy => Arc::new(LegacyController),
Self::Cua => Arc::new(CuaController::default()),
}
}
@ -137,7 +131,6 @@ impl FromStr for ComputerDriver {
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value.trim().to_ascii_lowercase().as_str() {
"legacy" => Ok(Self::Legacy),
"cua" => Ok(Self::Cua),
other => Err(UnknownComputerDriver(other.to_string())),
}
@ -145,7 +138,7 @@ impl FromStr for ComputerDriver {
}
#[derive(Debug, Error, Clone, PartialEq, Eq)]
#[error("unknown computer driver {0:?}; expected legacy or cua")]
#[error("unknown computer driver {0:?}; expected cua")]
pub struct UnknownComputerDriver(pub String);
#[async_trait]
@ -166,7 +159,7 @@ pub trait ComputerController: Send + Sync {
&self,
request: &BrowserRequest,
ctx: &ControlContext,
) -> Result<CdpPage, ControlError>;
) -> Result<BrowserPage, ControlError>;
async fn start_recording(
&self,
@ -193,10 +186,7 @@ mod tests {
#[test]
fn parses_driver_names() {
assert_eq!(
"legacy".parse::<ComputerDriver>().unwrap(),
ComputerDriver::Legacy
);
assert!("legacy".parse::<ComputerDriver>().is_err());
assert_eq!(
"CUA".parse::<ComputerDriver>().unwrap(),
ComputerDriver::Cua

170
crates/control/src/cua/browser.rs
View File

@ -5,8 +5,7 @@ use tokio::time::{Duration, sleep};
use super::ListedWindow;
use super::client::CuaClient;
use crate::controller::ControlError;
use crate::process::spawn_detached;
use crate::{BrowserRequest, CdpPage, launch_argv_on};
use crate::{BrowserPage, BrowserRequest, launch_argv_on};
#[derive(Debug, Clone)]
pub struct BrowserBind {
@ -14,7 +13,7 @@ pub struct BrowserBind {
pub window_id: u64,
pub target_id: String,
pub tab_id: String,
pub page: Option<CdpPage>,
pub page: Option<BrowserPage>,
}
pub fn is_cua_ref(selector: &str) -> bool {
@ -34,7 +33,7 @@ pub fn allowed_navigate_url(url: &str) -> bool {
url.starts_with("http://") || url.starts_with("https://") || url.starts_with("about:")
}
pub fn page_from_semantic(value: &Value) -> CdpPage {
pub fn page_from_semantic(value: &Value) -> BrowserPage {
let page = value.get("page").unwrap_or(value);
let url = page
.get("url")
@ -62,7 +61,7 @@ pub fn page_from_semantic(value: &Value) -> CdpPage {
}
let ok = value.get("status").and_then(Value::as_str) != Some("refused")
&& value.get("ok").and_then(Value::as_bool) != Some(false);
CdpPage {
BrowserPage {
ok,
error: if ok {
None
@ -147,7 +146,7 @@ fn number(value: &Value, key: &str) -> Option<u32> {
.map(|n| n as u32)
}
pub fn find_ref<'a>(page: &'a CdpPage, selector: &'a str) -> Option<&'a str> {
pub fn find_ref<'a>(page: &'a BrowserPage, selector: &'a str) -> Option<&'a str> {
if is_cua_ref(selector) {
return page
.elements
@ -227,9 +226,7 @@ pub async fn ensure_bind(
let mut listed = windows.to_vec();
if chromium_window(&listed).is_none() && ensure {
if let Some(argv) = launch_argv_on(display, profile, "browser", None) {
spawn_detached(&argv)
.await
.map_err(ControlError::internal)?;
super::launch::run(client, display, &argv).await?;
}
for _ in 0..24 {
sleep(Duration::from_millis(250)).await;
@ -300,7 +297,7 @@ pub async fn snapshot(
client: &CuaClient,
display: &str,
bind: &BrowserBind,
) -> Result<CdpPage, ControlError> {
) -> Result<BrowserPage, ControlError> {
let value = client
.call(
display,
@ -324,11 +321,11 @@ pub async fn run(
request: &BrowserRequest,
windows: &[ListedWindow],
bind: &mut Option<BrowserBind>,
) -> Result<CdpPage, ControlError> {
) -> Result<BrowserPage, ControlError> {
if request.action == "probe" {
return Ok(CdpPage {
return Ok(BrowserPage {
ok: chromium_window(windows).is_some(),
..CdpPage::default()
..BrowserPage::default()
});
}
let attached = match bind.as_ref() {
@ -340,11 +337,26 @@ pub async fn run(
}
};
if request.action == "ensure" {
return Ok(CdpPage {
return Ok(BrowserPage {
ok: true,
..CdpPage::default()
..BrowserPage::default()
});
}
if matches!(
request.action.as_str(),
"navigate" | "click" | "type" | "press"
) {
client
.call(
display,
"bring_to_front",
&json!({
"pid": attached.pid, "window_id": attached.window_id,
}),
&[],
)
.await?;
}
match request.action.as_str() {
"snapshot" => snapshot(client, display, &attached).await,
"wait" => {
@ -386,6 +398,7 @@ pub async fn run(
"key": key,
"pid": attached.pid,
"window_id": attached.window_id,
"delivery_mode": "foreground",
}),
&[],
)
@ -404,14 +417,14 @@ async fn click(
display: &str,
bind: &BrowserBind,
request: &BrowserRequest,
) -> Result<CdpPage, ControlError> {
) -> Result<BrowserPage, ControlError> {
let selector = request
.selector
.as_deref()
.ok_or_else(|| ControlError::InvalidAction("browser click needs a selector".into()))?;
let page = bind.page.as_ref().ok_or(ControlError::StaleReference)?;
let Some(r#ref) = find_ref(page, selector) else {
return Ok(CdpPage {
return Ok(BrowserPage {
ok: false,
error: Some(
"element gone: the page changed and ids were renumbered. Use the fresh element list in this result."
@ -421,7 +434,7 @@ async fn click(
title: page.title.clone(),
text: page.text.clone(),
elements: page.elements.clone(),
..CdpPage::default()
..BrowserPage::default()
});
};
let r#ref = r#ref.to_string();
@ -442,29 +455,57 @@ async fn click(
snapshot(client, display, bind).await
}
fn unique_native_web_entry<'a>(state: &'a Value, label: &str) -> Result<&'a Value, ControlError> {
if label.is_empty() {
return Err(ControlError::TargetNotFound);
}
let mut matches = state
.get("elements")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter(|element| {
element.get("label").and_then(Value::as_str) == Some(label)
&& element.get("in_web_content").and_then(Value::as_bool) == Some(true)
&& matches!(
element.get("role").and_then(Value::as_str),
Some("entry" | "password text" | "text")
)
&& element.get("frame").is_some_and(|frame| {
frame["w"].as_f64().unwrap_or(0.0) > 0.0
&& frame["h"].as_f64().unwrap_or(0.0) > 0.0
})
});
let first = matches.next().ok_or(ControlError::TargetNotFound)?;
if matches.next().is_some() {
return Err(ControlError::TargetNotFound);
}
Ok(first)
}
async fn type_into(
client: &CuaClient,
display: &str,
bind: &BrowserBind,
request: &BrowserRequest,
) -> Result<CdpPage, ControlError> {
) -> Result<BrowserPage, ControlError> {
let text = request.text.clone().unwrap_or_default();
tracing::info!(backend = "cua", tool = "browser_type", length = text.len());
if let Some(selector) = request.selector.as_deref() {
let page = bind.page.as_ref().ok_or(ControlError::StaleReference)?;
let Some(r#ref) = find_ref(page, selector) else {
return Ok(CdpPage {
return Ok(BrowserPage {
ok: false,
error: Some("target field is unavailable; no text inserted".into()),
url: page.url.clone(),
title: page.title.clone(),
text: page.text.clone(),
elements: page.elements.clone(),
..CdpPage::default()
..BrowserPage::default()
});
};
let r#ref = r#ref.to_string();
client
let typed = client
.call(
display,
"browser_type",
@ -477,7 +518,69 @@ async fn type_into(
}),
&[],
)
.await?;
.await;
match typed {
Ok(_) => {}
Err(ControlError::Unsupported) => {
// The pinned driver can refuse Input.insertText for email
// fields. Resolve a unique visible native web entry from Cua;
// never guess a pixel or a similarly named browser-chrome field.
let label = page
.elements
.iter()
.find(|element| element.selector.as_deref() == Some(r#ref.as_str()))
.map(|element| element.title.as_str())
.ok_or(ControlError::StaleReference)?;
let native = client.call(display, "get_window_state", &json!({
"pid": bind.pid, "window_id": bind.window_id, "include_screenshot": false,
}), &[]).await?;
let entry = unique_native_web_entry(&native, label)?;
let frame = &entry["frame"];
let x = frame["x"].as_f64().ok_or(ControlError::TargetNotFound)?
+ frame["w"].as_f64().unwrap_or(0.0) / 2.0;
let y = frame["y"].as_f64().ok_or(ControlError::TargetNotFound)?
+ frame["h"].as_f64().unwrap_or(0.0) / 2.0;
client
.call(
display,
"click",
&json!({
"x": x, "y": y, "scope": "desktop",
}),
&[],
)
.await?;
client
.call(
display,
"hotkey",
&json!({
"pid": bind.pid, "window_id": bind.window_id,
"keys": ["ctrl", "a"], "delivery_mode": "foreground",
}),
&[],
)
.await?;
if text.is_empty() {
client
.call(
display,
"press_key",
&json!({
"pid": bind.pid, "window_id": bind.window_id,
"key": "backspace", "delivery_mode": "foreground",
}),
&[],
)
.await?;
} else {
super::clipboard::paste(client, display, &text).await?;
}
}
Err(error) => return Err(error),
}
} else if !text.is_ascii() || text.contains('\n') {
super::clipboard::paste(client, display, &text).await?;
} else if !text.is_empty() {
client
.call(
@ -508,6 +611,25 @@ fn map_press_key(key: &str) -> String {
mod tests {
use super::*;
#[test]
fn native_typing_requires_a_unique_visible_web_field() {
let entry = json!({"label":"Email","role":"entry","in_web_content":true,"frame":{"w":100,"h":20},"element_token":"s1:1"});
assert_eq!(
unique_native_web_entry(&json!({"elements":[entry.clone()]}), "Email").unwrap()["element_token"],
"s1:1"
);
assert!(
unique_native_web_entry(&json!({"elements":[entry.clone(),entry.clone()]}), "Email")
.is_err()
);
let mut chrome = entry.clone();
chrome["in_web_content"] = json!(false);
assert!(unique_native_web_entry(&json!({"elements":[chrome]}), "Email").is_err());
let mut hidden = entry;
hidden["frame"]["w"] = json!(0);
assert!(unique_native_web_entry(&json!({"elements":[hidden]}), "Email").is_err());
}
#[test]
fn detects_snapshot_scoped_refs() {
assert!(is_cua_ref("p1:1"));
@ -518,7 +640,7 @@ mod tests {
}
#[test]
fn semantic_snapshot_becomes_cdp_page() {
fn semantic_snapshot_becomes_browser_page() {
let raw = json!({
"status": "ok",
"outline": "- button \"Smoke Click\"\n- textbox \"Smoke Entry\"",

108
crates/control/src/cua/client.rs
View File

@ -40,6 +40,17 @@ impl CuaClient {
/// give each one an implicit session that dies with the process. Trajectory
/// recording, snapshots, and browser binds only line up under one label.
pub fn session_for_display(display: &str) -> String {
let number = normalize_display(display)
.trim_start_matches(':')
.to_string();
if let Ok(name) =
std::fs::read_to_string(format!("/tmp/lazyboy/screen-{number}.agent-name"))
{
let name = public_agent_name(&name);
if !name.is_empty() {
return name;
}
}
format!(
"lazyboy-{}",
normalize_display(display).trim_start_matches(':')
@ -91,8 +102,21 @@ impl CuaClient {
) -> Result<Value, ControlError> {
let mut body = with_session_label(screen, payload);
let mut escalated = false;
let mut revived = false;
loop {
let outcome = self.attempt(screen, tool, &body, extra).await?;
if outcome.session_ended && !revived && tool != "start_session" {
let session = with_session_label(screen, &json!({}));
let started = self.attempt(screen, "start_session", &session, &[]).await?;
if let Some(error) = started.error {
return Err(error);
}
revived = true;
if !read_after_session_restart(tool) {
return Err(ControlError::StaleReference);
}
continue;
}
if !escalated
&& outcome.error.is_some()
&& let Some(mode) = recommended_delivery(&outcome.value)
@ -152,11 +176,43 @@ impl CuaClient {
duration_ms = started.elapsed().as_millis() as u64,
success = error.is_none()
);
Ok(Outcome { value, error })
let session_ended = !output.status.success()
&& [&*stdout, &*stderr].iter().any(|text| {
text.trim_start().starts_with("session '") && text.contains("has ended; tool call")
});
Ok(Outcome {
value,
error,
session_ended,
})
}
}
fn public_agent_name(name: &str) -> String {
name.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
.chars()
.filter(|ch| !ch.is_control())
.take(80)
.collect()
}
fn read_after_session_restart(tool: &str) -> bool {
matches!(
tool,
"get_desktop_state"
| "list_windows"
| "get_window_state"
| "get_accessibility_tree"
| "get_browser_state"
| "health_report"
| "get_cursor_position"
)
}
struct Outcome {
session_ended: bool,
value: Value,
error: Option<ControlError>,
}
@ -210,10 +266,12 @@ fn response_error(value: &Value) -> Option<ControlError> {
.is_some_and(|code| code != "ok")
|| value.get("ok").and_then(Value::as_bool) == Some(false)
|| value.get("isError").and_then(Value::as_bool) == Some(true)
|| matches!(
value.get("status").and_then(Value::as_str),
Some("refused" | "error")
)
|| ["status", "effect"].iter().any(|key| {
matches!(
value.get(*key).and_then(Value::as_str),
Some("refused" | "error")
)
})
{
Some(classify_cua_failure(&value.to_string()))
} else {
@ -348,7 +406,10 @@ pub fn first_array_of_objects<'a>(value: &'a Value, required: &str) -> Vec<&'a V
fn classify_cua_failure(text: &str) -> ControlError {
let lower = text.to_ascii_lowercase();
if lower.contains("stale") || (lower.contains("session") && lower.contains("ended")) {
if lower.contains("stale")
|| lower.contains("not a live binding in this session")
|| (lower.contains("session") && lower.contains("ended"))
{
ControlError::StaleReference
} else if lower.contains("not_found") || lower.contains("not found") {
ControlError::TargetNotFound
@ -358,7 +419,7 @@ fn classify_cua_failure(text: &str) -> ControlError {
ControlError::PermissionDenied
} else if lower.contains("invalid_action_target") {
ControlError::InvalidAction("Cua rejected the action target".into())
} else if lower.contains("unsupported") {
} else if lower.contains("unsupported") || lower.contains("route_unavailable") {
ControlError::Unsupported
} else {
// Driver diagnostics can echo typed text or credentials. Keep raw
@ -369,11 +430,44 @@ fn classify_cua_failure(text: &str) -> ControlError {
#[cfg(test)]
mod tests {
#[test]
fn agent_name_preserves_unicode_without_control_characters() {
assert_eq!(
super::public_agent_name(" 小幫手\n Alice\u{0007} "),
"小幫手 Alice"
);
assert_eq!(
super::public_agent_name(&"小".repeat(100)).chars().count(),
80
);
}
use super::*;
#[test]
fn expired_sessions_only_retry_observations() {
assert_eq!(
classify_cua_failure(
"confirmation provider failed: target bt-old is not a live binding in this session — re-run get_browser_state with pid + window_id"
),
ControlError::StaleReference
);
assert!(read_after_session_restart("get_desktop_state"));
assert!(read_after_session_restart("get_browser_state"));
for tool in ["click", "type_text", "browser_type", "hotkey", "launch_app"] {
assert!(!read_after_session_restart(tool));
}
}
#[test]
fn structured_refusals_are_errors_but_page_text_is_not() {
assert!(response_error(&serde_json::json!({"code": "invalid_action_target"})).is_some());
assert!(matches!(
response_error(
&json!({"effect":"refused","escalation":{"reason":"route_unavailable"}})
),
Some(ControlError::Unsupported)
));
assert!(response_error(&serde_json::json!({"outline": "❌ payment declined"})).is_none());
}

144
crates/control/src/cua/clipboard.rs Normal file
View File

@ -0,0 +1,144 @@
//! GTK clipboard editor operated via Cua, for Linux driver builds without
//! clipboard_read/write support. No direct X11/AT-SPI/clipboard subprocesses.
use super::{CuaClient, CuaController, ListedWindow};
use crate::ControlError;
use serde_json::{Value, json};
use tokio::time::{Duration, sleep};
async fn front(
client: &CuaClient,
display: &str,
window: &ListedWindow,
) -> Result<(), ControlError> {
client
.call(
display,
"bring_to_front",
&json!({"pid":window.pid,"window_id":window.id}),
&[],
)
.await?;
Ok(())
}
async fn key(
client: &CuaClient,
display: &str,
window: &ListedWindow,
keys: &[&str],
) -> Result<(), ControlError> {
client.call(display, "hotkey", &json!({"pid":window.pid,"window_id":window.id,"keys":keys,"delivery_mode":"foreground"}), &[]).await?;
Ok(())
}
fn shortcut<'a>(window: &ListedWindow, letter: &'a str) -> Vec<&'a str> {
if window.app_name.to_lowercase().contains("terminal") {
vec!["ctrl", "shift", letter]
} else {
vec!["ctrl", letter]
}
}
async fn active(client: &CuaClient, display: &str) -> Result<ListedWindow, ControlError> {
CuaController::list_windows_now(client, display)
.await?
.into_iter()
.max_by_key(|w| w.z)
.ok_or(ControlError::TargetNotFound)
}
async fn editor(client: &CuaClient, display: &str) -> Result<(ListedWindow, Value), ControlError> {
super::launch::run(
client,
display,
&[
"env".into(),
format!("DISPLAY={display}"),
"lazyboy-clipboard".into(),
],
)
.await?;
let window = CuaController::list_windows_now(client, display)
.await?
.into_iter()
.find(|w| w.title == "Clipboard · LazyBoy")
.ok_or(ControlError::TargetNotFound)?;
let state = client
.call(
display,
"get_window_state",
&json!({"pid":window.pid,"window_id":window.id,"include_screenshot":false}),
&[],
)
.await?;
Ok((window, state))
}
fn element<'a>(state: &'a Value, label: &str) -> Result<&'a Value, ControlError> {
state
.get("elements")
.and_then(Value::as_array)
.into_iter()
.flatten()
.find(|item| {
item.get("label")
.and_then(Value::as_str)
.is_some_and(|text| {
if label == "Clipboard text" {
text.starts_with("Clipboard text: ")
} else {
text == label
}
})
})
.ok_or(ControlError::TargetNotFound)
}
async fn restore(
client: &CuaClient,
display: &str,
editor: &ListedWindow,
original: &ListedWindow,
) -> Result<(), ControlError> {
key(client, display, editor, &["alt", "f4"]).await?;
front(client, display, original).await
}
pub(super) async fn paste(
client: &CuaClient,
display: &str,
text: &str,
) -> Result<(), ControlError> {
let original = active(client, display).await?;
let (window, state) = editor(client, display).await?;
let result = async {
let entry = element(&state, "Clipboard text")?;
client.call(display, "set_value", &json!({"pid":window.pid,"window_id":window.id,"element_token":entry["element_token"],"value":text}), &[]).await?;
// Fresh snapshot verifies exact content and supplies fresh action refs.
let state = client.call(display, "get_window_state", &json!({"pid":window.pid,"window_id":window.id,"include_screenshot":false}), &[]).await?;
if element(&state, "Clipboard text")?.get("label").and_then(Value::as_str) != Some(format!("Clipboard text: {text}").as_str()) {
return Err(ControlError::internal("clipboard text did not roundtrip; nothing pasted"));
}
let copy = element(&state, "Copy")?;
client.call(display, "click", &json!({"pid":window.pid,"window_id":window.id,"element_token":copy["element_token"],"delivery_mode":"foreground"}), &[]).await?;
front(client, display, &original).await?;
key(client, display, &original, &shortcut(&original,"v")).await?;
sleep(Duration::from_millis(100)).await;
Ok(())
}.await;
// Keep the editor alive until the destination consumes its clipboard.
let restored = restore(client, display, &window, &original).await;
result.and(restored)
}
pub(super) async fn copy(client: &CuaClient, display: &str) -> Result<String, ControlError> {
let original = active(client, display).await?;
front(client, display, &original).await?;
key(client, display, &original, &shortcut(&original, "c")).await?;
sleep(Duration::from_millis(100)).await;
let (window, state) = editor(client, display).await?;
let result = element(&state, "Clipboard text").map(|item| {
item.get("label")
.and_then(Value::as_str)
.unwrap_or("")
.strip_prefix("Clipboard text: ")
.unwrap_or("")
.to_string()
});
restore(client, display, &window, &original).await?;
result
}

65
crates/control/src/cua/launch.rs Normal file
View File

@ -0,0 +1,65 @@
//! Launch and foreground applications using Cua on the shared desktop.
use serde_json::json;
use tokio::time::{Duration, Instant, sleep};
use super::{CuaClient, CuaController};
use crate::ControlError;
pub(super) async fn run(
client: &CuaClient,
display: &str,
argv: &[String],
) -> Result<(), ControlError> {
let (program, arguments) = argv
.split_first()
.ok_or_else(|| ControlError::InvalidAction("empty application command".into()))?;
let before = CuaController::list_windows_now(client, display).await?;
let result = client
.call(
display,
"launch_app",
&json!({
"name": program, "additional_arguments": arguments,
}),
&[],
)
.await?;
let pid = result.get("pid").and_then(serde_json::Value::as_u64);
let browser = arguments.iter().any(|arg| arg == "lazyboy-browser");
let deadline = Instant::now() + Duration::from_secs(8);
loop {
let windows = CuaController::list_windows_now(client, display).await?;
let window = windows
.iter()
.find(|window| Some(window.pid) == pid)
.or_else(|| {
windows.iter().find(|window| {
!before
.iter()
.any(|old| old.id == window.id && old.title == window.title)
})
})
.or_else(|| {
if browser {
super::browser::chromium_window(&windows)
} else {
None
}
});
if let Some(window) = window {
client
.call(
display,
"bring_to_front",
&json!({"pid":window.pid,"window_id":window.id}),
&[],
)
.await?;
return Ok(());
}
if Instant::now() >= deadline {
return Err(ControlError::TargetNotFound);
}
sleep(Duration::from_millis(100)).await;
}
}

94
crates/control/src/cua/mod.rs
View File

@ -1,5 +1,7 @@
mod browser;
mod client;
mod clipboard;
mod launch;
mod native;
mod record;
mod translate;
@ -20,9 +22,8 @@ use tokio::time::{Duration, sleep};
use crate::controller::{
ComputerController, ComputerDriver, ControlContext, ControlError, ControllerHealth,
};
use crate::process::spawn_detached;
use crate::{
ActionRequest, ActionResult, BrowserRequest, CdpPage, RecordingRequest, RecordingResult,
ActionRequest, ActionResult, BrowserPage, BrowserRequest, RecordingRequest, RecordingResult,
RecordingSession, action_pause_ms, image_dimensions, normalize_display, observation_from_png,
observation_with_elements, teach_trajectory_dir,
};
@ -112,10 +113,8 @@ impl ComputerController for CuaController {
async fn observe(&self, ctx: &ControlContext) -> Result<ComputerObservation, ControlError> {
let _screen = self.lock_screen(&ctx.display).await;
self.browser
.lock()
.await
.remove(normalize_display(&ctx.display));
// A desktop screenshot does not change the browser binding. The next
// browser snapshot refreshes page refs; keep the session attachment warm.
self.observe_display(&ctx.display).await
}
@ -125,6 +124,14 @@ impl ComputerController for CuaController {
ctx: &ControlContext,
) -> Result<ActionResult, ControlError> {
let _screen = self.lock_screen(&ctx.display).await;
if matches!(request.actions.as_slice(), [ComputerAction::CopySelection]) {
let text = clipboard::copy(&self.client, &ctx.display).await?;
return Ok(ActionResult {
completed: 1,
clipboard_text: Some(text),
observation: None,
});
}
let display = ctx.display.as_str();
let profile = ctx.profile_path.as_deref();
let key = normalize_display(display).to_string();
@ -144,6 +151,7 @@ impl ComputerController for CuaController {
None
};
Ok(ActionResult {
clipboard_text: None,
completed,
observation,
})
@ -159,7 +167,7 @@ impl ComputerController for CuaController {
&self,
request: &BrowserRequest,
ctx: &ControlContext,
) -> Result<CdpPage, ControlError> {
) -> Result<BrowserPage, ControlError> {
let _screen = self.lock_screen(&ctx.display).await;
if !matches!(
request.action.as_str(),
@ -188,10 +196,10 @@ impl ComputerController for CuaController {
.await
{
Err(ControlError::BrowserUnavailable) if !had_bind => {
return Ok(CdpPage {
return Ok(BrowserPage {
ok: false,
error: Some("cdp unavailable".into()),
..CdpPage::default()
error: Some("Cua browser unavailable".into()),
..BrowserPage::default()
});
}
Err(error)
@ -236,27 +244,14 @@ impl ComputerController for CuaController {
.client
.call(&ctx.display, "stop_recording", &json!({}), &[])
.await;
if let Err(error) = self
.client
self.client
.call(
&ctx.display,
"start_recording",
&json!({ "output_dir": output_dir, "record_video": false }),
&[],
)
.await
{
tracing::warn!(error = %error, "cua start_recording failed");
}
if let Err(error) = crate::legacy::start_cdp_recorder(
&ctx.display,
ctx.profile_path.as_deref(),
&request.skill_id,
)
.await
{
tracing::warn!(error = %error, "cdp recorder start failed");
}
.await?;
Ok(RecordingSession {
skill_id: request.skill_id.clone(),
output_dir,
@ -265,14 +260,13 @@ impl ComputerController for CuaController {
async fn stop_recording(
&self,
request: &RecordingRequest,
_request: &RecordingRequest,
ctx: &ControlContext,
) -> Result<(), ControlError> {
let _ = self
.client
self.client
.call(&ctx.display, "stop_recording", &json!({}), &[])
.await;
crate::legacy::stop_cdp_recorder(&request.skill_id).await
.await?;
Ok(())
}
async fn collect_recording(
@ -280,9 +274,8 @@ impl ComputerController for CuaController {
request: &RecordingRequest,
_ctx: &ControlContext,
) -> Result<RecordingResult, ControlError> {
let mut events = crate::legacy::collect_cdp_events(&request.skill_id).await;
let dir = teach_trajectory_dir(&request.skill_id);
events.extend(record::events_from_dir(std::path::Path::new(&dir)));
let mut events = record::events_from_dir(std::path::Path::new(&dir));
events.sort_by_key(|event| event.get("at").and_then(Value::as_i64).unwrap_or(0));
let _ = tokio::fs::remove_dir_all(&dir).await;
Ok(RecordingResult { events })
@ -292,7 +285,7 @@ impl ComputerController for CuaController {
fn update_browser_page(
bind: &mut browser::BrowserBind,
action: &str,
result: &Result<CdpPage, ControlError>,
result: &Result<BrowserPage, ControlError>,
) {
// These checks neither observe nor mutate the page. Keep the refs returned
// by the previous snapshot usable for the next click/type.
@ -417,7 +410,7 @@ impl CuaController {
.enumerate()
.map(|(index, window)| UiElement {
id: (index + 1) as u32,
title: window.title.chars().take(80).collect(),
title: window.title.chars().take(256).collect(),
x: window.x.max(0) as u32,
y: window.y.max(0) as u32,
w: window.w,
@ -509,11 +502,16 @@ impl CuaController {
sleep(Duration::from_millis(ms)).await;
Ok(())
}
TranslatedAction::LegacyArgv { argv } => {
spawn_detached(&argv).await.map_err(ControlError::internal)
}
TranslatedAction::Launch { argv } => launch::run(&self.client, display, &argv).await,
TranslatedAction::FocusTitle { title } => self.focus_title(display, &title).await,
TranslatedAction::Cua { tool, mut payload } => {
if tool == "type_text"
&& let Some(text) = payload.get("text").and_then(Value::as_str)
&& (!text.is_ascii() || text.contains(['\n', '\r']))
{
return clipboard::paste(&self.client, display, text).await;
}
if tool == "scroll" && payload.get("x").is_none() {
let (x, y) = self.scroll_point(display).await;
payload["x"] = json!(x);
@ -608,12 +606,14 @@ pub(crate) struct ListedWindow {
pub(crate) z: i64,
}
fn map_browser_unavailable(result: Result<CdpPage, ControlError>) -> Result<CdpPage, ControlError> {
fn map_browser_unavailable(
result: Result<BrowserPage, ControlError>,
) -> Result<BrowserPage, ControlError> {
match result {
Err(ControlError::BrowserUnavailable) => Ok(CdpPage {
Err(ControlError::BrowserUnavailable) => Ok(BrowserPage {
ok: false,
error: Some("cdp unavailable".into()),
..CdpPage::default()
error: Some("Cua browser unavailable".into()),
..BrowserPage::default()
}),
other => other,
}
@ -754,19 +754,19 @@ mod tests {
window_id: 2,
target_id: "target".into(),
tab_id: "tab".into(),
page: Some(CdpPage {
page: Some(BrowserPage {
ok: true,
title: "original snapshot".into(),
..CdpPage::default()
..BrowserPage::default()
}),
};
for action in ["probe", "ensure"] {
update_browser_page(
&mut bind,
action,
&Ok(CdpPage {
&Ok(BrowserPage {
ok: true,
..CdpPage::default()
..BrowserPage::default()
}),
);
assert_eq!(bind.page.as_ref().unwrap().title, "original snapshot");
@ -776,10 +776,10 @@ mod tests {
update_browser_page(
&mut bind,
"snapshot",
&Ok(CdpPage {
&Ok(BrowserPage {
ok: true,
title: "fresh".into(),
..CdpPage::default()
..BrowserPage::default()
}),
);
assert_eq!(bind.page.as_ref().unwrap().title, "fresh");

15
crates/control/src/cua/native.rs
View File

@ -118,10 +118,23 @@ pub(super) async fn act(
verb: RefVerb,
text: Option<&str>,
) -> Result<(), ControlError> {
client
.call(
display,
"bring_to_front",
&json!({
"pid": target.pid, "window_id": target.window_id,
}),
&[],
)
.await?;
let mut payload =
json!({"pid": target.pid, "window_id": target.window_id, "element_token": target.token});
let tool = match verb {
RefVerb::Click => "click",
RefVerb::Click => {
payload["delivery_mode"] = json!("foreground");
"click"
}
RefVerb::SetValue => {
payload["value"] = json!(text.unwrap_or(""));
"set_value"

10
crates/control/src/cua/translate.rs
View File

@ -8,7 +8,7 @@ use crate::{launch_argv_on, open_argv_on};
pub enum TranslatedAction {
Cua { tool: &'static str, payload: Value },
Sleep { ms: u64 },
LegacyArgv { argv: Vec<String> },
Launch { argv: Vec<String> },
FocusTitle { title: String },
}
@ -19,15 +19,17 @@ pub fn translate_action(
) -> Result<TranslatedAction, ControlError> {
match action {
ComputerAction::Wait { ms } => Ok(TranslatedAction::Sleep { ms: u64::from(*ms) }),
ComputerAction::Open { path } => Ok(TranslatedAction::LegacyArgv {
ComputerAction::Open { path } => Ok(TranslatedAction::Launch {
argv: open_argv_on(display, profile, path),
}),
ComputerAction::Launch { application, uri } => {
let argv = launch_argv_on(display, profile, application, uri.as_deref())
.ok_or(ControlError::Unsupported)?;
Ok(TranslatedAction::LegacyArgv { argv })
Ok(TranslatedAction::Launch { argv })
}
ComputerAction::CopySelection | ComputerAction::Ref { .. } => {
Err(ControlError::Unsupported)
}
ComputerAction::Ref { .. } => Err(ControlError::Unsupported),
ComputerAction::Focus { title } => Ok(TranslatedAction::FocusTitle {
title: title.clone(),
}),

306
crates/control/src/legacy.rs
View File

@ -1,306 +0,0 @@
use async_trait::async_trait;
use lazyboy_contracts::{
ComputerAction, ComputerObservation, DEFAULT_SCREEN_HEIGHT, DEFAULT_SCREEN_WIDTH, RefVerb,
};
use tokio::time::{Duration, sleep};
use crate::controller::{
ComputerController, ComputerDriver, ControlContext, ControlError, ControllerHealth,
};
use crate::process::{
capture_stdout, run_output, run_stdout_text, run_stdout_text_timeout, spawn_detached,
};
use crate::{
ActionRequest, ActionResult, BrowserRequest, CdpPage, RecordingRequest, RecordingResult,
RecordingSession, a11y_command_on, action_pause_ms, cdp_command_on, cdp_record_command_on,
cdp_record_stop_command, devtools_port, image_dimensions, launch_argv_on, observation_from_png,
observation_with_elements, open_argv_on, parse_a11y_page, parse_cdp_page, parse_pointer_state,
parse_ui_elements, pointer_state_command_on, screenshot_command_on, teach_recorder_output,
teach_trajectory_dir, window_list_command_on, xdotool_argv_on,
};
#[derive(Debug, Default, Clone, Copy)]
pub struct LegacyController;
#[async_trait]
impl ComputerController for LegacyController {
fn backend(&self) -> ComputerDriver {
ComputerDriver::Legacy
}
async fn health(&self, ctx: &ControlContext) -> Result<ControllerHealth, ControlError> {
Ok(ControllerHealth {
backend: ComputerDriver::Legacy.as_str().to_string(),
version: None,
healthy: true,
degraded: false,
details: vec![format!("display {}", ctx.display)],
})
}
async fn observe(&self, ctx: &ControlContext) -> Result<ComputerObservation, ControlError> {
observe_display(&ctx.display).await
}
async fn act(
&self,
request: &ActionRequest,
ctx: &ControlContext,
) -> Result<ActionResult, ControlError> {
let display = ctx.display.as_str();
let profile = ctx.profile_path.as_deref();
let mut completed = 0usize;
for action in &request.actions {
apply_action(display, profile, action).await?;
let pause = action_pause_ms(action);
if pause > 0 {
sleep(Duration::from_millis(pause)).await;
}
completed += 1;
}
if request.settle_ms > 0 {
sleep(Duration::from_millis(u64::from(request.settle_ms))).await;
}
let observation = if request.observe {
Some(observe_display(display).await?)
} else {
None
};
Ok(ActionResult {
completed,
observation,
})
}
async fn browser(
&self,
request: &BrowserRequest,
ctx: &ControlContext,
) -> Result<CdpPage, ControlError> {
legacy_browser(request, ctx).await
}
async fn start_recording(
&self,
request: &RecordingRequest,
ctx: &ControlContext,
) -> Result<RecordingSession, ControlError> {
start_cdp_recorder(&ctx.display, ctx.profile_path.as_deref(), &request.skill_id).await?;
Ok(RecordingSession {
skill_id: request.skill_id.clone(),
output_dir: teach_trajectory_dir(&request.skill_id),
})
}
async fn stop_recording(
&self,
request: &RecordingRequest,
_ctx: &ControlContext,
) -> Result<(), ControlError> {
stop_cdp_recorder(&request.skill_id).await
}
async fn collect_recording(
&self,
request: &RecordingRequest,
_ctx: &ControlContext,
) -> Result<RecordingResult, ControlError> {
Ok(RecordingResult {
events: collect_cdp_events(&request.skill_id).await,
})
}
}
async fn legacy_browser(
request: &BrowserRequest,
ctx: &ControlContext,
) -> Result<CdpPage, ControlError> {
let mut body = serde_json::json!({
"action": request.action,
"ensure": request.ensure,
"display": ctx.display,
"port": devtools_port(&ctx.display),
});
if let Some(profile) = &ctx.profile_path {
body["profile"] = serde_json::json!(profile);
}
if let Some(url) = &request.url {
body["url"] = serde_json::json!(url);
}
if let Some(text) = &request.text {
body["text"] = serde_json::json!(text);
}
if let Some(key) = &request.key {
body["key"] = serde_json::json!(key);
}
if let Some(ms) = request.ms {
body["ms"] = serde_json::json!(ms);
}
if let Some(selector) = &request.selector {
body["selector"] = serde_json::json!(selector);
}
if let Some(wait_ms) = request.wait_ms {
body["waitMs"] = serde_json::json!(wait_ms);
}
let timeout_ms = if request.action == "click" {
request.wait_ms.unwrap_or(45_000).min(120_000) + 25_000
} else {
20_000
};
let argv = cdp_command_on(&ctx.display, ctx.profile_path.as_deref(), &body);
let raw = run_stdout_text_timeout(&argv, timeout_ms)
.await
.map_err(ControlError::internal)?;
Ok(parse_cdp_page(&raw))
}
pub async fn observe_display(display: &str) -> Result<ComputerObservation, ControlError> {
let frame = capture_stdout(&screenshot_command_on(display))
.await
.map_err(|_| ControlError::Internal("screenshot failed".into()))?;
// The capture is a JPEG of the root window, so the frame carries the real
// geometry; the constants are only a fallback for an undecodable frame.
let (width, height) =
image_dimensions(&frame).unwrap_or((DEFAULT_SCREEN_WIDTH, DEFAULT_SCREEN_HEIGHT));
let ((cursor, window), elements) =
tokio::join!(run_pointer_state(display), run_window_list(display));
Ok(observation_with_elements(
observation_from_png(frame, width, height, cursor, window),
elements,
))
}
async fn run_window_list(display: &str) -> Vec<lazyboy_contracts::UiElement> {
let Ok(raw) = run_stdout_text(&window_list_command_on(display)).await else {
return Vec::new();
};
parse_ui_elements(&raw)
}
async fn run_pointer_state(
display: &str,
) -> (
Option<lazyboy_contracts::CursorPosition>,
Option<lazyboy_contracts::ActiveWindow>,
) {
let Ok(raw) = run_stdout_text(&pointer_state_command_on(display)).await else {
return (None, None);
};
parse_pointer_state(&raw)
}
pub async fn apply_action(
display: &str,
profile: Option<&str>,
action: &ComputerAction,
) -> Result<(), ControlError> {
match action {
ComputerAction::Wait { ms } => {
sleep(Duration::from_millis(u64::from(*ms))).await;
Ok(())
}
ComputerAction::Open { path } => spawn_detached(&open_argv_on(display, profile, path))
.await
.map_err(ControlError::internal),
ComputerAction::Focus { .. } => run_xdotool(display, action).await,
ComputerAction::Launch { application, uri } => {
let argv = launch_argv_on(display, profile, application, uri.as_deref())
.ok_or(ControlError::Unsupported)?;
spawn_detached(&argv).await.map_err(ControlError::internal)
}
ComputerAction::Ref {
verb,
target,
ref_kind,
text,
} => apply_ref(display, profile, *verb, target, ref_kind, text.as_deref()).await,
other => run_xdotool(display, other).await,
}
}
async fn run_xdotool(display: &str, action: &ComputerAction) -> Result<(), ControlError> {
let argv = xdotool_argv_on(display, action).ok_or(ControlError::Unsupported)?;
let output = run_output(&argv).await.map_err(ControlError::internal)?;
if output.status.success() {
Ok(())
} else {
Err(ControlError::internal(String::from_utf8_lossy(
&output.stderr,
)))
}
}
async fn apply_ref(
display: &str,
profile: Option<&str>,
verb: RefVerb,
target: &str,
kind: &str,
text: Option<&str>,
) -> Result<(), ControlError> {
let action = match verb {
RefVerb::Click => "click",
RefVerb::SetValue => "type",
RefVerb::Focus => "focus",
};
let mut request = serde_json::json!({
"action": action,
"selector": target,
"display": display,
"ensure": false,
});
if let Some(text) = text {
request["text"] = serde_json::json!(text);
}
let argv = if kind == "dom" {
cdp_command_on(display, profile, &request)
} else {
a11y_command_on(display, &request)
};
let raw = run_stdout_text(&argv)
.await
.map_err(ControlError::internal)?;
let ok = if kind == "dom" {
parse_cdp_page(&raw).ok
} else {
parse_a11y_page(&raw).ok
};
if ok {
Ok(())
} else {
Err(ControlError::internal(raw))
}
}
pub(crate) async fn start_cdp_recorder(
display: &str,
profile: Option<&str>,
skill_id: &str,
) -> Result<(), ControlError> {
if skill_id.trim().is_empty() {
return Err(ControlError::InvalidAction(
"recording needs a skill id".into(),
));
}
let argv = cdp_record_command_on(display, profile, skill_id);
run_output(&argv).await.map_err(ControlError::internal)?;
Ok(())
}
pub(crate) async fn stop_cdp_recorder(skill_id: &str) -> Result<(), ControlError> {
let argv = cdp_record_stop_command(skill_id);
let _ = run_output(&argv).await;
Ok(())
}
pub(crate) async fn collect_cdp_events(skill_id: &str) -> Vec<serde_json::Value> {
let path = teach_recorder_output(skill_id);
let Ok(text) = tokio::fs::read_to_string(&path).await else {
return Vec::new();
};
let _ = tokio::fs::remove_file(&path).await;
text.lines()
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
.filter(|event| event.get("t").and_then(serde_json::Value::as_str) != Some("recorder"))
.collect()
}

7
crates/control/src/lib.rs
View File

@ -1,14 +1,12 @@
mod a11y;
mod actions;
mod cdp;
mod browser_page;
mod controller;
mod cua;
mod lease;
mod legacy;
mod observe;
mod overlay;
mod path;
mod process;
mod sandbox;
mod screen;
mod takeover;
@ -16,11 +14,10 @@ mod x11;
pub use a11y::*;
pub use actions::*;
pub use cdp::*;
pub use browser_page::*;
pub use controller::*;
pub use cua::{CuaClient, CuaController, TranslatedAction, translate_action};
pub use lease::*;
pub use legacy::LegacyController;
pub use observe::*;
pub use overlay::*;
pub use path::*;

4
crates/control/src/observe.rs
View File

@ -26,7 +26,7 @@ pub fn observation_from_png(
}
/// Pixel size of a captured frame, or `None` when the bytes are not decodable.
/// Each driver captures with a different codec (the legacy pipeline emits JPEG
/// Capture inputs may use different codecs (imported frames may be JPEG
/// from `xwd | convert`, the Cua driver writes PNG), so the dimensions have to
/// come from the frame itself rather than from a configured constant.
pub fn image_dimensions(bytes: &[u8]) -> Option<(u32, u32)> {
@ -138,7 +138,7 @@ mod tests {
.unwrap();
assert_eq!(image_dimensions(&png.into_inner()), Some((96, 48)));
// The legacy driver captures JPEG, so a hardcoded size would drift the
// Imported captures can use JPEG, so a hardcoded size would drift the
// moment the Xvfb geometry changes.
let mut jpeg = Cursor::new(Vec::new());
JpegEncoder::new_with_quality(&mut jpeg, 60)

71
crates/control/src/process.rs
View File

@ -1,71 +0,0 @@
use std::process::Stdio;
use std::time::Duration;
use tokio::io::AsyncReadExt;
use tokio::process::Command;
pub async fn run_output(argv: &[String]) -> Result<std::process::Output, String> {
if argv.is_empty() {
return Err("empty command".into());
}
Command::new(&argv[0])
.args(&argv[1..])
.output()
.await
.map_err(|error| error.to_string())
}
pub async fn run_stdout_text(argv: &[String]) -> Result<String, String> {
let output = run_output(argv).await?;
Ok(if output.stdout.is_empty() {
String::from_utf8_lossy(&output.stderr).into_owned()
} else {
String::from_utf8_lossy(&output.stdout).into_owned()
})
}
pub async fn run_stdout_text_timeout(argv: &[String], timeout_ms: u64) -> Result<String, String> {
tokio::time::timeout(
Duration::from_millis(timeout_ms.max(100)),
run_stdout_text(argv),
)
.await
.map_err(|_| "timed out".to_string())?
}
pub async fn capture_stdout(argv: &[String]) -> Result<Vec<u8>, String> {
if argv.is_empty() {
return Err("empty command".into());
}
let mut child = Command::new(&argv[0])
.args(&argv[1..])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|error| error.to_string())?;
let mut stdout = Vec::new();
if let Some(mut pipe) = child.stdout.take() {
pipe.read_to_end(&mut stdout)
.await
.map_err(|error| error.to_string())?;
}
let status = child.wait().await.map_err(|error| error.to_string())?;
if !status.success() || stdout.is_empty() {
return Err("command failed".into());
}
Ok(stdout)
}
pub async fn spawn_detached(argv: &[String]) -> Result<(), String> {
if argv.is_empty() {
return Err("empty command".into());
}
Command::new(&argv[0])
.args(&argv[1..])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.map_err(|error| error.to_string())?;
Ok(())
}

8
crates/control/src/sandbox.rs
View File

@ -129,6 +129,10 @@ pub struct EnsureScreenRequest {
pub slot: u32,
pub profile_path: String,
pub bot_id: String,
#[serde(default)]
pub bot_name: String,
#[serde(default)]
pub bot_color: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@ -140,6 +144,8 @@ pub struct EnsureScreenResult {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ActionResult {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub clipboard_text: Option<String>,
pub completed: usize,
pub observation: Option<ComputerObservation>,
}
@ -260,7 +266,7 @@ pub trait SandboxProvider: Send + Sync {
computer: &ComputerRef,
request: BrowserRequest,
context: &AdapterContext,
) -> Result<crate::CdpPage, SandboxError> {
) -> Result<crate::BrowserPage, SandboxError> {
let _ = (computer, request, context);
Err(SandboxError::message("browser is unavailable"))
}

315
crates/control/src/x11.rs
View File

@ -1,11 +1,11 @@
use lazyboy_contracts::{ComputerAction, PointerButton, PointerType, ScrollDirection};
use lazyboy_contracts::{ComputerAction, PointerType};
pub fn is_browser_title(title: &str) -> bool {
let title = title.to_lowercase();
title.contains("chromium") || title.contains("chrome")
}
use crate::screen::{PRIMARY_DISPLAY, normalize_display};
use crate::screen::normalize_display;
pub const HOME: &str = "/home/lazyboy";
@ -13,138 +13,6 @@ fn display_env(display: &str) -> String {
format!("DISPLAY={}", normalize_display(display))
}
pub fn xdotool_argv(action: &ComputerAction) -> Option<Vec<String>> {
xdotool_argv_on(PRIMARY_DISPLAY, action)
}
pub fn xdotool_argv_on(display: &str, action: &ComputerAction) -> Option<Vec<String>> {
let mut argv = vec!["env".into(), display_env(display), "xdotool".into()];
match action {
ComputerAction::Pointer {
x,
y,
pointer_type,
button,
} => {
let button_n = match button.unwrap_or(PointerButton::Left) {
PointerButton::Left => "1",
PointerButton::Middle => "2",
PointerButton::Right => "3",
};
match pointer_type {
PointerType::Move => {
argv.extend([
"mousemove".into(),
"--sync".into(),
"--".into(),
x.to_string(),
y.to_string(),
]);
}
PointerType::Click => {
argv.extend([
"mousemove".into(),
"--sync".into(),
"--".into(),
x.to_string(),
y.to_string(),
"click".into(),
"--delay".into(),
"40".into(),
button_n.into(),
]);
}
PointerType::Down => {
argv.extend([
"mousemove".into(),
"--".into(),
x.to_string(),
y.to_string(),
"mousedown".into(),
button_n.into(),
]);
}
PointerType::Up => {
argv.extend([
"mousemove".into(),
"--".into(),
x.to_string(),
y.to_string(),
"mouseup".into(),
button_n.into(),
]);
}
}
}
ComputerAction::Key { key, modifiers } => {
let combo = match modifiers {
Some(items) if !items.is_empty() => format!("{}+{key}", items.join("+")),
_ => key.clone(),
};
argv.extend(["key".into(), "--clearmodifiers".into(), combo]);
}
ComputerAction::Clipboard { text } => {
let quoted = shell_single_quote(text);
if looks_like_typed_ascii(text) {
argv.extend([
"type".into(),
"--delay".into(),
"16".into(),
"--".into(),
text.clone(),
]);
} else {
return Some(vec![
"env".into(),
display_env(display),
"bash".into(),
"-lc".into(),
format!(
"printf %s {quoted} | xclip -selection clipboard && xdotool key --clearmodifiers ctrl+v"
),
]);
}
}
ComputerAction::Scroll { direction, amount } => {
let button = match direction {
ScrollDirection::Up => "4",
ScrollDirection::Down => "5",
};
argv.extend([
"click".into(),
"--repeat".into(),
amount.unwrap_or(12).to_string(),
"--delay".into(),
"15".into(),
button.into(),
]);
}
ComputerAction::Focus { title } => {
let quoted = shell_single_quote(title);
return Some(vec![
"env".into(),
display_env(display),
"bash".into(),
"-lc".into(),
format!(
"wmctrl -a {quoted} || xdotool search --name {quoted} windowactivate --sync windowfocus"
),
]);
}
ComputerAction::Wait { .. }
| ComputerAction::Open { .. }
| ComputerAction::Launch { .. }
| ComputerAction::Ref { .. } => {
return None;
}
}
Some(argv)
}
fn shell_single_quote(value: &str) -> String {
format!("'{}'", value.replace('\'', "'\\''"))
}
fn looks_like_typed_ascii(text: &str) -> bool {
text.len() <= 48
&& text
@ -164,73 +32,11 @@ pub fn action_pause_ms(action: &ComputerAction) -> u64 {
ComputerAction::Clipboard { .. } | ComputerAction::Key { .. } => 35,
ComputerAction::Focus { .. } => 90,
ComputerAction::Open { .. } | ComputerAction::Launch { .. } => 220,
ComputerAction::Wait { .. } => 0,
ComputerAction::Wait { .. } | ComputerAction::CopySelection => 0,
ComputerAction::Ref { .. } => 55,
}
}
pub fn pointer_state_command_on(display: &str) -> Vec<String> {
vec![
"env".into(),
display_env(display),
"python3".into(),
"-c".into(),
r#"
import json, subprocess
def out(args):
try:
return subprocess.check_output(args, stderr=subprocess.DEVNULL, text=True).strip()
except Exception:
return ""
vals = {}
for line in out(["xdotool", "getmouselocation", "--shell"]).splitlines():
if "=" in line:
key, value = line.split("=", 1)
vals[key] = value
wid = out(["xdotool", "getactivewindow"])
title = out(["xdotool", "getwindowname", wid]) if wid else ""
print(json.dumps({"x": int(vals.get("X") or 0), "y": int(vals.get("Y") or 0), "id": wid, "title": title}))
"#
.into(),
]
}
pub fn window_list_command_on(display: &str) -> Vec<String> {
vec![
"env".into(),
display_env(display),
"python3".into(),
"-c".into(),
r#"
import json, subprocess
def out(args):
try:
return subprocess.check_output(args, stderr=subprocess.DEVNULL, text=True)
except Exception:
return ""
els = []
n = 1
for line in out(["wmctrl", "-lG"]).splitlines():
parts = line.split(None, 7)
if len(parts) < 7:
continue
try:
x, y, w, h = int(parts[2]), int(parts[3]), int(parts[4]), int(parts[5])
except ValueError:
continue
if w < 32 or h < 16:
continue
title = parts[7].strip() if len(parts) > 7 else parts[6]
if not title or title in ("Desktop", "xfce4-panel"):
continue
els.append({"id": n, "title": title[:80], "kind": "window", "x": max(0, x), "y": max(0, y), "w": w, "h": h})
n += 1
print(json.dumps(els))
"#
.into(),
]
}
pub fn parse_ui_elements(raw: &str) -> Vec<lazyboy_contracts::UiElement> {
let value: serde_json::Value =
serde_json::from_str(raw.trim()).unwrap_or(serde_json::Value::Null);
@ -267,39 +73,6 @@ pub fn parse_ui_elements(raw: &str) -> Vec<lazyboy_contracts::UiElement> {
.collect()
}
pub fn parse_pointer_state(
raw: &str,
) -> (
Option<lazyboy_contracts::CursorPosition>,
Option<lazyboy_contracts::ActiveWindow>,
) {
let value: serde_json::Value =
serde_json::from_str(raw.trim()).unwrap_or(serde_json::Value::Null);
let cursor = match (
value.get("x").and_then(serde_json::Value::as_i64),
value.get("y").and_then(serde_json::Value::as_i64),
) {
(Some(x), Some(y)) => Some(lazyboy_contracts::CursorPosition {
x: x as i32,
y: y as i32,
}),
_ => None,
};
let window = value
.get("id")
.and_then(serde_json::Value::as_str)
.filter(|id| !id.is_empty())
.map(|id| lazyboy_contracts::ActiveWindow {
id: id.to_string(),
title: value
.get("title")
.and_then(serde_json::Value::as_str)
.filter(|title| !title.is_empty())
.map(str::to_string),
});
(cursor, window)
}
pub fn open_argv_on(display: &str, profile: Option<&str>, path: &str) -> Vec<String> {
if path.starts_with("http://") || path.starts_with("https://") {
return browser_argv(display, profile, Some(path));
@ -343,98 +116,16 @@ fn browser_argv(display: &str, profile: Option<&str>, uri: Option<&str>) -> Vec<
argv.push(format!("LAZYBOY_BROWSER_PROFILE={profile}"));
}
argv.push("lazyboy-browser".into());
argv.push(format!(
"--remote-debugging-port={}",
crate::devtools_port(display)
));
argv.push("--remote-allow-origins=*".into());
if let Some(uri) = uri {
argv.push(uri.into());
}
argv
}
pub fn screenshot_command_on(display: &str) -> Vec<String> {
vec![
"bash".into(),
"-lc".into(),
format!(
"DISPLAY={} xwd -root -silent | convert xwd:- -quality 60 jpeg:-",
normalize_display(display)
),
]
}
/// Native clipboard path; text is supplied on stdin, never process arguments.
pub fn paste_command_on(display: &str) -> Vec<String> {
vec![
"env".into(),
display_env(display),
"python3".into(),
"-c".into(),
include_str!("clipboard.py").into(),
]
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn click_maps_to_xdotool() {
let argv = xdotool_argv(&ComputerAction::Pointer {
x: 12,
y: 40,
pointer_type: PointerType::Click,
button: Some(PointerButton::Left),
})
.unwrap();
assert!(argv.contains(&"click".into()));
assert!(argv.contains(&"12".into()));
}
#[test]
fn pointer_up_moves_before_release() {
let argv = xdotool_argv(&ComputerAction::Pointer {
x: 80,
y: 90,
pointer_type: PointerType::Up,
button: Some(PointerButton::Left),
})
.unwrap();
assert!(argv.contains(&"mousemove".into()));
assert!(argv.contains(&"80".into()));
assert!(argv.contains(&"mouseup".into()));
}
#[test]
fn extra_display_is_injected_into_input_commands() {
let argv = xdotool_argv_on(
":2",
&ComputerAction::Pointer {
x: 4,
y: 8,
pointer_type: PointerType::Move,
button: None,
},
)
.unwrap();
assert!(argv.contains(&"DISPLAY=:2".into()));
let shot = screenshot_command_on(":3");
assert!(shot.last().unwrap().contains("DISPLAY=:3"));
assert!(shot.last().unwrap().contains("jpeg:-"));
let browser = launch_argv_on(
":2",
Some("/home/lazyboy/.browser-profiles/bots/a"),
"browser",
None,
)
.unwrap();
assert!(browser.contains(&"DISPLAY=:2".into()));
assert!(browser.iter().any(|item| item.contains("bots/a")));
assert!(browser.contains(&"--remote-debugging-port=9223".into()));
}
#[test]
fn parses_window_list_elements() {
let elements =

2
crates/controld/src/main.rs
View File

@ -158,7 +158,7 @@ async fn act(
);
match app.controller.act(&request, &ctx).await {
Ok(result) => {
let mut body = serde_json::json!({ "completed": result.completed });
let mut body = serde_json::json!({ "completed": result.completed, "clipboardText": result.clipboard_text });
if let Some(observation) = result.observation
&& let serde_json::Value::Object(map) = observation_to_control_json(&observation)
{

11
crates/sandbox/src/docker.rs
View File

@ -6,7 +6,7 @@ use lazyboy_contracts::{
SandboxKind,
};
use lazyboy_control::{
ActionRequest, ActionResult, AdapterContext, BrowserRequest, CdpPage, CommandRequest,
ActionRequest, ActionResult, AdapterContext, BrowserPage, BrowserRequest, CommandRequest,
CommandResult, ComputerRef, EnsureScreenRequest, EnsureScreenResult, FileEntry,
ProvisionRequest, RecordingRequest, RecordingResult, RecordingSession, SandboxError,
SandboxProvider, ScreenSession, image_dimensions, observation_from_png,
@ -241,11 +241,18 @@ impl SandboxProvider for DockerSandbox {
.send()
.await
.map_err(|error| SandboxError::message(error.to_string()))?;
let response = response
.error_for_status()
.map_err(|error| SandboxError::message(error.to_string()))?;
let body: Value = response
.json()
.await
.map_err(|error| SandboxError::message(error.to_string()))?;
Ok(ActionResult {
clipboard_text: body
.get("clipboardText")
.and_then(Value::as_str)
.map(str::to_string),
completed: body.get("completed").and_then(Value::as_u64).unwrap_or(0) as usize,
observation: if body.get("png_base64").is_some() {
Some(decode_observation(&body)?)
@ -260,7 +267,7 @@ impl SandboxProvider for DockerSandbox {
computer: &ComputerRef,
request: BrowserRequest,
context: &AdapterContext,
) -> Result<CdpPage, SandboxError> {
) -> Result<BrowserPage, SandboxError> {
let response = self
.client
.post(self.url(&format!("/computers/{}/browser", computer.id)))

9
crates/sandbox/src/fake.rs
View File

@ -4,7 +4,7 @@ use std::sync::Mutex;
use async_trait::async_trait;
use lazyboy_contracts::{ComputerObservation, SandboxKind};
use lazyboy_control::{
ActionRequest, ActionResult, AdapterContext, BrowserRequest, CdpPage, CommandRequest,
ActionRequest, ActionResult, AdapterContext, BrowserPage, BrowserRequest, CommandRequest,
CommandResult, ComputerRef, FileEntry, ProvisionRequest, RecordingRequest, RecordingResult,
RecordingSession, SandboxError, SandboxProvider, ScreenSession, observation_from_png,
};
@ -94,6 +94,7 @@ impl SandboxProvider for FakeSandbox {
context: &AdapterContext,
) -> Result<ActionResult, SandboxError> {
Ok(ActionResult {
clipboard_text: None,
completed: request.actions.len(),
observation: if request.observe {
Some(self.observe(computer, context).await?)
@ -108,12 +109,12 @@ impl SandboxProvider for FakeSandbox {
_computer: &ComputerRef,
_request: BrowserRequest,
_context: &AdapterContext,
) -> Result<CdpPage, SandboxError> {
Ok(CdpPage {
) -> Result<BrowserPage, SandboxError> {
Ok(BrowserPage {
ok: true,
url: "about:blank".into(),
title: "fake".into(),
..CdpPage::default()
..BrowserPage::default()
})
}

111
crates/supervisor/src/docker.rs
View File

@ -15,8 +15,7 @@ use futures_util::StreamExt;
use lazyboy_control::{
ActionRequest, BrowserRequest, CommandRequest, CommandResult, EnsureScreenRequest,
EnsureScreenResult, HOME, RecordingRequest, ScreenTarget, TEAM_SCREEN_LIMIT, normalize_display,
normalize_workspace_path, pointer_state_command_on, screen_layout, screenshot_command_on,
window_list_command_on,
normalize_workspace_path, screen_layout,
};
use tokio::time::{Duration, sleep};
@ -300,87 +299,7 @@ impl DockerHost {
id: &str,
target: &ScreenTarget,
) -> Result<ObservePayload, String> {
let mut body = if let Ok(value) = self.control_observe_json(id, target).await {
value
} else {
let (stdout, stderr, code) = self
.exec_raw(
id,
&screenshot_command_on(&target.display),
None,
target,
None,
)
.await?;
if code != 0 {
return Err(String::from_utf8_lossy(&stderr).into_owned());
}
let mut body = serde_json::json!({
"png_base64": base64::engine::general_purpose::STANDARD.encode(&stdout)
});
if let Ok(meta) = self.pointer_state(id, target).await
&& let serde_json::Value::Object(map) = meta
&& let Some(obj) = body.as_object_mut()
{
if let (Some(x), Some(y)) = (map.get("x"), map.get("y")) {
obj.insert("cursor".into(), serde_json::json!({ "x": x, "y": y }));
}
if map
.get("id")
.and_then(serde_json::Value::as_str)
.is_some_and(|id| !id.is_empty())
{
obj.insert(
"activeWindow".into(),
serde_json::json!({ "id": map.get("id"), "title": map.get("title") }),
);
}
}
body
};
self.attach_window_elements(id, target, &mut body).await;
ObservePayload::from_json(body)
}
async fn attach_window_elements(
&self,
id: &str,
target: &ScreenTarget,
body: &mut serde_json::Value,
) {
if body
.get("elements")
.and_then(serde_json::Value::as_array)
.is_some_and(|items| !items.is_empty())
{
return;
}
let Ok(result) = self
.exec_argv(id, &window_list_command_on(&target.display), None, target)
.await
else {
return;
};
if result.code != 0 {
return;
}
if let Ok(elements) = serde_json::from_str::<serde_json::Value>(&result.stdout) {
body["elements"] = elements;
}
}
async fn pointer_state(
&self,
id: &str,
target: &ScreenTarget,
) -> Result<serde_json::Value, String> {
let result = self
.exec_argv(id, &pointer_state_command_on(&target.display), None, target)
.await?;
if result.code != 0 {
return Err(result.stderr);
}
serde_json::from_str(&result.stdout).map_err(|error| error.to_string())
ObservePayload::from_json(self.control_observe_json(id, target).await?)
}
pub async fn act(&self, id: &str, request: ActionRequest) -> Result<serde_json::Value, String> {
@ -453,7 +372,14 @@ impl DockerHost {
if name.is_empty() {
return Err("computer container has no name".into());
}
self.attach_screen_network(&name, network).await;
let attached = info
.network_settings
.as_ref()
.and_then(|settings| settings.networks.as_ref())
.is_some_and(|networks| networks.contains_key(network));
if !attached {
self.attach_screen_network(&name, network).await?;
}
return Ok(name);
}
sleep(Duration::from_millis(100)).await;
@ -461,7 +387,13 @@ impl DockerHost {
Err("computer is not running".into())
}
async fn attach_screen_network(&self, name: &str, network: &str) {
async fn attach_screen_network(&self, name: &str, network: &str) -> Result<(), String> {
// Refuse a missing network before connect: Docker can otherwise retain
// a broken attachment that prevents the container's next restart.
self.docker
.inspect_network::<String>(network, None)
.await
.map_err(|error| format!("screen network {network} is unavailable: {error}"))?;
let result = self
.docker
.connect_network(
@ -475,9 +407,10 @@ impl DockerHost {
if let Err(error) = result {
let text = error.to_string();
if !text.to_lowercase().contains("already") {
tracing::warn!("attach {name} to screen network {network}: {text}");
return Err(format!("attach {name} to screen network {network}: {text}"));
}
}
Ok(())
}
async fn published_host_port(&self, id: &str, view_port: u16) -> Result<String, String> {
@ -516,9 +449,11 @@ impl DockerHost {
) -> Result<EnsureScreenResult, String> {
let layout = screen_layout(request.slot).map_err(|error| error.to_string())?;
let script = format!(
"lazyboy-screen ensure {} {}",
"lazyboy-screen ensure {} {} {} {}",
request.slot,
shell_single_quote(&request.profile_path)
shell_single_quote(&request.profile_path),
shell_single_quote(&request.bot_name),
shell_single_quote(&request.bot_color)
);
let result = self
.exec_argv(

2
docker-compose.cua.yml
View File

@ -1,4 +1,4 @@
# Opt in without changing the default legacy rollout or overwriting its image.
# Optional separate Cua image tag; the base stack also uses Cua exclusively.
# docker compose -f docker-compose.yml -f docker-compose.cua.yml up -d --build
services:
computer:

2
docker-compose.yml
View File

@ -51,7 +51,7 @@ services:
LAZYBOY_COMPUTER_MEMORY_MB: ${LAZYBOY_COMPUTER_MEMORY_MB:-2048}
LAZYBOY_COMPUTER_PIDS: ${LAZYBOY_COMPUTER_PIDS:-2048}
LAZYBOY_COMPUTER_SUDO: ${LAZYBOY_COMPUTER_SUDO:-false}
LAZYBOY_COMPUTER_DRIVER: ${LAZYBOY_COMPUTER_DRIVER:-legacy}
LAZYBOY_COMPUTER_DRIVER: cua
LAZYBOY_LXCFS_ROOT: /var/lib/lxcfs
SUPERVISOR_BIND: 0.0.0.0:7091
DATA_DIR: /data

85
docs/agent-cursor.md Normal file
View File

@ -0,0 +1,85 @@
# Agent cursor on the shared desktop
The computer image enables Cua's native cursor overlay. Cua draws it in X11, so
both the embedded viewer and an enlarged/direct VNC viewer receive the same
cursor and name. There is no separate frontend cursor or coordinate replay.
The API supplies the bot's current name and `avatarColor` when ensuring/restoring its screen. The
screen launcher atomically writes these settings in the display's temporary runtime
directory. The Cua client uses that name as its public session label, keeping it
consistent across observation, native/browser input, and session revival.
Display sockets provide isolation even when two agents have the same name.
Renaming starts a new public session: take a fresh browser snapshot before using
browser references again. Mutations using stale references are not replayed.
Calls that explicitly provide a session retain their explicit label; unnamed
screens retain `lazyboy-N` for compatibility.
Cua owns the cursor animation, action feedback, idle hiding, and label truncation.
Long names may be shortened in its badge. The cursor is a synthetic agent cursor,
independent of the viewer's local mouse pointer. No extra input is injected to
animate it.
## Selected agent colors
The cursor fill, glow, and name badge use the agent's existing `#RRGGBB` color.
Saving the agent's appearance also refreshes the color on an existing running
screen. This cosmetic command does not start a stopped computer, focus an app,
rename the Cua session, or replay input. Ensure/restore reapplies the persisted
setting if the running desktop could not be reached during a save.
`image/computer/cua-color.patch` adds a small override to Cua's shared color
function. The launcher sets `LAZYBOY_CURSOR_COLOR_FILE` separately for each display
daemon. `cua-color.rs` reads and caches that bounded hex value for 100 ms; missing
or invalid values retain Cua's original palette. Both cursor artwork and its
badge already consume the shared color function, so their styling stays aligned.
Names/session identities do not change when the color changes.
## Chinese names
Official Cua Driver 0.23.2 binaries embed Inter in their session badge renderer.
Inter has no Chinese glyphs, and the renderer does not consult fontconfig. Merely
installing system CJK fonts does not fix this.
The computer Dockerfile builds the pinned 0.23.2 source with the existing Huninn
2.1 UI font in that embedded asset. Both downloads have SHA-256 checks. Apart from
the cosmetic color override, driver code and its locked dependency graph remain unchanged. The build enables
`portal-input` and retains the release's Rust toolchain. Cua's MIT and the font's
OFL notices ship in `/usr/share/licenses/cua-driver/`.
The initial image build now also compiles Cua; subsequent builds reuse its own
Cargo cache. Existing containers require recreation from the new image to enable
the overlay and use the embedded Chinese font. This change does not deploy or
replace production computers automatically.
## Verification
- Workspace tests: 205 passed; the separate saved-login integration test is ignored
by the unit-test suite.
- Workspace Clippy with warnings denied, Rust formatting, and launcher shell syntax.
- Two standalone color-module tests passed (hex validation, black/white, live
refresh, missing-file fallback, and isolation between display files).
- Complete desktop smoke passed, including native/browser input, terminal and
clipboard Unicode, dual-display isolation, session recovery, browser-cookie
persistence through restart, and the named-cursor test.
- Saved-login integration passed against real HTTPS fields without submission.
- English and Chinese labels were visually verified in desktop screenshots.
A separate read-only VNC framebuffer capture also showed the Chinese name and
synthetic pointer in the actual VNC stream, separate from the OS pointer.
- VNC pixel assertions found the exact selected RGB values `#8B5CF6`, `#22C55E`,
and `#E11D48` in the cursor while preserving the daemon PID and session name.
- An isolated API/supervisor/database integration test verified that persisted
`avatarColor` reaches a newly booted screen, PATCH updates the running cursor,
and editing a stopped agent does not start its computer. Test resources were removed.
Verified locally on 2026-09-08, Linux arm64. Image:
`lazyboy/computer:cua-cursor-color`,
`sha256:5c6e7f6befa7d9b69d6b6ff56a29692dd08854ad1273e939020ac06370ee85e6`.
Production services and existing application computers have not been replaced.
Deploy the updated API/supervisor and recreate desktops from this image together.
Color verification: `/tmp/lazyboy-cursor-color-tests.log`,
`/tmp/lazyboy-cursor-color-clippy.log`, `/tmp/lazyboy-cursor-color-smoke.log`,
`/tmp/lazyboy-cursor-color-api-test.log`, and
`/tmp/lazyboy-cursor-color-smoke/cua-cursor-green.png`.
The preceding name/font verification and saved-login run remain in
`/tmp/lazyboy-cursor-smoke.log` and `/tmp/lazyboy-cursor-login.log`.

30
docs/agent-experience.md
View File

@ -29,32 +29,14 @@
## 終端機是一台活的 tmux
`shell` 工具以前每條命令都是一次性的 `bash -lc`：`cd` 留不住、`export` 留不住、背景起的服務
跟著一起死，模型每次都要重新走回工作目錄，遇到互動式程式就整個卡死。現在每個終端機名字對應
容器內一個 tmux session（`lazyboy-main`），AI 用的就是人在桌面上會用的那個 shell。
`shell` 透過 Cua 在共用 VNC 桌面上開啟有名稱的終端機。相同 `session` 保留工作目錄、
環境變數與背景工作；`keys: "C-c"` 會在該終端機送出中斷，`reset` 會換成乾淨的登入 shell。
有了持久終端機，模型才做得順這些事：
輸出以桌面截圖回傳。模型必須確認提示字元已回來才能輸入下一條命令；省略 `command`
即可再次查看，長輸出可以透過 Cua 捲動終端機。`wait_ms` 預設 1000、上限 10000，
等待結束不會終止命令。終端機若被關閉，下次呼叫會重新開啟，已關閉 shell 的環境不會還原。
- `python -m venv .venv && source .venv/bin/activate` 之後，下一條命令還在同一個環境裡。
- `npm run dev &` 之後可以繼續做別的，回來用 `log_lines` 讀同一個終端機的輸出。
- `ssh`、`gdb`、`psql`、Python REPL 這類會問你話的程式，用 `keys` 回它，而不是直接超時。
- 按錯 Ctrl-C 只會中斷那台終端機裡正在跑的東西，不會把整個工作環境帶走。
**完成是怎麼判定的。** API 進到容器是一次性的 exec（argv 進、stdout 出，沒有串流），所以
「跑完了沒」不是看 exec 有沒有結束，而是看終端機裡印了什麼：每次呼叫會在 shell 內 source 一個
檔名隨機的腳本，腳本用 `trap ... EXIT` 印出 `LB_END <nonce> rc=<code>`；還看到 `LB_READY`
代表上一件事已經被中斷、終端機是可用的。標記只在**行首**比對——命令本身會被 shell echo 一次，
那行也帶有標記，比錯就會以為已經跑完了。
**死了會自己站起來。** `exit`、`exec bash`、被 kill 掉都會讓 pane 不見；這時候終端機重新
`respawn`，並把剛才的工作目錄與 `export` 過的環境變數還原回去，模型不需要重新交代。
人可以隨時看同一台終端機，或直接接手（指令見
[容器內終端機](./operations.md#容器內終端機)）。這不隻是好看：AI 卡在同一個地方時，你看到的
就是它看到的那個 shell。
行為用 `python3 tests/shell-session.test.py` 驗證（10 個案例，含 exit code 還原、自我重生、
中斷後可續用）。
檔案列出、分頁讀取與寫入也使用這個可見終端機。所有這些工具都需要視覺模型與桌面控制鎖。
## 聊天是推送，不是輪詢

6
docs/architecture.md
View File

@ -59,8 +59,8 @@ lazyboy-supervisor (:7091, internal only)
├── provision / pause / resume / stop
├── CPU / memory / PID limits
└── isolated computer containers
├── Chromium + CDP
├── XFCE + AT-SPI
├── Cua Driver → Chromium / XFCE
├── visible terminal + clipboard editor
├── Xvfb + x11vnc + websockify
└── per-computer persisted home
```
@ -73,7 +73,7 @@ lazyboy-supervisor (:7091, internal only)
| `crates/api` | 對外 Axum API、Agent run、Session、排程、記憶、MCP、保險箱 |
| `crates/harness` | 模型供應商、憑證解析與語音契約 |
| `crates/supervisor` | Docker 電腦生命週期、隔離與資源上限 |
| `crates/control` | CDP、AT-SPI、X11 與畫面觀察操作 |
| `crates/control` | Cua 瀏覽器、原生視窗與共用桌面觀察；終端機與剪貼簿操作皆經 Cua |
| `crates/controld` | 電腦容器內部的 localhost 控制服務 |
| `crates/contracts` | 跨 crate 的 Bot、Run、Computer、Voice 資料契約 |
| `PostgreSQL` | 對話、run、記憶、排程、憑證與保留政策 |

2
docs/cua-compatibility.md
View File

@ -1,5 +1,7 @@
# Cua Driver compatibility (LazyBoy desktop)
> 歷史紀錄：本文描述 2026-09-07 的雙後端驗證，已非現況。目前只保留 Cua；操作方式見 [operations.md](operations.md)，最新驗證狀態見 [cua-migration-progress.md](cua-migration-progress.md)。
This report answers one question, from a real `make cua-smoke` run on 2026-09-07:
> Can Cua Driver reliably control the existing LazyBoy XFCE + Xvfb desktop container?

58
docs/cua-migration-progress.md Normal file
View File

@ -0,0 +1,58 @@
# Cua-only migration verification
Verified locally on 2026-09-08 (Linux arm64, Cua Driver 0.23.2). Source changes and local acceptance are complete. Production deployment is not part of this verification.
The subsequent named-cursor feature, Chinese badge font, and its separate image
verification are documented in [Agent cursor on the shared desktop](agent-cursor.md).
## Requirements and evidence
| Requirement | Implementation | Verification |
| --- | --- | --- |
| Remove the old control implementation | Cua is the only ComputerDriver. Deleted LegacyController, direct CDP/AT-SPI Python controllers, clipboard.py, process helpers and the tmux shell. Removed xdotool/xclip packages. | Source audit; old backend names rejected; final running image contains neither binary. |
| See what Cua is doing | Actions foreground the existing browser/native window on the bot's shared display. Shell/file tools use visible named terminals; clipboard operations use a visible GTK editor. | Native/browser/noVNC smoke; terminal and clipboard integration; actual VNC demonstration. |
| All computer actions through Cua | Browser, native pointer/key/ref actions, launch/focus, shell/file tools, clipboard and saved-login use the Cua controller. Removed blanket browser-pixel blocking so canvas/unsupported controls can use fresh screenshot coordinates through Cua. | Adapter, terminal, clipboard, login and dual-display tests; API tool-path audit found no hidden shell execution in agent computer/file tools. |
| Time on every conversation record | Every persisted message renders MessageTime, including attachment/chip messages. Date/time includes seconds, ISO datetime and full local-time tooltip. | Real user and assistant error messages inspected at desktop and 500px width; frontend tests/typecheck/build. |
| Faster/reliable connections | Reuse browser bindings; start VNC before Cua; record the daemon PID; do not inherit startup locks; skip repeated network attachment; reject missing networks before connect and fall back to host ports; recover expired Cua sessions for reads without replaying mutations. | Warm ensure reuses one daemon with an available lock; missing-network fallback leaves no stale attachment; session-expiry observation/browser recovery and mutation refusal pass. |
Infrastructure provisioning/storage/database calls and independent connected-service MCP facilities remain. They are not alternate desktop controllers. Computer, browser, terminal and agent workspace-file interactions use Cua.
## Acceptance results
Final desktop image `lazyboy/computer:cua-work`:
`sha256:4ad46205023694a26ff06f9e969d7c7e0ac43b98cae97a4483d2ba3379462117`
- `cargo test --workspace`: 204 passed; the environment-dependent login integration is ignored by default and separately passed.
- `cargo clippy --workspace --all-targets -- -D warnings`: passed.
- `cargo fmt --all --check` and `git diff --check`: passed.
- Frontend: 44 tests passed; TypeScript/Vite build passed.
- `scripts/cua-smoke-test.sh --docker --repeat 1 --image lazyboy/computer:cua-work`: passed on the final image. Covers native/browser/noVNC, terminal persistence/Unicode/interrupt/reset, clipboard Unicode/multiline/copy, two-display isolation, session expiry, and browser-cookie persistence across pause/restart.
- `COMPUTER_IMAGE=lazyboy/computer:cua-work scripts/cua-login-test.sh`: passed. Uses a trusted local HTTPS fixture in a disposable container and the production field-filling function, checks both exact values and verifies no submission.
- Manual VNC demonstration switched the target desktop from Chromium to its terminal. Teaching retained 2 window events and 3 screenshots, including the final frame. The target's recording was driven by human-style VNC input, not target-side Cua action calls.
Local measurements are samples, not production benchmarks: boot request 2.358s; warm screen URL request 0.100s; missing-network fallback 0.086s with unchanged Docker network attachments. A full container replacement/restart took 10.419s, including Docker shutdown.
## Driver compatibility fixes
- A zero CLI exit code is insufficient: `effect: refused` is treated as failure along with `status: refused`.
- Email input can refuse Cua browser typing. Its fallback resolves one uniquely labelled visible native web entry through Cua, clicks the fresh observed bounds, selects all and pastes through Cua. Duplicate labels, browser chrome and zero-size fields are rejected.
- Native type_text loses Unicode in terminals. Shell commands use ASCII Bash literals encoding UTF-8 bytes; general Unicode/multiline paste uses the Cua-operated clipboard editor. Zsh confirms multiline bracketed paste with another Enter.
- The GTK helper exposes exact clipboard text through its accessibility label because the pinned driver does not return GTK entry values.
- Expired driver sessions are revived for observations. Mutations rejected at expiry are not replayed. Expired browser bindings are classified as stale and re-bound for read requests.
- Browser semantic clicks use `dom_event`; callers still inspect results. Unsupported controls can be operated with Cua coordinates from a fresh screenshot.
## Teaching and environment limits
Cua trajectories record driver invocations, not raw human VNC clicks/keys. Teaching retains window changes and visual keyframes. The start message and model prompt describe that accurately and require review of missing/ambiguous steps. Model failure no longer claims the skill was learned. The local environment has no real model key, so model-generated playbook quality was not tested; recording persistence and missing-key handling were verified.
Local API: `http://127.0.0.1:3111`; supervisor7191. The test bot `1ae00840-ceaf-4197-957d-661df677b015` is running the final image with one Cua daemon. Generated test credentials are in the local .env; no real provider credentials were used. API login sessions are in-memory, so restarting the API requires signing in again (existing behavior).
The disposable Postgres test database uses tmpfs on15434; the pre-existing compose database volume was left intact. The separate old `lazyboy-cua-verify` container is a UI-test viewer, not the final target desktop. Temporary smoke/login containers are removed by their scripts.
## Local evidence files
- `/tmp/lazyboy-workspace-tests.log`, `/tmp/lazyboy-workspace-clippy.log`
- `/tmp/lazyboy-acceptance-smoke.log`, `/tmp/lazyboy-acceptance-login.log`
- `/tmp/lazyboy-network-fallback-test.log`, `/tmp/lazyboy-session-recovery-test.log`
- `/tmp/lazyboy-chat-time.png`, `/tmp/lazyboy-chat-mobile.png`, `/tmp/teach-vnc.png`
- `/tmp/lazyboy-web-build-final.log`, `/tmp/lazyboy-final-frontend-tests.log`

2
docs/cua-review.md
View File

@ -1,5 +1,7 @@
# Cua 遷移檢查（2026-09-07）
> 歷史紀錄：本文描述 2026-09-07 的雙後端驗證，已非現況。目前只保留 Cua；操作方式見 [operations.md](operations.md)，最新驗證狀態見 [cua-migration-progress.md](cua-migration-progress.md)。
結論：`a.md` Phase 1 規格尚未全部勾完，但 **opt-in Cua 已可在現有 XFCE + Xvfb 桌面容器使用**。生產預設仍是 `legacy`。
本次在 Apple Silicon（linux/arm64）上以 `lazyboy/computer:local` + Cua Driver **0.23.2** 重跑隔離桌面驗收。

14
docs/development.md
View File

@ -34,18 +34,12 @@ npm run build
cd ../..
node --test tests/frontend.test.mjs
python3 tests/control.test.py
python3 tests/log-rotation.test.py
python3 tests/shell-session.test.py # 持久終端機腳本，只需要 tmux
# Cua Driver 能否控制現有 XFCE + Xvfb 桌面（會建 computer image）
make cua-smoke
# 結果摘要見 docs/cua-compatibility.md、docs/cua-review.md
# 生產路徑預設仍是 legacy。要在本機明確跑 Cua：
# make cua-smoke
# docker compose -f docker-compose.yml -f docker-compose.cua.yml up -d --build
# 或 LAZYBOY_COMPUTER_DRIVER=cua 寫進 .env 後重建 supervisor 與桌面容器。
# Agent 工具 schema 不變。
# 生產路徑只使用 Cua；更新後請重建 supervisor 與桌面映像。
# Python 整合測試用 docker compose exec 連進 Postgres，自己建一次性資料庫後清掉
python3 tests/retention.test.py
@ -152,3 +146,9 @@ LazyBoy/
- [ ] 若提供容器映像，再加入 SBOM、簽章與可重現版本發布流程。
> 不建議現在顯示 CI passing、coverage、OpenSSF 或 Best Practices 徽章：目前 repository 沒有對應的公開結果，徽章會失真或直接顯示 unknown。
### Cua saved-login integration
`COMPUTER_IMAGE=lazyboy/computer:cua-work scripts/cua-login-test.sh` starts a disposable desktop, installs a test-only certificate utility, trusts a generated localhost certificate inside that container, and opens the HTTPS fixture through Cua. It runs the same field-filling function as `use_saved_login`, checks both exact fixture values, and verifies that the form was not submitted. No model key or real login is used; the container is removed on exit. Build the desktop image from the current tree first.
Cua 0.23.2 can return `effect: refused` for an Email field despite a zero CLI exit status. The adapter treats that as a failure. A classified unsupported browser typing route can use a uniquely labelled native web field from Cua, click its freshly observed bounds, and paste through the Cua-operated clipboard editor. Other errors remain errors.

40
docs/operations.md
View File

@ -57,7 +57,7 @@
| `LAZYBOY_COMPUTER_MEMORY_MB` | 每台電腦記憶體 | `2048` |
| `LAZYBOY_COMPUTER_PIDS` | 每台電腦 PID 上限 | `2048` |
| `LAZYBOY_COMPUTER_SUDO` | 容器內免密碼 sudo；重建桌面容器後生效 | `false` |
| `LAZYBOY_COMPUTER_DRIVER` | 桌面控制後端：`legacy`（預設，CDP/AT-SPI/xdotool）或 `cua`（opt-in Cua Driver）。改完需重建桌面容器。本機可用 `docker compose -f docker-compose.yml -f docker-compose.cua.yml up -d --build` | `legacy` |
| `LAZYBOY_COMPUTER_DRIVER` | 只支援 `cua`。舊後端已移除；更新映像後需重建桌面容器。 | `cua` |
| `LAZYBOY_MEMORY_ENABLED` | 長期記憶 | `true` |
完整清單與保留政策請見 [`.env.example`](../.env.example)。
@ -138,39 +138,13 @@ LAZYBOY_RUN_HARD_MINUTES=240 # 時間保險絲
## 容器內終端機
AI 的 `shell` 不再每條命令開一個新的 `bash -lc`，而是用**有名字的 tmux 終端機**：工作目錄、
`export`、背景程序、互動式程式都留在原處，跟人用同一台終端機一樣。
Agent 透過 Cua 操作 VNC 上的真實終端機。`session` 指定持續使用的視窗；`command` 輸入命令，
省略則只查看畫面。`keys: "C-c"` 可中斷前景命令，`reset: true` 重新建立乾淨的登入 shell。
`cwd` 只有明確指定時才改變既有終端機的工作目錄。
| 參數 | 作用 |
| --- | --- |
| `command` | 要執行的命令 |
| `session` | 終端機名稱，預設 `main`（tmux 內是 `lazyboy-main`）；不同的工作可以分台跑 |
| `wait_ms` | 最多等多久，預設 20 秒、上限 110 秒。時間到還沒結束就回傳「還在跑」 |
| `log_lines` | 不給 `command` 時，讀終端機目前顯示的內容 |
| `keys` | 送按鍵或文字：`C-c`、`Enter`，或逐字輸入給互動式提示 |
| `reset` | 收掉這台終端機重開一台 |
| `cwd` | 先切到這個目錄再執行 |
- 長工作**不要**把 `wait_ms` 拉很長：先讓它回傳「還在跑」，之後用 `log_lines` 續讀。觀察與輪詢
不被當成鬼打牆（見[任務跑多久：輪次政策](#任務跑多久輪次政策)）。
- 終端機被 `exit`、`exec bash` 或 Ctrl-C 打死時會自動重建，並回到原本的目錄、重新載入原本
`export` 的環境變數。
- 人可以隨時看同一台終端機。終端機跑在容器裡的 uid 1000，容器名稱是 `lb-<主線>`：
```bash
docker exec -it -u 1000:1000 lb-team-local-space tmux ls
docker exec -it -u 1000:1000 lb-team-local-space tmux attach -t lazyboy-main # Ctrl-b d 離開
```
聊天時直接叫 AI「把終端機開給我看」也可以，它會用 `lazyboy-shell show` 在桌面開一個視窗，
你按同一段鍵盤就能接手。
- 桌面映像檔需要 `tmux`。映像檔裡沒有 `lazyboy-shell` 時，命令退回一次性 `bash -lc`（可用，
但不保留狀態），重建後再請 AI `reset` 一次即可：
```bash
docker compose build computer # 或 make computer
```
`wait_ms` 預設 1000 毫秒、最多 10000 毫秒。回傳的是截圖，等待時間到不代表命令完成；
請看提示字元與畫面上的結果，長工作可以稍後再次查看。長輸出可用 Cua 捲動。
直接在共用 VNC 點選同一個視窗即可觀看與接管，不需要額外開 tmux 工作階段。
## 網站連線驗證

69
image/computer/Dockerfile
View File

@ -15,16 +15,34 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
--mount=type=cache,target=/src/target,id=lazyboy-computer-release,sharing=locked \
cargo build --locked --release -p lazyboy-controld && cp /src/target/release/lazyboy-controld /lazyboy-controld
# Cua 0.23.2 embeds Inter for session badges, which has no Chinese glyphs.
# Build the same pinned release with our existing OFL Huninn UI font embedded.
# The cosmetic patch reads the bot's selected color; input/permissions are unchanged.
FROM rust:1.97.1-bookworm AS cua_driver
RUN apt-get update && apt-get install -y --no-install-recommends \
libx11-dev libxi-dev libxtst-dev libxrandr-dev libxfixes-dev libxkbcommon-dev \
libwayland-dev pkg-config patch && rm -rf /var/lib/apt/lists/*
WORKDIR /src
RUN curl -fsSL https://codeload.github.com/trycua/cua/tar.gz/refs/tags/cua-driver-rs-v0.23.2 -o /tmp/cua.tar.gz \
&& echo '151c72982c9f06bf168bd00f9611cef6760f2fc2459916dc216810ac24200f3d /tmp/cua.tar.gz' | sha256sum -c \
&& tar -xzf /tmp/cua.tar.gz --strip-components=1 \
&& rm /tmp/cua.tar.gz
RUN curl -fsSL https://github.com/justfont/open-huninn-font/releases/download/v2.1/jf-openhuninn-2.1.ttf -o /tmp/huninn.ttf \
&& echo '9d5bf4932d31fe94c18cd8cfddc98bc1b14ce10f4e354c682179db290a99c825 /tmp/huninn.ttf' | sha256sum -c \
&& cp /tmp/huninn.ttf libs/cua-driver/rust/crates/cursor-overlay/assets/Inter.ttf
WORKDIR /src/libs/cua-driver/rust
COPY image/computer/cua-color.rs crates/cursor-overlay/src/lazyboy_color.rs
COPY image/computer/cua-color.patch /tmp/cua-color.patch
RUN patch --batch --fuzz=0 -p1 < /tmp/cua-color.patch
RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
--mount=type=cache,target=/src/libs/cua-driver/rust/target,id=lazyboy-cua-cjk,sharing=locked \
cargo build --locked --release -p cua-driver --features portal-input \
&& cp target/release/cua-driver /cua-driver
FROM debian:bookworm-slim
# Keep Chromium, XFCE, CJK, AT-SPI, git. Skip Debian novnc (pulls nodejs) and
# fonts-noto-core (Latin is DejaVu/Liberation/huninn).
#
# x11-apps + imagemagick look removable and are not: the legacy driver's
# screenshot path is `xwd -root -silent | convert xwd:- … jpeg:-`
# (control/src/x11.rs screenshot_command_on), reached by legacy observe_display
# and by the supervisor's exec fallback. Nothing else in the image provides
# /usr/bin/xwd or /usr/bin/convert, so removing either breaks the default driver.
RUN printf '%s\n' \
'path-include=/usr/share/locale/zh_TW/*' \
'path-include=/usr/share/locale/zh/*' \
@ -43,25 +61,20 @@ RUN printf '%s\n' \
locales \
procps \
python3 \
python3-websocket \
sudo \
tmux \
gosu \
util-linux \
websockify \
wmctrl \
x11-apps \
x11-utils \
x11vnc \
xclip \
xdg-utils \
imagemagick \
xfce4-panel \
xfce4-settings \
xfce4-terminal \
xfconf \
xfdesktop4 \
xdotool \
xfwm4 \
xvfb \
thunar \
@ -112,33 +125,20 @@ RUN useradd --create-home --uid 1000 --shell /bin/zsh lazyboy \
&& mkdir -p /home/lazyboy /tmp/lazyboy /usr/share/lazyboy/skel /usr/share/lazyboy/xfce-skel /etc/gtk-3.0 /etc/fonts/conf.d \
&& chown -R 1000:1000 /home/lazyboy /tmp/lazyboy
# Pin Cua Driver outside the persisted /home/lazyboy bind-mount.
# Checksums are from the matching official release; select the target architecture.
ARG CUA_DRIVER_RS_VERSION=0.23.2
ARG TARGETARCH
ARG CUA_DRIVER_RS_SHA256_AMD64=01bf8339ec129cc00f4b4b2c6056ef1a7c5b52df39ff83ad17c9b16818aec500
ARG CUA_DRIVER_RS_SHA256_ARM64=be22768a207796a4bc1de50c52f32f9ef680b5e86e58c059e02eec2caba2e7bb
RUN case "$TARGETARCH" in \
amd64) cua_arch=x86_64; cua_sha="$CUA_DRIVER_RS_SHA256_AMD64" ;; \
arm64) cua_arch=arm64; cua_sha="$CUA_DRIVER_RS_SHA256_ARM64" ;; \
*) echo "Unsupported Cua architecture: $TARGETARCH" >&2; exit 1 ;; \
esac \
&& curl -fsSL -o /tmp/cua-driver.tar.gz \
"https://github.com/trycua/cua/releases/download/cua-driver-rs-v${CUA_DRIVER_RS_VERSION}/cua-driver-rs-${CUA_DRIVER_RS_VERSION}-linux-${cua_arch}-binary.tar.gz" \
&& echo "${cua_sha} /tmp/cua-driver.tar.gz" | sha256sum -c \
&& mkdir -p /usr/local/lib/cua-driver \
&& tar -xzf /tmp/cua-driver.tar.gz -C /usr/local/lib/cua-driver \
&& chmod 755 /usr/local/lib/cua-driver/cua-driver \
&& ln -sf /usr/local/lib/cua-driver/cua-driver /usr/local/bin/cua-driver \
&& rm -f /tmp/cua-driver.tar.gz \
&& cua-driver --version
# Install outside the persisted /home/lazyboy bind-mount.
COPY --from=cua_driver --chmod=755 /cua-driver /usr/local/lib/cua-driver/cua-driver
COPY --from=cua_driver /src/LICENSE.md /usr/share/licenses/cua-driver/LICENSE.md
COPY image/computer/licenses/huninn-OFL.txt /usr/share/licenses/cua-driver/huninn-OFL.txt
RUN ln -sf /usr/local/lib/cua-driver/cua-driver /usr/local/bin/cua-driver \
&& cua-driver --version
COPY --from=controld --chmod=755 /lazyboy-controld /usr/local/bin/lazyboy-controld
COPY --chmod=755 image/computer/rotate-logs.py /usr/local/bin/lazyboy-rotate-logs
COPY --chmod=755 image/computer/lazyboy-screen /usr/local/bin/lazyboy-screen
COPY --chmod=755 image/computer/lazyboy-browser /usr/local/bin/lazyboy-browser
COPY --chmod=755 image/computer/lazyboy-terminal /usr/local/bin/lazyboy-terminal
COPY --chmod=755 image/computer/lazyboy-shell /usr/local/bin/lazyboy-shell
COPY --chmod=755 image/computer/lazyboy-terminal-reset /usr/local/bin/lazyboy-terminal-reset
COPY --chmod=755 image/computer/lazyboy-clipboard /usr/local/bin/lazyboy-clipboard
COPY --chmod=644 apps/web/vnc.html /usr/share/novnc/vnc_lite.html
COPY --chmod=644 apps/web/vnc.html /usr/share/novnc/index.html
COPY --chmod=644 image/computer/fonts.conf /etc/fonts/conf.d/99-lazyboy-cjk.conf
@ -177,6 +177,11 @@ COPY --chmod=644 image/computer/cua-smoke.html /usr/share/lazyboy/cua-smoke.html
COPY --chmod=755 image/computer/cua-smoke-gtk.py /usr/local/bin/lazyboy-cua-smoke-gtk
COPY --chmod=755 scripts/cua-smoke-inner.py /usr/local/bin/lazyboy-cua-smoke
COPY --chmod=755 scripts/cua-adapter-test.py /usr/local/bin/lazyboy-cua-adapter-test
COPY --chmod=755 scripts/cua-terminal-test.py /usr/local/bin/lazyboy-cua-terminal-test
COPY --chmod=755 scripts/cua-clipboard-test.py /usr/local/bin/lazyboy-cua-clipboard-test
COPY --chmod=755 scripts/cua-session-test.py /usr/local/bin/lazyboy-cua-session-test
COPY --chmod=755 scripts/cua-cursor-test.py /usr/local/bin/lazyboy-cua-cursor-test
COPY --chmod=755 scripts/cua-cursor-color-test.py /usr/local/bin/lazyboy-cua-cursor-color-test
COPY --chmod=755 scripts/cua-isolation-test.py /usr/local/bin/lazyboy-cua-isolation-test
COPY --chmod=755 scripts/cua-smoke-test.sh /usr/local/bin/lazyboy-cua-smoke-host

23
image/computer/cua-color.patch Normal file
View File

@ -0,0 +1,23 @@
--- a/crates/cursor-overlay/src/lib.rs
+++ b/crates/cursor-overlay/src/lib.rs
@@ -5,6 +5,8 @@
//! - `MotionConfig` — glide duration, spring, dwell, idle-hide timings
//! - `CubicBezier` + `PathPlanner` — Bezier path math (ported 1:1 from C#)
//! - `OverlayCommand` — messages sent from MCP tools to the overlay thread
+
+mod lazyboy_color;
pub mod badge_glyphs;
pub mod bezier;
--- a/crates/cursor-overlay/src/theme.rs
+++ b/crates/cursor-overlay/src/theme.rs
@@ -39,6 +39,9 @@
/// hash into the former multi-cursor palette so concurrent runs are visually
/// distinct without accepting an agent-controlled styling argument.
pub fn session_fill_rgba(session_id: &str) -> [u8; 4] {
+ if let Some(color) = crate::lazyboy_color::configured_color() {
+ return color;
+ }
if session_id.is_empty() || session_id == "default" {
return DEFAULT_CURSOR_FILL;
}

114
image/computer/cua-color.rs Normal file
View File

@ -0,0 +1,114 @@
//! LazyBoy's display-local cosmetic override for Cua's built-in cursor.
//! Input/session authority is unchanged. The launcher owns the file path.
use std::io::Read;
use std::path::PathBuf;
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant};
struct ColorCache {
path: Option<PathBuf>,
checked: Option<Instant>,
color: Option<[u8; 4]>,
}
impl ColorCache {
fn get(&mut self) -> Option<[u8; 4]> {
if self
.checked
.is_some_and(|time| time.elapsed() < Duration::from_millis(100))
{
return self.color;
}
self.checked = Some(Instant::now());
self.color = self.path.as_ref().and_then(|path| {
let mut value = String::new();
std::fs::File::open(path)
.ok()?
.take(16)
.read_to_string(&mut value)
.ok()?;
parse_color(&value)
});
self.color
}
}
pub(crate) fn configured_color() -> Option<[u8; 4]> {
static CACHE: OnceLock<Mutex<ColorCache>> = OnceLock::new();
CACHE
.get_or_init(|| {
Mutex::new(ColorCache {
path: std::env::var_os("LAZYBOY_CURSOR_COLOR_FILE").map(PathBuf::from),
checked: None,
color: None,
})
})
.lock()
.ok()?
.get()
}
fn parse_color(value: &str) -> Option<[u8; 4]> {
let hex = value.trim().strip_prefix('#')?;
if hex.len() != 6 || !hex.bytes().all(|byte| byte.is_ascii_hexdigit()) {
return None;
}
Some([
u8::from_str_radix(&hex[0..2], 16).ok()?,
u8::from_str_radix(&hex[2..4], 16).ok()?,
u8::from_str_radix(&hex[4..6], 16).ok()?,
255,
])
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn accepts_user_hex_colors_including_black_and_white() {
assert_eq!(parse_color("#8b5Cf6\n"), Some([139, 92, 246, 255]));
assert_eq!(parse_color("#000000"), Some([0, 0, 0, 255]));
assert_eq!(parse_color("#FFFFFF"), Some([255, 255, 255, 255]));
for value in [
"red",
"#FFF",
"#GG0000",
"#12345678",
"#中文",
"#123456;touch /tmp/x",
] {
assert_eq!(parse_color(value), None);
}
}
#[test]
fn updates_are_display_local_and_invalid_files_reset_to_fallback() {
let root = std::env::temp_dir().join(format!("lazyboy-color-test-{}", std::process::id()));
std::fs::create_dir_all(&root).unwrap();
let a = root.join("a");
let b = root.join("b");
std::fs::write(&a, "#123456").unwrap();
std::fs::write(&b, "#ABCDEF").unwrap();
let mut first = ColorCache {
path: Some(a.clone()),
checked: None,
color: None,
};
let mut second = ColorCache {
path: Some(b),
checked: None,
color: None,
};
assert_eq!(first.get(), Some([18, 52, 86, 255]));
assert_eq!(second.get(), Some([171, 205, 239, 255]));
std::fs::write(&a, "#FF0000").unwrap();
first.checked = None;
assert_eq!(first.get(), Some([255, 0, 0, 255]));
assert_eq!(second.get(), Some([171, 205, 239, 255]));
std::fs::remove_file(a).unwrap();
first.checked = None;
assert_eq!(first.get(), None);
std::fs::remove_dir_all(root).unwrap();
}
}

2
image/computer/lazyboy-browser
View File

@ -16,7 +16,7 @@ else
fi
mkdir -p "$PROFILE"
# The bot drives this same window over DevTools (port 9221 + display number,
# see crates/control/src/cdp.rs). Every launch path (boot, panel launcher,
# configured by this launcher). Every launch path (boot, panel launcher,
# xdg-open, launch_app) must open it, otherwise the model cannot attach to the
# browser the human is looking at and would have to restart it.
DEVTOOLS_PORT=$((9221 + display))

42
image/computer/lazyboy-clipboard Normal file
View File

@ -0,0 +1,42 @@
#!/usr/bin/env python3
"""Visible GTK clipboard editor. All interaction is through Cua controls."""
import os
import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk, Gdk, GLib
GLib.set_prgname('lazyboy-clipboard')
number = ''.join(c for c in os.environ.get('DISPLAY', ':1') if c.isdigit()) or '1'
app = Gtk.Application(application_id='net.lazyboy.Clipboard.d' + number)
window = None
entry = None
def activate(application):
global window, entry
clipboard = Gtk.Clipboard.get(Gdk.SELECTION_CLIPBOARD)
if window is None:
window = Gtk.ApplicationWindow(application=application, title='Clipboard · LazyBoy')
window.set_default_size(520, 120)
# Retain clipboard ownership without leaving a window on the desktop.
def hide(widget, _event):
widget.hide()
return True
window.connect('delete-event', hide)
box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=10)
box.set_border_width(16)
window.add(box)
entry = Gtk.Entry()
entry.connect('changed', lambda field: field.get_accessible().set_name('Clipboard text: ' + field.get_text()))
entry.get_accessible().set_name('Clipboard text: ')
box.pack_start(entry, True, True, 0)
button = Gtk.Button(label='Copy')
def copy(_):
clipboard.set_text(entry.get_text(), -1)
button.connect('clicked', copy)
box.pack_start(button, False, False, 0)
entry.set_text(clipboard.wait_for_text() or '')
window.show_all()
window.present()
app.connect('activate', activate)
app.run([])

70
image/computer/lazyboy-screen
View File

@ -103,7 +103,7 @@ start_xvfb() {
return 0
fi
rm -f "/tmp/.X${number}-lock" "/tmp/.X11-unix/X${number}"
Xvfb "$display" -screen 0 1280x800x24 -ac +extension RANDR +render -noreset >>"${log}-xvfb.log" 2>&1 &
Xvfb "$display" -screen 0 1280x800x24 -ac +extension RANDR +render -noreset >>"${log}-xvfb.log" 2>&1 9>&- &
echo $! > "${log}-xvfb.pid"
wait_display "$display"
}
@ -141,11 +141,11 @@ start_atspi() {
fi
done
if [[ -n "$launcher" ]]; then
DISPLAY="$display" "$launcher" --launch-immediately >>"${log}-atspi-bus.log" 2>&1 &
DISPLAY="$display" "$launcher" --launch-immediately >>"${log}-atspi-bus.log" 2>&1 9>&- &
echo $! >"${log}-atspi-bus.pid"
fi
if [[ -n "$registry" ]]; then
DISPLAY="$display" "$registry" >>"${log}-atspi-registry.log" 2>&1 &
DISPLAY="$display" "$registry" >>"${log}-atspi-registry.log" 2>&1 9>&- &
echo $! >"${log}-atspi-registry.pid"
fi
sleep 0.2
@ -175,12 +175,12 @@ start_desktop() {
xfconfd --daemon >/dev/null 2>&1 || true
fi
if command -v xfwm4 >/dev/null 2>&1; then
xfwm4 --compositor=off --display="$display" --sm-client-disable >>"${log}-wm.log" 2>&1 &
xfwm4 --compositor=off --display="$display" --sm-client-disable >>"${log}-wm.log" 2>&1 9>&- &
echo $! >"${log}-wm.pid"
sleep 0.3
xfdesktop --disable-wm-check --sm-client-disable >>"${log}-desktop.log" 2>&1 &
xfdesktop --disable-wm-check --sm-client-disable >>"${log}-desktop.log" 2>&1 9>&- &
echo $! >"${log}-desktop.pid"
xfce4-panel --disable-wm-check --sm-client-disable >>"${log}-panel.log" 2>&1 &
xfce4-panel --disable-wm-check --sm-client-disable >>"${log}-panel.log" 2>&1 9>&- &
echo $! >"${log}-panel.pid"
else
echo "XFCE is missing" >&2
@ -196,7 +196,7 @@ start_vnc() {
local log="$4"
if ! port_open "$vnc_port"; then
x11vnc -display "$display" -forever -shared -nopw -listen 127.0.0.1 -rfbport "$vnc_port" \
-xkb -repeat -cursor arrow -noxdamage -ncache 0 >>"${log}-x11vnc.log" 2>&1 &
-xkb -repeat -cursor arrow -noxdamage -ncache 0 >>"${log}-x11vnc.log" 2>&1 9>&- &
fi
if ! port_open "$view_port"; then
local novnc=/usr/share/novnc
@ -205,7 +205,7 @@ start_vnc() {
return 1
fi
websockify --heartbeat=30 --web="$novnc" "0.0.0.0:${view_port}" "127.0.0.1:${vnc_port}" \
>>"${log}-novnc.log" 2>&1 &
>>"${log}-novnc.log" 2>&1 9>&- &
fi
wait_port "$vnc_port"
wait_port "$view_port"
@ -215,7 +215,6 @@ start_cua_driver() {
local display="$1"
local log="$2"
local number="${display#:}"
[[ "${LAZYBOY_COMPUTER_DRIVER:-legacy}" == "cua" ]] || return 0
if ! command -v cua-driver >/dev/null 2>&1; then
echo "Cua backend selected but cua-driver is not installed" >&2
return 1
@ -228,6 +227,7 @@ start_cua_driver() {
return 1
fi
export CUA_DRIVER_RS_HOME="$ROOT/cua-home-${number}"
export LAZYBOY_CURSOR_COLOR_FILE="$ROOT/screen-${number}.agent-color"
if [[ -s "$ROOT/screen-${number}.dbus" ]]; then
export DBUS_SESSION_BUS_ADDRESS="$(cat "$ROOT/screen-${number}.dbus")"
fi
@ -241,9 +241,10 @@ start_cua_driver() {
--grant existing-profile \
--socket "$sock" \
--pid-file "${log}-cua.pid" \
--no-overlay \
>>"${log}-cua.log" 2>&1 &
>>"${log}-cua.log" 2>&1 9>&- &
local bg=$!
# serve does not persist --pid-file in the pinned driver release.
printf '%s\n' "$bg" > "${log}-cua.pid"
local n
for n in $(seq 1 50); do
if [[ -S "$sock" ]]; then
@ -267,7 +268,7 @@ start_xterm() {
if alive_pidfile "${log}-xterm.pid"; then
return 0
fi
DISPLAY="$display" SHELL=/bin/zsh lazyboy-terminal >>"${log}-xterm.log" 2>&1 &
DISPLAY="$display" SHELL=/bin/zsh lazyboy-terminal >>"${log}-xterm.log" 2>&1 9>&- &
echo $! > "${log}-xterm.pid"
}
@ -285,18 +286,48 @@ start_browser() {
fi
rm -f "$profile/SingletonLock" "$profile/SingletonCookie" "$profile/SingletonSocket"
DISPLAY="$display" HOME="$HOME" LAZYBOY_BROWSER_PROFILE="$profile" \
lazyboy-browser https://duckduckgo.com >>"${log}-browser.log" 2>&1 &
lazyboy-browser https://duckduckgo.com >>"${log}-browser.log" 2>&1 9>&- &
echo $! > "${log}-browser.pid"
}
# Cosmetic update only: saving a color must not boot or focus a desktop.
set_agent_color() {
local slot="$1" color="$2"
if ! [[ "$slot" =~ ^[0-9]+$ ]] || (( slot < 0 || slot >= LIMIT )); then
echo "invalid screen slot" >&2
return 1
fi
if ! [[ "$color" =~ ^#[[:xdigit:]]{6}$ ]]; then
echo "invalid agent color: expected #RRGGBB" >&2
return 1
fi
local number=$((slot + 1)) color_file
color_file="$(mktemp "$ROOT/screen-${number}.agent-color.XXXXXX")"
printf '%s' "$color" > "$color_file"
mv -f "$color_file" "$ROOT/screen-${number}.agent-color"
}
ensure_slot() {
local slot="$1"
local profile="${2:-}"
local agent_name="${3:-}"
local agent_color="${4:-}"
if ! [[ "$slot" =~ ^[0-9]+$ ]] || (( slot < 0 || slot >= LIMIT )); then
echo "invalid screen slot" >&2
return 1
fi
local number=$((slot + 1))
if [[ -n "$agent_color" ]]; then
set_agent_color "$slot" "$agent_color" || return 1
fi
# Cua renders the public session label beside its synthetic cursor. Keep
# the label per display; an atomic replace avoids partial names during calls.
if [[ -n "$agent_name" ]]; then
local name_file
name_file="$(mktemp "$ROOT/screen-${number}.agent-name.XXXXXX")"
printf '%s' "$agent_name" > "$name_file"
mv -f "$name_file" "$ROOT/screen-${number}.agent-name"
fi
local display=":${number}"
local vnc_port=$((5900 + slot))
local view_port=$((6080 + slot))
@ -309,7 +340,7 @@ ensure_slot() {
if [[ -n "$profile" ]]; then
printf '%s\n' "$profile" >"$ROOT/screen-${number}.profile"
fi
if xdpyinfo -display "$display" >/dev/null 2>&1 && port_open "$vnc_port" && port_open "$view_port"; then
if xdpyinfo -display "$display" >/dev/null 2>&1 9>&- && port_open "$vnc_port" && port_open "$view_port"; then
start_cua_driver "$display" "$log" || return 1
if [[ -n "$profile" ]]; then
start_browser "$display" "$profile" "$log"
@ -323,7 +354,7 @@ ensure_slot() {
echo "screen ${slot} is busy starting" >&2
exit 1
}
if xdpyinfo -display "$display" >/dev/null 2>&1 && port_open "$vnc_port" && port_open "$view_port"; then
if xdpyinfo -display "$display" >/dev/null 2>&1 9>&- && port_open "$vnc_port" && port_open "$view_port"; then
start_cua_driver "$display" "$log" || exit 1
if [[ -n "$profile" ]]; then
start_browser "$display" "$profile" "$log"
@ -338,9 +369,9 @@ ensure_slot() {
exit 1
}
start_desktop "$display" "$xfce_home" "$log"
start_vnc "$display" "$vnc_port" "$view_port" "$log" || exit 1
start_cua_driver "$display" "$log" || exit 1
start_xterm "$display" "$log"
start_vnc "$display" "$vnc_port" "$view_port" "$log" || exit 1
if [[ -n "$profile" ]]; then
start_browser "$display" "$profile" "$log"
fi
@ -362,14 +393,17 @@ boot_primary() {
}
case "$cmd" in
color)
set_agent_color "${1:-}" "${2:-}"
;;
boot-primary)
boot_primary
;;
ensure)
ensure_slot "${1:-0}" "${2:-}"
ensure_slot "${1:-0}" "${2:-}" "${3:-}" "${4:-}"
;;
*)
echo "usage: lazyboy-screen boot-primary|ensure <slot> [profile]" >&2
echo "usage: lazyboy-screen color <slot> <#RRGGBB> | boot-primary | ensure <slot> [profile] [agent-name] [agent-color]" >&2
exit 2
;;
esac

441
image/computer/lazyboy-shell
View File

@ -1,441 +0,0 @@
#!/bin/sh
# Persistent terminals for the agent, on top of tmux.
#
# A `shell` call used to be a fresh `bash -lc`: the working directory, exports,
# and anything started in the background died with the call, so the model had to
# re-derive its way back to a working shell every time. A named tmux session per
# agent terminal keeps what a human keeps - one place to stand, jobs that stay
# alive, a Ctrl-C that interrupts the right thing - and the human can watch the
# same terminal on the desktop with `show`.
#
# The API reaches it through the container exec path (one-shot, argv only, no
# streaming), so finishing is detected by markers in the pane rather than by the
# exit of the exec itself. Markers are matched only at the start of a line: the
# shell echoes the typed command, and that echo contains both markers too.
#
# lazyboy-shell run <session> <wait_ms> <command> run, wait, print output + exit code
# lazyboy-shell log <session> [lines] what the terminal shows right now
# lazyboy-shell keys <session> <key>... C-c / Enter / literal text
# lazyboy-shell show <session> open it on the desktop for a human
# lazyboy-shell reset <session> drop the session and start clean
# lazyboy-shell list sessions and their state
#
# Every call is sourced into the live shell (`. file`), which is what makes
# state survive and also what would let `exit` take the terminal down; a trap on
# EXIT turns that into a normal end marker and the pane is restarted.
set -eu
# Wide enough that compiler output and `ls -l` do not wrap, short enough to stay
# cheap to capture after every call.
COLS=220
ROWS=50
HISTORY=50000
BACK=4000
# Guard the model's context: keep the head and the tail of very chatty commands.
MAX_OUT=20000
# A marker that scrolled away must not lock a terminal forever.
STALE_AFTER=600
export LANG="${LANG:-zh_TW.UTF-8}"
export LC_ALL="${LC_ALL:-zh_TW.UTF-8}"
export TMUX_TMPDIR="${TMUX_TMPDIR:-/tmp}"
STATE_DIR="${LAZYBOY_SHELL_STATE:-${TMPDIR:-/tmp}/lazyboy-shell}"
die() {
printf 'error: %s\n' "$1" >&2
exit 2
}
need_tmux() {
command -v tmux >/dev/null 2>&1 ||
die "tmux is missing from this desktop image (rebuild with: make computer)"
}
# Paths end up inside the shell code the pane sources; single quotes are the
# only thing that has to survive.
sq() {
printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")"
}
# Session names are model input: keep them boring, and namespace them so an
# agent terminal never collides with a tmux session the human started.
session_of() {
case "$1" in
'' | *[!A-Za-z0-9._-]*) die "session name must use [A-Za-z0-9._-], got: $1" ;;
esac
printf 'lazyboy-%s' "$1"
}
# Pane commands name an explicit pane: a detached tmux server has no current
# pane for the session-only forms to resolve against, and set-option reads the
# target as a window, so even session options go through the pane.
pane_target() {
printf '=%s:0.0' "$1"
}
pane_of() {
tmux capture-pane -p -J -S "-$BACK" -t "$(pane_target "$1")" 2>/dev/null || true
}
# Tmux paints; the model reads text.
clean() {
sed -e 's/\x1b\[[0-9;?]*[a-zA-Z]//g' -e 's/\x1b[()][A-Za-z0-9]//g' -e 's/\x1b[>=]//g' |
tr -d '\r'
}
# A pane is a fixed grid, so below the prompt there are only empty rows. Tailing
# one without trimming reads as a blank screen.
trim_blanks() {
awk '{ line[NR] = $0; if ($0 !~ /^[ \t]*$/) last = NR }
END { for (i = 1; i <= last; i++) print line[i] }'
}
# Everything the terminal showed since this call printed its start marker.
after_start() {
printf '%s\n' "$1" | awk -v start="LB_START $2" '
index($0, start) == 1 { keep = 1; next }
keep { print }'
}
finished_in() {
printf '%s\n' "$1" | grep -q "^LB_END $2 rc=[0-9][0-9]*$"
}
exit_code_in() {
printf '%s\n' "$1" | sed -n "s/^LB_END $2 rc=\\([0-9][0-9]*\\)\$/\\1/p" | tail -n 1
}
# The shell is back at its prompt for this call, without having printed an end
# marker: the command was interrupted (or abandoned the runner). Either way the
# terminal belongs to the next command.
ready_in() {
after_start "$1" "$2" | grep -q '^LB_READY$'
}
is_dead() {
[ "$(tmux display-message -p -t "$(pane_target "$1")" '#{pane_dead}' 2>/dev/null)" = "1" ]
}
# Fallback for when the integration itself is gone (the model ran `exec bash`,
# replaced PROMPT_COMMAND, ...): a shell sitting on its prompt is free.
at_prompt() {
case "$(tmux display-message -p -t "$(pane_target "$1")" '#{pane_current_command}' 2>/dev/null)" in
bash | -bash | zsh | -zsh | sh | dash | '' ) ;;
*) return 1 ;;
esac
after_start "$(pane_of "$1")" "${2:-}" | clean | trim_blanks | tail -n 1 |
grep -qE '[#$%>][[:space:]]*$'
}
# Free for the next command even though this call never reported a result:
# interrupted at a prompt, replaced its own shell, or died outright.
released() {
# Distinct names: sh functions share the caller's variables.
lb_pane="$1"
lb_pending="$2"
lb_name="$3"
if ready_in "$lb_pane" "$lb_pending"; then
return 0
fi
is_dead "$lb_name" || at_prompt "$lb_name" "$lb_pending"
}
truncate_out() {
awk -v max="$MAX_OUT" '
{ if (length(all) < max) all = all $0 "\n"; else dropped = 1 }
END {
printf "%s", all
if (dropped) printf "\n…（輸出過長，已截斷尾段；完整輸出請在 shell 裡用 > 寫進檔案再讀）\n"
}'
}
# Text between this call's start marker and its end marker.
output_between() {
printf '%s\n' "$1" | awk -v start="LB_START $2" -v end="LB_END $2 " '
index($0, start) == 1 { keep = 1; next }
index($0, end) == 1 { keep = 0 }
keep { print }'
}
state_dir_ready() {
mkdir -p "$STATE_DIR"
# Snapshots hold exported variables, so keep them out of other users' reach.
chmod 700 "$STATE_DIR" 2>/dev/null || true
}
# The shell in a dead pane is restarted where it left off: same directory, and
# the next command re-loads the exported variables from the last snapshot.
revive_pane() {
name="$1"
env_file="$STATE_DIR/$name.env"
dir="${HOME:-/tmp}"
if [ -f "$env_file" ]; then
saved=$(sed -n '1p' "$env_file" 2>/dev/null || true)
if [ -n "$saved" ] && [ -d "$saved" ]; then
dir="$saved"
fi
fi
tmux respawn-pane -k -t "$(pane_target "$name")" -c "$dir" /bin/bash -i 2>/dev/null || true
: >"$STATE_DIR/$name.revive"
}
# REVIVED tells the caller whether it has to say something about the restart.
REVIVED=no
ensure_session() {
name="$1"
state_dir_ready
if ! tmux has-session -t "=$name" 2>/dev/null; then
tmux new-session -d -s "$name" -x "$COLS" -y "$ROWS" -c "${HOME:-/tmp}" "/bin/bash -i"
fi
target=$(pane_target "$name")
# remain-on-exit keeps a dead pane readable, so `exit` in a command does not
# swallow the output the model still needs; the history has to outlive a
# compile.
tmux set-option -t "$target" history-limit "$HISTORY" 2>/dev/null || true
tmux set-option -t "$target" remain-on-exit on 2>/dev/null || true
if is_dead "$name"; then
revive_pane "$name"
REVIVED=yes
fi
}
# The shell code a call types into the pane. It is sourced, so `cd` and exports
# land in the terminal's own shell; it carries its own integration, so a model
# that clobbers PROMPT_COMMAND or replaces the shell only loses it for one call.
write_runner() {
run_file="$1"
name="$2"
nonce="$3"
command="$4"
env_file="$STATE_DIR/$name.env"
{
printf '%s\n' '# LazyBoy agent terminal (see image/computer/lazyboy-shell)'
printf 'LB_STATE_DIR=%s\n' "$(sq "$STATE_DIR")"
printf '%s\n' 'lb_ready() { printf "\nLB_READY\n"; }'
printf 'LB_ENV=%s\n' "$(sq "$env_file")"
printf '%s\n' 'lb_snapshot() { { pwd -P; export -p; } >"$LB_ENV" 2>/dev/null; }'
printf '%s\n' 'lb_exit() { printf "\nLB_END %s rc=%s\n" "${LB_NONCE:-shell}" "$1"; lb_snapshot; }'
printf '%s\n' 'case "${PROMPT_COMMAND-}" in *lb_ready*) : ;; *) PROMPT_COMMAND="lb_ready${PROMPT_COMMAND:+;$PROMPT_COMMAND}" ;; esac'
printf '%s\n' 'trap '"'"'lb_exit "$?"'"'"' EXIT'
if [ -f "$STATE_DIR/$name.revive" ] && [ -f "$env_file" ]; then
printf '%s\n' '{ cd "$(sed -n 1p -- '"$(sq "$env_file")"')" 2>/dev/null;' \
'eval "$(tail -n +2 -- '"$(sq "$env_file")"' 2>/dev/null)"; } >/dev/null 2>&1 || true'
rm -f "$STATE_DIR/$name.revive"
fi
printf 'LB_NONCE=%s\n' "$(sq "$nonce")"
printf 'printf %s\n' "'\nLB_START $nonce\n'"
printf '%s\n' "$command"
printf '%s\n' '__lb_rc=$?'
printf 'printf %s\n' "'\nLB_END $nonce rc=%s\n' \"\$__lb_rc\""
} >"$run_file"
}
do_run() {
[ "$#" -ge 3 ] || die "run needs <session> <wait_ms> <command>"
session="$1"
wait_ms="$2"
command="$3"
case "$wait_ms" in '' | *[!0-9]*) die "wait_ms must be a number" ;; esac
need_tmux
name=$(session_of "$session")
ensure_session "$name"
nonce="$(date +%s%N)-$$"
pending_file="$STATE_DIR/$name.pending"
run_file="$STATE_DIR/$name.sh"
# An earlier call may still own this terminal. Typing now would feed the
# running program instead of the shell, so say so rather than corrupt it.
pending=$(sed -n 's/^nonce=\(.*\)$/\1/p' "$pending_file" 2>/dev/null || true)
started=$(sed -n 's/^started=\(.*\)$/\1/p' "$pending_file" 2>/dev/null || true)
if [ -n "$pending" ]; then
pane=$(pane_of "$name")
if finished_in "$pane" "$pending" || released "$pane" "$pending" "$name"; then
# Finished, interrupted, or crashed: the marker is gone or meaningless,
# and the next command can have the terminal.
pending=""
rm -f "$pending_file"
elif [ -n "$started" ] && [ $(( $(date +%s) - started )) -ge "$STALE_AFTER" ] &&
printf '%s\n' "$pane" | clean | trim_blanks | tail -n 3 | grep -qE '[\$#>] ?$'; then
# A marker that scrolled out of the capture window would otherwise lock
# this terminal forever; an idle prompt after ten minutes means free.
pending=""
rm -f "$pending_file"
fi
if [ -n "$pending" ]; then
printf 'status=running session=%s\n' "$session"
printf 'This terminal is still busy with an earlier command, so nothing was typed.\n'
printf 'Read it with shell {"session":"%s","logLines":120}, interrupt it with\n' "$session"
printf '{"session":"%s","keys":"C-c"}, or reset it with {"session":"%s","reset":true}.\n' "$session" "$session"
printf -- '--- terminal ---\n'
pane_of "$name" | clean | trim_blanks | tail -n 60 | truncate_out
return 0
fi
fi
write_runner "$run_file" "$name" "$nonce" "$command"
{
printf 'nonce=%s\n' "$nonce"
printf 'started=%s\n' "$(date +%s)"
} >"$pending_file"
# One short typed line: the runner itself lives in a file, so nothing about
# the command needs quoting and long commands cannot outgrow send-keys.
target=$(pane_target "$name")
tmux send-keys -t "$target" -l -- ". $(sq "$run_file")"
tmux send-keys -t "$target" Enter
deadline=$(( $(date +%s) + wait_ms / 1000 + 1 ))
while [ "$(date +%s)" -lt "$deadline" ]; do
pane=$(pane_of "$name")
if finished_in "$pane" "$nonce"; then
rm -f "$pending_file"
printf 'status=done session=%s exit=%s\n' "$session" "$(exit_code_in "$pane" "$nonce")"
printf -- '--- output ---\n'
output_between "$pane" "$nonce" | clean | truncate_out
# `exit` in a command, or a shell that died by itself: hand the next
# command a terminal again instead of a dead pane.
if is_dead "$name"; then
revive_pane "$name"
printf 'note: the shell in this terminal exited; it was restarted in the same directory,\n'
printf 'so cd and exported variables are back but background jobs of that shell are gone.\n'
fi
return 0
fi
if is_dead "$name"; then
break
fi
sleep 0.1
done
if is_dead "$name"; then
rm -f "$pending_file"
printf 'status=closed session=%s\n' "$session"
printf 'The shell in this terminal exited before it could report a result.\n'
printf -- '--- terminal ---\n'
pane_of "$name" | clean | trim_blanks | tail -n 60 | truncate_out
revive_pane "$name"
printf 'note: it was restarted in the same directory and is ready for the next command.\n'
return 0
fi
printf 'status=running session=%s waitedMs=%s\n' "$session" "$wait_ms"
printf 'The command is still running; the output so far follows. Do not type another command into\n'
printf 'this terminal - poll with shell {"session":"%s","logLines":120} or interrupt with {"keys":"C-c"}.\n' "$session"
printf -- '--- terminal ---\n'
pane_of "$name" | clean | trim_blanks | tail -n 60 | truncate_out
}
do_log() {
[ "$#" -ge 1 ] || die "log needs <session>"
session="$1"
lines="${2:-80}"
case "$lines" in '' | *[!0-9]*) die "lines must be a number" ;; esac
need_tmux
name=$(session_of "$session")
tmux has-session -t "=$name" 2>/dev/null || {
printf 'status=idle session=%s\nThis terminal has not been used yet.\n' "$session"
return 0
}
pending=$(sed -n 's/^nonce=\(.*\)$/\1/p' "$STATE_DIR/$name.pending" 2>/dev/null || true)
pane=$(pane_of "$name")
if is_dead "$name"; then
printf 'status=closed session=%s\n' "$session"
elif [ -n "$pending" ] && ! finished_in "$pane" "$pending" &&
! released "$pane" "$pending" "$name"; then
printf 'status=running session=%s\nThe command from the earlier call is still running.\n' "$session"
else
rm -f "$STATE_DIR/$name.pending"
printf 'status=idle session=%s\n' "$session"
fi
printf -- '--- terminal ---\n'
printf '%s\n' "$pane" | clean | trim_blanks | tail -n "$lines" | truncate_out
}
# A key name tmux understands is sent as a key; anything else is typed as text.
send_one() {
target=$(pane_target "$name")
case "$1" in
C-* | M-* | Enter | Return | Escape | Esc | Tab | BSpace | DC | IC | \
Up | Down | Left | Right | Home | End | PageUp | PageDown | F[1-9] | F1[0-2])
tmux send-keys -t "$target" "$1"
;;
*)
tmux send-keys -t "$target" -l -- "$1"
;;
esac
}
do_keys() {
[ "$#" -ge 2 ] || die "keys needs <session> <key>..."
session="$1"
shift
need_tmux
name=$(session_of "$session")
ensure_session "$name"
for key in "$@"; do
case "$key" in
# Ctrl-C throws the end marker away with the command, which would leave
# the terminal looking busy forever; the interrupt *is* the release.
C-c | C-\\ | C-z) rm -f "$STATE_DIR/$name.pending" ;;
esac
send_one "$key"
done
printf 'status=sent session=%s keys=%s\n' "$session" "$*"
printf 'Use shell {"session":"%s","logLines":60} to see what it did.\n' "$session"
}
do_show() {
[ "$#" -ge 1 ] || die "show needs <session>"
session="$1"
need_tmux
name=$(session_of "$session")
ensure_session "$name"
command -v xfce4-terminal >/dev/null 2>&1 ||
die "no desktop terminal available to show this session"
# Detached so the window outlives this exec: the human sees the same terminal
# the agent types in and can click into it to take over.
setsid nohup xfce4-terminal --disable-server --geometry=112x30+64+64 \
--title="終端機 $session" --command="tmux attach -t $name" >/dev/null 2>&1 &
printf 'status=shown session=%s\nThe terminal is open on the desktop screen.\n' "$session"
}
do_reset() {
[ "$#" -ge 1 ] || die "reset needs <session>"
need_tmux
name=$(session_of "$1")
tmux kill-session -t "=$name" 2>/dev/null || true
rm -f "$STATE_DIR/$name.pending" "$STATE_DIR/$name.revive" "$STATE_DIR/$name.env"
ensure_session "$name"
printf 'status=reset session=%s\n' "$1"
}
do_list() {
need_tmux
panes=$(tmux list-panes -a -F '#{session_name}|#{pane_dead}' 2>/dev/null || true)
if [ -z "$panes" ]; then
printf 'status=empty\nNo agent terminals are running.\n'
return 0
fi
printf '%s\n' "$panes" | while IFS='|' read -r name dead; do
case "$name" in
lazyboy-*)
if [ "$dead" = 1 ]; then
printf 'session=%s state=closed\n' "${name#lazyboy-}"
else
printf 'session=%s state=open\n' "${name#lazyboy-}"
fi
;;
esac
done
}
program=${0##*/}
case "${1:-}" in
run) shift; do_run "$@" ;;
log) shift; do_log "$@" ;;
keys) shift; do_keys "$@" ;;
show) shift; do_show "$@" ;;
reset) shift; do_reset "$@" ;;
list) shift; do_list ;;
*) die "usage: $program run|log|keys|show|reset|list ..." ;;
esac

1
image/computer/lazyboy-terminal
View File

@ -13,6 +13,7 @@ if command -v xfce4-terminal >/dev/null 2>&1; then
exec xfce4-terminal \
--disable-server \
--geometry=92x28+48+48 \
--dynamic-title-mode=none \
--title=終端機 \
--command="/bin/zsh -l" \
"$@"

16
image/computer/lazyboy-terminal-reset Normal file
View File

@ -0,0 +1,16 @@
#!/bin/sh
# Invoked visibly from the terminal through Cua. Replace this shell with a
# clean login shell while retaining only the desktop connection environment.
set -eu
pkill -TERM -P "$$" 2>/dev/null || true
cd /home/lazyboy
exec env -i \
HOME=/home/lazyboy USER=lazyboy LOGNAME=lazyboy SHELL=/bin/zsh \
PATH=/home/lazyboy/.local/bin:/usr/local/bin:/usr/bin:/bin \
TERM=xterm-256color COLORTERM=truecolor \
LANG="${LANG:-zh_TW.UTF-8}" LC_ALL="${LC_ALL:-zh_TW.UTF-8}" \
DISPLAY="${DISPLAY:-:1}" \
DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-}" \
XDG_CONFIG_HOME="${XDG_CONFIG_HOME:-/home/lazyboy/.config}" \
XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-}" \
/bin/zsh -l

101
image/computer/licenses/huninn-OFL.txt Normal file
View File

@ -0,0 +1,101 @@
[Kosugi Maru]
Copyright (c) 2010 MOTOYA CO.,LTD.
[Varela Round]
Copyright (c) 2011-2016 The Varela Round Project Authors (https://github.com/alefalefalef/Varela-Round-Hebrew/), with Reserved Font Names 'Varela' and 'Varela Round'.
[jf open huninn]
Copyright (c) 2020-2024 The jf open huninn font is redistributed by justfont Co., LTD., with Reserved Font Names 'open huninn' and 'huninn'. under SIL Open Font License. The Hanzi part of this project was derived from Kosugi Maru under Apache-2.0 (https://www.apache.org/licenses/LICENSE-2.0).
This Font Software is licensed under the SIL Open Font License, Version 1.1.
This license is copied below, and is also available with a FAQ at:
http://scripts.sil.org/OFL
-----------------------------------------------------------
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
-----------------------------------------------------------
PREAMBLE
The goals of the Open Font License (OFL) are to stimulate worldwide
development of collaborative font projects, to support the font creation
efforts of academic and linguistic communities, and to provide a free and
open framework in which fonts may be shared and improved in partnership
with others.
The OFL allows the licensed fonts to be used, studied, modified and
redistributed freely as long as they are not sold by themselves. The
fonts, including any derivative works, can be bundled, embedded,
redistributed and/or sold with any software provided that any reserved
names are not used by derivative works. The fonts and derivatives,
however, cannot be released under any other type of license. The
requirement for fonts to remain under this license does not apply
to any document created using the fonts or their derivatives.
DEFINITIONS
"Font Software" refers to the set of files released by the Copyright
Holder(s) under this license and clearly marked as such. This may
include source files, build scripts and documentation.
"Reserved Font Name" refers to any names specified as such after the
copyright statement(s).
"Original Version" refers to the collection of Font Software components as
distributed by the Copyright Holder(s).
"Modified Version" refers to any derivative made by adding to, deleting,
or substituting -- in part or in whole -- any of the components of the
Original Version, by changing formats or by porting the Font Software to a
new environment.
"Author" refers to any designer, engineer, programmer, technical
writer or other person who contributed to the Font Software.
PERMISSION & CONDITIONS
Permission is hereby granted, free of charge, to any person obtaining
a copy of the Font Software, to use, study, copy, merge, embed, modify,
redistribute, and sell modified and unmodified copies of the Font
Software, subject to the following conditions:
1) Neither the Font Software nor any of its individual components,
in Original or Modified Versions, may be sold by itself.
2) Original or Modified Versions of the Font Software may be bundled,
redistributed and/or sold with any software, provided that each copy
contains the above copyright notice and this license. These can be
included either as stand-alone text files, human-readable headers or
in the appropriate machine-readable metadata fields within text or
binary files as long as those fields can be easily viewed by the user.
3) No Modified Version of the Font Software may use the Reserved Font
Name(s) unless explicit written permission is granted by the corresponding
Copyright Holder. This restriction only applies to the primary font name as
presented to the users.
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
Software shall not be used to promote, endorse or advertise any
Modified Version, except to acknowledge the contribution(s) of the
Copyright Holder(s) and the Author(s) or with their explicit written
permission.
5) The Font Software, modified or unmodified, in part or in whole,
must be distributed entirely under this license, and must not be
distributed under any other license. The requirement for fonts to
remain under this license does not apply to any document created
using the Font Software.
TERMINATION
This license becomes null and void if any of the above conditions are
not met.
DISCLAIMER
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
OTHER DEALINGS IN THE FONT SOFTWARE.

35
scripts/cua-clipboard-test.py Normal file
View File

@ -0,0 +1,35 @@
#!/usr/bin/env python3
"""Clipboard roundtrip and terminal paste through Cua in a disposable desktop."""
import base64
import importlib.machinery
import importlib.util
import time
from pathlib import Path
loader = importlib.machinery.SourceFileLoader('adapter','/usr/local/bin/lazyboy-cua-adapter-test')
spec = importlib.util.spec_from_loader(loader.name,loader)
adapter = importlib.util.module_from_spec(spec)
loader.exec_module(adapter)
adapter.wait_health()
name='LazyBoy clipboard test ' + str(time.monotonic_ns())
out=Path('/tmp/lazyboy/cua-clipboard-test.txt')
out.unlink(missing_ok=True)
def act(actions):
return adapter.api('/act',{'actions':actions,'observe':True,'settle_ms':500})
def key(value):
return {'kind':'key','key':value}
act([{'kind':'launch','application':'terminal','uri':'--title='+name}])
act([{'kind':'focus','title':name},{'kind':'clipboard','text':"printf '中文🙂\\nsecond line\\n' > /tmp/lazyboy/cua-clipboard-test.txt"},key('return')])
assert out.read_text()=='中文🙂\nsecond line\n',repr(out.read_bytes())
# Multiline paste must be bracketed into the same terminal, not silently lose
# characters or submit incomplete lines through an X11 typing fallback.
out.unlink()
act([{'kind':'focus','title':name},{'kind':'clipboard','text':"printf '%s' '中文\n第二行🙂' > /tmp/lazyboy/cua-clipboard-test.txt"},key('return')])
# The configured zsh confirms bracketed multiline paste before execution.
act([key('return')])
assert out.read_text()=='中文\n第二行🙂',repr(out.read_bytes())
act([{'kind':'focus','title':name},key('ctrl+shift+a')])
result=act([{'kind':'copyselection'}])
assert '第二行' in result['clipboardText'],result
print('Cua clipboard Unicode, multiline terminal paste and copy roundtrip passed',flush=True)

107
scripts/cua-cursor-color-test.py Normal file
View File

@ -0,0 +1,107 @@
#!/usr/bin/env python3
"""Verify selected colors in the actual VNC framebuffer, without VNC input."""
import importlib.machinery
import importlib.util
import struct
import socket
import subprocess
import time
import zlib
from pathlib import Path
loader = importlib.machinery.SourceFileLoader('adapter', '/usr/local/bin/lazyboy-cua-adapter-test')
spec = importlib.util.spec_from_loader(loader.name, loader)
adapter = importlib.util.module_from_spec(spec)
loader.exec_module(adapter)
def capture():
with socket.create_connection(('127.0.0.1', 5900), 5) as stream:
def read(size):
data = bytearray()
while len(data) < size:
chunk = stream.recv(size - len(data))
assert chunk, 'VNC closed before a complete framebuffer'
data.extend(chunk)
return bytes(data)
assert read(12).startswith(b'RFB ')
stream.sendall(b'RFB 003.008\n')
assert 1 in read(read(1)[0]), 'disposable desktop must offer unauthenticated loopback VNC'
stream.sendall(b'\x01')
assert read(4) == b'\0' * 4
stream.sendall(b'\x01')
width, height = struct.unpack('>HH', read(4))
read(16)
read(struct.unpack('>I', read(4))[0])
assert 0 < width <= 4096 and 0 < height <= 4096
# Request raw pixels in RGBX byte order. Never send keys or pointer input.
stream.sendall(b'\0' * 4 + struct.pack('>BBBBHHHBBBxxx', 32, 24, 0, 1, 255, 255, 255, 0, 8, 16))
stream.sendall(struct.pack('>BBHi', 2, 0, 1, 0))
stream.sendall(struct.pack('>BBHHHH', 3, 0, 0, 0, width, height))
pixels = bytearray(width * height * 4)
while True:
kind = read(1)[0]
if kind == 2:
continue
if kind == 3:
read(3)
read(struct.unpack('>I', read(4))[0])
continue
assert kind == 0, kind
read(1)
for _ in range(struct.unpack('>H', read(2))[0]):
x, y, w, h, encoding = struct.unpack('>HHHHi', read(12))
assert encoding == 0 and x + w <= width and y + h <= height
rectangle = read(w * h * 4)
for row in range(h):
start = ((y + row) * width + x) * 4
pixels[start:start + w * 4] = rectangle[row * w * 4:(row + 1) * w * 4]
return width, height, pixels
def save_png(path, width, height, pixels):
pixels[3::4] = b'\xff' * (width * height)
rows = b''.join(b'\0' + pixels[y * width * 4:(y + 1) * width * 4] for y in range(height))
def chunk(kind, data):
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
Path(path).write_bytes(b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 6, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(rows)) + chunk(b'IEND', b''))
adapter.wait_health()
root = Path('/tmp/lazyboy')
files = [root / 'screen-1.agent-name', root / 'screen-1.agent-color', root / 'screen-2.agent-color']
previous = {path: path.read_bytes() if path.exists() else None for path in files}
name = '小幫手 Alice'
try:
subprocess.run(['lazyboy-screen', 'ensure', '0', '', name, '#8B5CF6'], check=True)
adapter.api('/browser', {'action': 'snapshot', 'ensure': True})
adapter.api('/browser', {'action': 'navigate', 'url': 'about:blank', 'ensure': True})
pid = (root / 'screen-1-cua.pid').read_text()
for color, label in [('#8B5CF6', 'purple'), ('#22C55E', 'green'), ('#E11D48', 'pink')]:
subprocess.run(['lazyboy-screen', 'color', '0', color], check=True)
subprocess.run(['lazyboy-screen', 'color', '1', '#123456'], check=True)
assert (root / 'screen-1-cua.pid').read_text() == pid, 'color change restarted Cua'
assert files[0].read_text() == name, 'color change renamed the session'
adapter.api('/act', {'actions': [{'kind': 'pointer', 'type': 'click', 'x': 640, 'y': 400}], 'observe': False, 'settle_ms': 100})
expected = bytes.fromhex(color[1:])
deadline = time.monotonic() + 5
while True:
width, height, pixels = capture()
matches = sum(pixels[(y * width + x) * 4:(y * width + x) * 4 + 3] == expected
for y in range(380, 425) for x in range(620, 665))
if matches >= 3:
break
assert time.monotonic() < deadline, (color, 'selected RGB absent from VNC cursor')
time.sleep(0.1)
save_png(f'/tmp/cua-cursor-{label}.png', width, height, pixels)
print(f'VNC cursor {color}: {matches} exact RGB pixels, same session and daemon', flush=True)
rejected = subprocess.run(['lazyboy-screen', 'color', '0', 'red'], capture_output=True)
assert rejected.returncode != 0 and files[1].read_text() == '#E11D48'
finally:
adapter.smoke.cua_call('end_session', {'session': name})
for path, value in previous.items():
if value is None:
path.unlink(missing_ok=True)
else:
path.write_bytes(value)
print('Selected cursor colors, live updates, display isolation and invalid-color rejection passed')

51
scripts/cua-cursor-test.py Normal file
View File

@ -0,0 +1,51 @@
#!/usr/bin/env python3
"""Check named cursor sessions and save a real desktop frame for visual review.
Run as the desktop user in a disposable computer container. The screenshot is
intentional evidence: Cua's Linux cursor-state response does not report geometry,
and a successful state call alone cannot prove that the overlay was drawn.
"""
import base64
import importlib.machinery
import importlib.util
import subprocess
from pathlib import Path
loader = importlib.machinery.SourceFileLoader('adapter', '/usr/local/bin/lazyboy-cua-adapter-test')
spec = importlib.util.spec_from_loader(loader.name, loader)
adapter = importlib.util.module_from_spec(spec)
loader.exec_module(adapter)
adapter.wait_health()
adapter.api('/browser', {'action': 'snapshot', 'ensure': True})
name_file = Path('/tmp/lazyboy/screen-1.agent-name')
previous = name_file.read_bytes() if name_file.exists() else None
names = ['Alice', '小幫手 Alice']
try:
for index, name in enumerate(names):
subprocess.run(['lazyboy-screen', 'ensure', '0', '', name], check=True)
assert name_file.read_text() == name
# This also exercises rebinding after changing a screen's public label.
adapter.api('/browser', {'action': 'snapshot', 'ensure': True})
adapter.api('/browser', {'action': 'navigate', 'url': 'about:blank', 'ensure': True})
frame = adapter.api('/act', {
'actions': [{'kind': 'pointer', 'type': 'click', 'x': 640, 'y': 400}],
'observe': True, 'settle_ms': 250,
})
state = adapter.smoke.cua_call('get_agent_cursor_state', {'session': name})['parsed']
assert state['session'] == name and state['enabled'], state
# Use /observe rather than depending on the action response envelope.
frame = adapter.observe()
Path(f'/tmp/cua-cursor-{index}.png').write_bytes(base64.b64decode(frame['png_base64']))
adapter.smoke.cua_call('end_session', {'session': name})
adapter.observe() # Read recovery must revive this name, not lazyboy-1.
state = adapter.smoke.cua_call('get_agent_cursor_state', {'session': name})['parsed']
assert state['session'] == name
adapter.smoke.cua_call('end_session', {'session': name})
finally:
if previous is None:
name_file.unlink(missing_ok=True)
else:
name_file.write_bytes(previous)
print('Named Cua cursor: English/Chinese labels, rename/rebind, expiry recovery passed')
print('Visually inspect /tmp/cua-cursor-0.png and /tmp/cua-cursor-1.png for rendered names')

7
scripts/cua-login-fixture.py Normal file
View File

@ -0,0 +1,7 @@
import http.server,ssl
PAGE=b'''<!doctype html><html><head><title>CUA saved-login fixture</title></head><body><h1>Saved login verification</h1><form><label>Email<input id="email" type="email" autocomplete="username"></label><label>Password<input id="password" type="password" autocomplete="current-password"></label><button type="submit">Sign in</button></form><p id="verification">Waiting for input</p><p id="result">Not submitted</p><script>document.querySelector('form').onsubmit=e=>{e.preventDefault();document.querySelector('#result').textContent='Submitted'};document.querySelectorAll('input').forEach(e=>e.oninput=()=>{document.querySelector('#verification').textContent=document.querySelector('#email').value==='cua@example.test'&&document.querySelector('#password').value==='fixture-only-123'?'Both fields verified':'CUA saved-login fixture'})</script></body></html>'''
class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200);self.send_header('Content-Type','text/html');self.end_headers();self.wfile.write(PAGE)
server=http.server.HTTPServer(('127.0.0.1',8443),Handler)
ctx=ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER);ctx.load_cert_chain('/tmp/login-cert.pem','/tmp/login-key.pem');server.socket=ctx.wrap_socket(server.socket,server_side=True);server.serve_forever()

27
scripts/cua-login-test.sh Executable file
View File

@ -0,0 +1,27 @@
#!/usr/bin/env bash
# End-to-end saved-login field selection and typing in a disposable Cua desktop.
# The local certificate is trusted only inside this throwaway container.
set -euo pipefail
image="${COMPUTER_IMAGE:-lazyboy/computer:local}"
name=$(docker run -d --shm-size=512m -e LAZYBOY_CONTROL_TOKEN=login-fixture-only "$image")
trap 'docker rm -f "$name" >/dev/null 2>&1 || true' EXIT
for _ in $(seq 1 120); do
if docker exec "$name" test -f /tmp/lazyboy/ready; then break; fi
sleep 1
done
docker exec "$name" test -f /tmp/lazyboy/ready
docker exec -u 0 "$name" sh -c 'apt-get update -qq && apt-get install -y -qq libnss3-tools' >/dev/null 2>&1
docker exec "$name" openssl req -x509 -newkey rsa:2048 -nodes \
-keyout /tmp/login-key.pem -out /tmp/login-cert.pem -days 2 \
-subj /CN=localhost -addext subjectAltName=DNS:localhost \
-addext basicConstraints=critical,CA:TRUE >/dev/null 2>&1
docker exec -u 1000:1000 "$name" sh -c '
mkdir -p "$HOME/.pki/nssdb"
certutil -N --empty-password -d sql:"$HOME/.pki/nssdb"
certutil -A -d sql:"$HOME/.pki/nssdb" -n lazyboy-local-fixture -t "C,," -i /tmp/login-cert.pem
'
root="$(cd "$(dirname "$0")/.." && pwd)"
docker cp "$root/scripts/cua-login-fixture.py" "$name":/tmp/login-fixture.py
docker exec -d "$name" python3 /tmp/login-fixture.py
CUA_LOGIN_TEST_CONTAINER="$name" cargo test --manifest-path "$root/Cargo.toml" \
-p lazyboy-api saved_login_fills_real_cua_fields_without_submitting -- --ignored

22
scripts/cua-session-test.py Normal file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env python3
"""Exercise recovery from Cua lifecycle-session expiry on a disposable desktop."""
import importlib.machinery
import importlib.util
loader = importlib.machinery.SourceFileLoader('adapter', '/usr/local/bin/lazyboy-cua-adapter-test')
spec = importlib.util.spec_from_loader(loader.name, loader)
adapter = importlib.util.module_from_spec(spec)
loader.exec_module(adapter)
adapter.wait_health()
# Mint and cache a browser binding before the session is lost.
adapter.api('/browser', {'action': 'snapshot', 'ensure': True})
adapter.smoke.cua_call('end_session', {'session': 'lazyboy-1'})
frame = adapter.observe()
assert frame['png_base64'], 'read-only recovery must return a shared-desktop frame'
page = adapter.api('/browser', {'action': 'snapshot', 'ensure': True})
assert page['ok'], 'the cached browser binding must be refreshed after session revival'
# A mutation presented directly after expiry must be rejected, not replayed.
adapter.smoke.cua_call('end_session', {'session': 'lazyboy-1'})
adapter.act({'kind': 'key', 'key': 'a'}, expect_error=True)
adapter.observe()
print('Cua session expiry: observation recovered, browser rebound, mutation not replayed', flush=True)

35
scripts/cua-smoke-test.sh
View File

@ -79,6 +79,15 @@ if ! docker exec -u 1000:1000 "$name" test -f /tmp/lazyboy/ready; then
exit 1
fi
# Warm reconnects must reuse one daemon and release the startup lock.
for _ in 1 2 3; do
docker exec -u 1000:1000 "$name" lazyboy-screen ensure 0 >/dev/null
done
docker exec -u 1000:1000 "$name" sh -c '
test "$(pgrep -c -x cua-driver)" -eq 1 &&
flock -n /tmp/lazyboy/screen-0.lock true
'
echo "running $repeat Cua smoke iterations"
set +e
docker exec -u 1000:1000 "$name" /usr/local/bin/lazyboy-cua-smoke --repeat "$repeat" --ready-timeout "$ready_timeout"
@ -87,10 +96,22 @@ if [[ "$code" -eq 0 ]]; then
docker exec -u 1000:1000 "$name" python3 /usr/local/bin/lazyboy-cua-adapter-test --repeat "$repeat"
code=$?
fi
if [[ "$code" -eq 0 ]]; then
docker exec -u 1000:1000 "$name" python3 /usr/local/bin/lazyboy-cua-terminal-test
code=$?
fi
if [[ "$code" -eq 0 ]]; then
docker exec -u 1000:1000 "$name" python3 /usr/local/bin/lazyboy-cua-clipboard-test
code=$?
fi
if [[ "$code" -eq 0 ]]; then
docker exec -u 1000:1000 "$name" python3 /usr/local/bin/lazyboy-cua-isolation-test
code=$?
fi
if [[ "$code" -eq 0 ]]; then
docker exec -u 1000:1000 "$name" python3 /usr/local/bin/lazyboy-cua-session-test
code=$?
fi
if [[ "$code" -eq 0 ]]; then
docker pause "$name" >/dev/null && docker unpause "$name" >/dev/null
code=$?
@ -127,11 +148,25 @@ if [[ "$code" -eq 0 ]]; then
docker exec -u 1000:1000 "$name" python3 /usr/local/bin/lazyboy-cua-adapter-test --check-persistence
code=$?
fi
if [[ "$code" -eq 0 ]]; then
docker exec -u 1000:1000 "$name" python3 /usr/local/bin/lazyboy-cua-cursor-test
code=$?
fi
if [[ "$code" -eq 0 ]]; then
docker exec -u 1000:1000 "$name" python3 /usr/local/bin/lazyboy-cua-cursor-color-test
code=$?
fi
set -e
out="${CUA_SMOKE_OUT:-/tmp/lazyboy-cua-smoke-last}"
mkdir -p "$out"
docker cp "$name:/tmp/lazyboy/cua-smoke-report/." "$out/" 2>/dev/null || true
for index in 0 1; do
docker cp "$name:/tmp/cua-cursor-${index}.png" "$out/cua-cursor-${index}.png" 2>/dev/null || true
done
for color in purple green pink; do
docker cp "$name:/tmp/cua-cursor-${color}.png" "$out/cua-cursor-${color}.png" 2>/dev/null || true
done
docker exec -u 1000:1000 "$name" sh -c 'tail -n 80 /tmp/lazyboy/screen-1-cua.log 2>/dev/null || true' \
>"$out/cua-driver.log" || true

70
scripts/cua-terminal-test.py Normal file
View File

@ -0,0 +1,70 @@
#!/usr/bin/env python3
"""Real terminal persistence/interrupt/reset via controld's Cua action route.
Run inside a disposable computer container. Files are test oracles, never an
agent-side output channel. Every application/input action goes through Cua.
"""
import base64
import importlib.machinery
import importlib.util
import time
import os
from pathlib import Path
loader = importlib.machinery.SourceFileLoader("adapter", "/usr/local/bin/lazyboy-cua-adapter-test")
spec = importlib.util.spec_from_loader(loader.name, loader)
adapter = importlib.util.module_from_spec(spec)
loader.exec_module(adapter)
adapter.wait_health()
root = Path('/tmp/lazyboy/cua-terminal-test')
root.mkdir(exist_ok=True)
title = 'LazyBoy terminal cua-persistence-' + str(time.monotonic_ns())
for filename in ['unicode.txt', 'persistence.txt', 'interrupt.txt', 'reset.txt', 'background.pid']:
(root / filename).unlink(missing_ok=True)
def act(actions, name, settle=1000):
result = adapter.api('/act', {'actions': actions, 'observe': True, 'settle_ms': settle})
frame = base64.b64decode(result['png_base64'])
assert frame.startswith(b'\x89PNG'), 'missing shared-desktop frame'
(root / (name + '.png')).write_bytes(frame)
return result
def key(value):
return {'kind': 'key', 'key': value}
def encoded_command(value):
encoded = ''.join('\\\\' if b == 92 else "\\'" if b == 39 else chr(b) if 32 <= b <= 126 else f'\\x{b:02x}' for b in value.encode())
return "eval $'" + encoded + "'"
def command(value, name):
value = encoded_command(value)
return act([{'kind':'focus', 'title':title}, {'kind':'clipboard','text':value}, key('return')], name)
first_command = "export CUA_TERMINAL_TEST=retained; cd /tmp/lazyboy/cua-terminal-test; printf '中文 hello-cua\\n' > unicode.txt"
act([{'kind':'launch','application':'terminal','uri':'--title='+title},
{'kind':'clipboard','text':encoded_command(first_command)}, key('return')], 'first')
assert (root/'unicode.txt').read_text() == '中文 hello-cua\n'
command("printf '%s\\n' \"$CUA_TERMINAL_TEST\" \"$PWD\" > persistence.txt", 'persist')
assert (root/'persistence.txt').read_text() == 'retained\n/tmp/lazyboy/cua-terminal-test\n'
command("sleep 30", 'running')
act([{'kind':'focus','title':title},key('ctrl+c')], 'interrupt', 300)
command("printf 'interrupted-ok\\n' > interrupt.txt", 'after-interrupt')
assert (root/'interrupt.txt').read_text() == 'interrupted-ok\n'
command("sleep 60 & echo $! > background.pid", 'background')
background_pid = int((root/'background.pid').read_text())
os.kill(background_pid, 0)
command("exec /usr/local/bin/lazyboy-terminal-reset", 'reset')
try:
os.kill(background_pid, 0)
except ProcessLookupError:
pass
else:
raise AssertionError('reset left its background job running')
command("printf '%s\\n' \"${CUA_TERMINAL_TEST-unset}\" \"$PWD\" > /tmp/lazyboy/cua-terminal-test/reset.txt", 'after-reset')
assert (root/'reset.txt').read_text() == 'unset\n/home/lazyboy\n'
command("printf 'CUA terminal: 中文, persistence, interrupt and reset passed\\n'", 'complete')
print('Cua terminal persistence, Unicode, interrupt and reset passed', flush=True)

75
tests/control.test.py
View File

@ -1,75 +0,0 @@
import contextlib
import io
import json
import importlib.util
import subprocess
import unittest
from pathlib import Path
from unittest.mock import patch
def module(name, path):
spec=importlib.util.spec_from_file_location(name,path)
mod=importlib.util.module_from_spec(spec);spec.loader.exec_module(mod);return mod
clipboard=module('clipboard',Path('crates/control/src/clipboard.py'))
cdp=module('cdp',Path('crates/control/src/cdp.py'))
class ClipboardTest(unittest.TestCase):
def test_confirmed_unicode_then_terminal_shortcut(self):
calls=[]
def fake(argv,**kwargs):
calls.append(argv)
data=b''
if '-out' in argv: data='中文\nemoji🙂'.encode()
if 'getactivewindow' in argv:data=b'123'
if 'WM_CLASS' in argv:data=b'xfce4-terminal'
return subprocess.CompletedProcess(argv,0,stdout=data)
with patch.object(clipboard,'run',fake): clipboard.paste('中文\nemoji🙂')
self.assertEqual(calls[-1][-1],'ctrl+shift+v')
self.assertEqual(sum('key' in a for a in calls),1)
def test_no_paste_when_sync_fails(self):
calls=[]
def fake(argv,**kwargs):calls.append(argv);return subprocess.CompletedProcess(argv,0,stdout=b'stale')
with patch.object(clipboard,'run',fake),patch.object(clipboard.time,'monotonic',side_effect=[0,3]):
with self.assertRaises(RuntimeError):clipboard.paste('new')
self.assertFalse(any('key' in a for a in calls))
def test_cdp_handles_events_before_response(self):
class Socket:
def __init__(self):self.items=iter(['{"method":"Page.event"}','{"id":1,"result":{"ok":true}}'])
def send(self,x):pass
def settimeout(self,x):pass
def recv(self):return next(self.items)
ws=cdp.Ws.__new__(cdp.Ws);ws.sock=Socket();ws.n=0
self.assertEqual(ws.call('Runtime.test'),{'ok':True})
class BrowserActionTest(unittest.TestCase):
def run_action(self, request, evaluation=None):
calls=[]
class Socket:
def call(self, method, params=None): calls.append((method, params)); return {}
def close(self): pass
output=io.StringIO()
def evaluate(ws, expression, arg=None):
if expression == cdp.SNAP_JS:
return {"url":"https://example.test/next", "title":"Next", "text":"Saved", "elements":[{"id":1,"title":"Continue"}]}
return evaluation if evaluation is not None else {"ok":True,"x":10,"y":20}
with patch.object(cdp.sys,'argv',['cdp',json.dumps(request)]), patch.object(cdp,'probe',return_value=True), patch.object(cdp,'connect',return_value=Socket()), patch.object(cdp,'evaluate',side_effect=evaluate), patch.object(cdp,'pointer'), patch.object(cdp,'wait_for_visual_update'), patch.object(cdp,'wait_until_enabled',return_value=0), patch.object(cdp.time,'sleep'), contextlib.redirect_stdout(output):
try: cdp.main()
except SystemExit: pass
return json.loads(output.getvalue()), calls
def test_every_browser_action_returns_current_elements_and_text(self):
for action in ['click','type','press','wait','navigate']:
with self.subTest(action=action):
result,_=self.run_action({"action":action,"selector":"#field","text":"hello","url":"https://example.test"})
self.assertTrue(result['ok'])
self.assertEqual(result['text'],'Saved')
self.assertEqual(result['elements'][0]['title'],'Continue')
def test_missing_field_never_types_into_previous_focus(self):
result,calls=self.run_action({"action":"type","selector":"#gone","text":"private"},{"ok":False,"error":"element gone"})
self.assertFalse(result['ok'])
self.assertFalse(any(method=='Input.insertText' for method,_ in calls))
if __name__=='__main__':unittest.main()

8
tests/frontend.test.mjs
View File

@ -40,13 +40,7 @@ test('VNC paste delegates once to confirmed backend and rejects another source',
handlers.message({origin:'http://localhost',source:{},data:{type:'lazyboy-host-clipboard',text:'bad'}});assert.equal(sent.at(-1).text,'中文\nhello');
});
test('saved login rejects HTTP, lookalike hosts, and missing host before touching fields',()=>{
const py=fs.readFileSync('crates/control/src/cdp.py','utf8');const expression=py.match(/FILL_LOGIN_JS = r"""([\s\S]*?)"""/)[1];
for(const [protocol,hostname,expectedHost] of [['https:','evil.example','bank.example'],['http:','bank.example','bank.example'],['https:','bank.example.evil','bank.example'],['https:','bank.example','']]) {
const evaluate=vm.runInNewContext(`(${expression})`,{location:{protocol,hostname},document:{querySelectorAll(){throw Error('must not touch fields')}}});
assert.equal(evaluate({username:'u',password:'secret',expectedHost}).ok,false);
}
});
const mdJs=ts.transpileModule(fs.readFileSync('apps/web/src/markdown.tsx','utf8'),{compilerOptions:{module:ts.ModuleKind.CommonJS,jsx:ts.JsxEmit.ReactJSX}}).outputText;
const mdBox={exports:{},require:(name)=>{

130
tests/shell-session.test.py
View File

@ -1,130 +0,0 @@
"""The agent's persistent terminal (image/computer/lazyboy-shell).
Run on the host or inside the desktop image: it needs tmux only, and talks to a
private tmux socket in a temp directory so it never touches a real session.
"""
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[1]
SCRIPT = ROOT / 'image' / 'computer' / 'lazyboy-shell'
@unittest.skipUnless(shutil.which('tmux'), 'tmux is not installed')
class ShellSessionTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.temp = tempfile.TemporaryDirectory()
cls.env = {
'PATH': os.environ.get('PATH', '/usr/bin:/bin'),
'HOME': cls.temp.name,
'TMUX_TMPDIR': f'{cls.temp.name}/tmux',
'LAZYBOY_SHELL_STATE': f'{cls.temp.name}/state',
'LANG': 'C.UTF-8',
'LC_ALL': 'C.UTF-8',
}
os.makedirs(cls.env['TMUX_TMPDIR'], exist_ok=True)
os.makedirs(cls.env['LAZYBOY_SHELL_STATE'], exist_ok=True)
@classmethod
def tearDownClass(cls):
subprocess.run(['tmux', 'kill-server'], env=cls.env,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
cls.temp.cleanup()
def shell(self, *argv, timeout=40):
return subprocess.run([str(SCRIPT), *argv], env=self.env,
capture_output=True, text=True, timeout=timeout)
def run_(self, command, session='main', wait_ms=8000):
return self.shell('run', session, str(wait_ms), command)
def test_reports_output_and_exit_code(self):
result = self.run_('echo ready')
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn('status=done session=main exit=0', result.stdout)
self.assertIn('ready', result.stdout)
failed = self.run_('echo boom >&2; exit 7')
self.assertIn('exit=7', failed.stdout)
self.assertIn('boom', failed.stdout)
def test_working_directory_and_env_survive_between_calls(self):
# The whole point: this is one terminal, not a fresh process per call.
self.run_('cd /tmp && export LAZYBOY_MARKER=kept')
here = self.run_('pwd; echo $LAZYBOY_MARKER')
self.assertIn('/tmp', here.stdout)
self.assertIn('kept', here.stdout)
def test_a_still_running_command_is_reported_not_typed_over(self):
slow = self.run_('echo starting; sleep 30', wait_ms=500)
self.assertIn('status=running', slow.stdout)
self.assertIn('starting', slow.stdout)
crowded = self.run_('echo second', wait_ms=1000)
self.assertIn('status=running', crowded.stdout)
self.assertIn('nothing was typed', crowded.stdout)
self.assertNotIn('second', crowded.stdout)
# Ctrl-C is the human answer: it releases the terminal, and the next
# command runs in the same shell.
self.assertIn('status=sent', self.shell('keys', 'main', 'C-c').stdout)
after = self.run_('echo usable', wait_ms=8000)
self.assertIn('status=done', after.stdout)
self.assertIn('usable', after.stdout)
def test_log_reads_the_terminal_without_typing_anything(self):
self.run_('echo logged', session='poll')
logged = self.shell('log', 'poll', '40')
self.assertIn('status=idle', logged.stdout)
self.assertIn('logged', logged.stdout)
def test_a_command_that_exits_the_shell_leaves_a_usable_terminal(self):
# `exit` is a command like any other: the code is reported, and the next
# call gets the same terminal back where it stood.
exited = self.run_('export LB_KEEP=through_exit; cd /etc; exit 3',
session='exiter')
self.assertIn('exit=3', exited.stdout)
self.assertIn('restarted', exited.stdout)
after = self.run_('pwd; echo marker=$LB_KEEP', session='exiter')
self.assertIn('status=done', after.stdout)
self.assertIn('/etc', after.stdout)
self.assertIn('marker=through_exit', after.stdout)
def test_a_shell_replaced_by_the_command_is_adopted(self):
# `exec bash` swallows the end marker; the terminal must not stay locked.
self.run_('exec bash', session='swapped', wait_ms=1500)
after = self.run_('echo recovered', session='swapped')
self.assertIn('status=done', after.stdout)
self.assertIn('recovered', after.stdout)
def test_log_reports_whether_the_command_is_still_running(self):
self.run_('echo starting; sleep 30', session='watched', wait_ms=500)
busy = self.shell('log', 'watched', '20')
self.assertIn('status=running', busy.stdout)
self.assertIn('starting', busy.stdout)
self.shell('keys', 'watched', 'C-c')
idle = self.shell('log', 'watched', '20')
self.assertIn('status=idle', idle.stdout)
def test_sessions_are_listed_and_resettable(self):
self.run_('echo hi', session='alpha')
self.assertIn('session=alpha state=open', self.shell('list').stdout)
self.assertIn('session=alpha', self.shell('reset', 'alpha').stdout)
self.run_('pwd', session='alpha')
def test_rejects_session_names_that_are_not_boring(self):
for bad in ('', 'two words', 'a;rm -rf /'):
result = self.run_('echo nope', session=bad)
self.assertEqual(result.returncode, 2, bad)
self.assertIn('session name', result.stderr)
def test_unknown_subcommand_is_an_error(self):
result = self.shell('nonsense')
self.assertEqual(result.returncode, 2)
self.assertIn('usage:', result.stderr)
if __name__ == '__main__':
unittest.main()