add cua

.env.example

@@ -22,9 +22,9 @@ LAZYBOY_COMPUTER_PIDS=2048
 # Only affects the Agent desktop container. Disabled by default.
 LAZYBOY_COMPUTER_SUDO=false
 # Computer-control backend inside each desktop container. Rebuild/recreate
-# computers after changing. `cua` is Cua Driver (default); `legacy` is
+# computers after changing. `cua` is the opt-in Cua Driver backend; `legacy` (default) is
 # CDP/AT-SPI/xdotool rollback.
-LAZYBOY_COMPUTER_DRIVER=cua
+LAZYBOY_COMPUTER_DRIVER=legacy
 # Linux only (optional): point this at the host's LXCFS root to make htop/free
 # report the per-Agent cgroup quota. Leave the default empty directory on macOS.
 LAZYBOY_LXCFS_ROOT=./data/lxcfs

a.md

@@ -1,5 +1,7 @@
 # LazyBoy → Cua Driver Migration Plan
 
+> 2026-09-07 檢查：Phase 1 規格尚未全部勾完（生產預設仍是 legacy；takeover／錄製端到端未另開測）。opt-in Cua 已可在現有桌面容器使用，驗收見 [docs/cua-review.md](docs/cua-review.md)。本文件仍是目標規格，不能視為完成證明。
+
 > **Purpose:** This document is an implementation specification for a coding agent.
 >
 > Repository: `https://code.30cm.net/daniel.w/lazyBoy`

crates/api/src/tools.rs

@@ -539,7 +539,11 @@ async fn attach_ui_elements(
         .as_ref()
         .map(|page| !page.ok || page.elements.is_empty())
         .unwrap_or(true);
-    let a11y = a11y_snapshot(ctx, include_browser).await;
+    let a11y = if observation.native_observation_complete {
+        None
+    } else {
+        a11y_snapshot(ctx, include_browser).await
+    };
     let page_elements = page
         .as_ref()
         .map(|page| page.elements.as_slice())
@@ -549,7 +553,21 @@ async fn attach_ui_elements(
         .filter(|page| page.ok)
         .map(|page| page.elements.as_slice())
         .unwrap_or(&[]);
-    observation.elements = merge_ui_elements(observation.elements, page_elements, a11y_elements);
+    if observation.native_observation_complete {
+        let native: Vec<_> = observation
+            .elements
+            .iter()
+            .filter(|element| element.kind.as_deref() == Some("a11y"))
+            .cloned()
+            .collect();
+        observation
+            .elements
+            .retain(|element| element.kind.as_deref() != Some("a11y"));
+        observation.elements = merge_ui_elements(observation.elements, page_elements, &native);
+    } else {
+        observation.elements =
+            merge_ui_elements(observation.elements, page_elements, a11y_elements);
+    }
     let mut note = note.to_string();
     if let Some(page) = page.as_ref().filter(|page| page.ok) {
         if !page.url.is_empty() || !page.title.is_empty() {
@@ -980,6 +998,11 @@ async fn apply_semantic_actions(ctx: &ToolCtx, items: &mut [Value], elements: &[
         else {
             continue;
         };
+        // Snapshot-scoped Cua targets must reach the controller unchanged.
+        // Legacy AT-SPI cannot resolve them, and pixel fallback would bypass staleness checks.
+        if target.starts_with("cua:") {
+            continue;
+        }
         let ref_kind = item
             .get("refKind")
             .and_then(Value::as_str)

crates/contracts/src/action.rs

@@ -137,6 +137,9 @@ impl UiElement {
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct ComputerObservation {
+    /// The controller already populated native semantics; skip legacy enrichment.
+    #[serde(default)]
+    pub native_observation_complete: bool,
     pub frame_id: String,
     pub captured_at: String,
     pub mime_type: String,

crates/control/src/controller.rs

@@ -90,7 +90,7 @@ fn truncate_error(text: &str) -> String {
     if trimmed.len() <= LIMIT {
         trimmed.to_string()
     } else {
-        format!("{}…", &trimmed[..LIMIT])
+        format!("{}…", &trimmed[..trimmed.floor_char_boundary(LIMIT)])
     }
 }
 
@@ -105,15 +105,15 @@ impl ComputerDriver {
 
     pub fn from_env() -> Self {
         match std::env::var(Self::ENV) {
-            Ok(value) if value.trim().is_empty() => Self::Cua,
+            Ok(value) if value.trim().is_empty() => Self::Legacy,
             Ok(value) => match value.parse() {
                 Ok(driver) => driver,
                 Err(error) => {
-                    tracing::error!("{error}; using cua");
+                    tracing::error!("{error}; using legacy");
-                    Self::Cua
+                    Self::Legacy
                 }
             },
-            Err(_) => Self::Cua,
+            Err(_) => Self::Legacy,
         }
     }
 
@@ -205,6 +205,15 @@ mod tests {
         assert_eq!(ComputerDriver::Cua.as_str(), "cua");
     }
 
+    #[test]
+    fn long_unicode_errors_do_not_panic() {
+        let text = "錯".repeat(300);
+        let error = ControlError::internal(&text).to_string();
+        assert!(error.ends_with('…'));
+        assert!(error.len() <= 803);
+        assert!(text.starts_with(error.trim_end_matches('…')));
+    }
+
     #[test]
     fn invalid_browser_action_is_a_client_error() {
         let error = ControlError::InvalidAction(

crates/control/src/cua/browser.rs

@@ -3,7 +3,7 @@ use serde_json::{Value, json};
 use tokio::time::{Duration, sleep};
 
 use super::ListedWindow;
-use super::client::{CuaClient, first_array_of_objects};
+use super::client::CuaClient;
 use crate::controller::ControlError;
 use crate::process::spawn_detached;
 use crate::{BrowserRequest, CdpPage, launch_argv_on};
@@ -16,6 +16,7 @@ pub struct BrowserBind {
     pub window_id: u64,
     pub target_id: String,
     pub tab_id: String,
+    pub page: Option<CdpPage>,
 }
 
 pub fn is_cua_ref(selector: &str) -> bool {
@@ -49,7 +50,13 @@ pub fn page_from_semantic(value: &Value) -> CdpPage {
         .to_string();
     let outline = value.get("outline").and_then(Value::as_str).unwrap_or("");
     let mut elements = Vec::new();
-    for (index, item) in first_array_of_objects(value, "ref").into_iter().enumerate() {
+    for (index, item) in value
+        .get("refs")
+        .and_then(Value::as_array)
+        .into_iter()
+        .flatten()
+        .enumerate()
+    {
         let Some(element) = element_from_ref(index as u32 + 1, item) else {
             continue;
         };
@@ -98,7 +105,7 @@ fn element_from_ref(id: u32, item: &Value) -> Option<UiElement> {
         .and_then(Value::as_str)
         .unwrap_or("in_viewport");
     let (x, y, w, h) = match item.get("frame") {
-        Some(frame) => (
+        Some(frame) if frame.is_object() => (
             number(frame, "x").unwrap_or(0),
             number(frame, "y").unwrap_or(0),
             number(frame, "w")
@@ -108,8 +115,8 @@ fn element_from_ref(id: u32, item: &Value) -> Option<UiElement> {
                 .or_else(|| number(frame, "height"))
                 .unwrap_or(0),
         ),
-        None if visibility == "in_viewport" => (0, 0, 1, 1),
+        _ if visibility == "in_viewport" => (0, 0, 1, 1),
-        None => (0, 0, 0, 0),
+        _ => (0, 0, 0, 0),
     };
     Some(UiElement {
         id,
@@ -148,8 +155,7 @@ pub fn find_ref<'a>(page: &'a CdpPage, selector: &'a str) -> Option<&'a str> {
             .elements
             .iter()
             .find(|element| element.selector.as_deref() == Some(selector))
-            .and_then(|element| element.selector.as_deref())
+            .and_then(|element| element.selector.as_deref());
-            .or(Some(selector));
     }
     if let Ok(id) = selector.parse::<u32>() {
         return page
@@ -158,23 +164,20 @@ pub fn find_ref<'a>(page: &'a CdpPage, selector: &'a str) -> Option<&'a str> {
             .find(|element| element.id == id)
             .and_then(|element| element.selector.as_deref());
     }
-    let needle = selector
+    // Labels are accepted only when exact and unique; do not reinterpret CSS
-        .trim_start_matches(['#', '.', '['])
+    // fragments as substring matches that can click a different control.
-        .trim_end_matches(']')
+    if selector.trim().is_empty() {
-        .to_ascii_lowercase();
+        return None;
-    page.elements.iter().find_map(|element| {
-        let title = element.title.to_ascii_lowercase();
-        let role = element
-            .role
-            .clone()
-            .unwrap_or_default()
-            .to_ascii_lowercase();
-        if title.contains(&needle) || role == needle {
-            element.selector.as_deref()
-        } else {
-            None
     }
-    })
+    let mut matches = page
+        .elements
+        .iter()
+        .filter(|element| element.title == selector);
+    let first = matches.next()?;
+    if matches.next().is_some() {
+        return None;
+    }
+    first.selector.as_deref()
 }
 
 pub fn chromium_window(windows: &[ListedWindow]) -> Option<&ListedWindow> {
@@ -249,6 +252,14 @@ async fn attach(
     display: &str,
     window: &ListedWindow,
 ) -> Result<BrowserBind, ControlError> {
+    client
+        .call(
+            display,
+            "start_session",
+            &json!({ "session": SESSION }),
+            &[],
+        )
+        .await?;
     let pid = window.pid;
     let window_id = window.id;
     let prepare = client
@@ -290,6 +301,7 @@ async fn attach(
         window_id,
         target_id,
         tab_id,
+        page: None,
     })
 }
 
@@ -409,11 +421,7 @@ async fn click(
         .selector
         .as_deref()
         .ok_or_else(|| ControlError::InvalidAction("browser click needs a selector".into()))?;
-    let wait_ms = request.wait_ms.unwrap_or(45_000).min(120_000);
+    let page = bind.page.as_ref().ok_or(ControlError::StaleReference)?;
-    let deadline = tokio::time::Instant::now() + Duration::from_millis(wait_ms);
-    let mut waited = 0.0f64;
-    loop {
-        let page = snapshot(client, display, bind).await?;
     let Some(r#ref) = find_ref(&page, selector) else {
         return Ok(CdpPage {
                 ok: false,
@@ -421,15 +429,15 @@ async fn click(
                     "element gone: the page changed and ids were renumbered. Use the fresh element list in this result."
                         .into(),
                 ),
-                url: page.url,
+                url: page.url.clone(),
-                title: page.title,
+                title: page.title.clone(),
-                text: page.text,
+                text: page.text.clone(),
-                elements: page.elements,
+                elements: page.elements.clone(),
                 ..CdpPage::default()
             });
     };
     let r#ref = r#ref.to_string();
-        match client
+    client
         .call(
             display,
             "browser_click",
@@ -442,24 +450,9 @@ async fn click(
             }),
             &[],
         )
-            .await
+        .await?;
-        {
-            Ok(_) => {
     sleep(Duration::from_millis(250)).await;
-                let mut after = snapshot(client, display, bind).await?;
+    snapshot(client, display, bind).await
-                if waited >= 1.0 {
-                    after.waited_seconds = Some((waited * 10.0).round() / 10.0);
-                }
-                return Ok(after);
-            }
-            Err(error) if tokio::time::Instant::now() < deadline => {
-                waited += 0.5;
-                tracing::info!(error = %error, "browser click retrying");
-                sleep(Duration::from_millis(500)).await;
-            }
-            Err(error) => return Err(error),
-        }
-    }
 }
 
 async fn type_into(
@@ -471,15 +464,15 @@ async fn type_into(
     let text = request.text.clone().unwrap_or_default();
     tracing::info!(backend = "cua", tool = "browser_type", length = text.len());
     if let Some(selector) = request.selector.as_deref() {
-        let page = snapshot(client, display, bind).await?;
+        let page = bind.page.as_ref().ok_or(ControlError::StaleReference)?;
         let Some(r#ref) = find_ref(&page, selector) else {
             return Ok(CdpPage {
                 ok: false,
                 error: Some("target field is unavailable; no text inserted".into()),
-                url: page.url,
+                url: page.url.clone(),
-                title: page.title,
+                title: page.title.clone(),
-                text: page.text,
+                text: page.text.clone(),
-                elements: page.elements,
+                elements: page.elements.clone(),
                 ..CdpPage::default()
             });
         };
@@ -545,8 +538,9 @@ mod tests {
             "status": "ok",
             "outline": "- button \"Smoke Click\"\n- textbox \"Smoke Entry\"",
             "page": { "title": "LazyBoy Cua Smoke", "url": "http://127.0.0.1:8765/cua-smoke.html" },
+            "content_refs": [{ "ref": "p1:0", "name": "Page heading", "role": "heading" }],
             "refs": [
-                { "name": "Smoke Click", "ref": "p1:1", "role": "button", "visibility": "in_viewport" },
+                { "name": "Smoke Click", "ref": "p1:1", "role": "button", "frame": "main", "visibility": "in_viewport" },
                 { "name": "Smoke Entry", "ref": "p1:2", "role": "textbox", "visibility": "in_viewport" }
             ]
         });
@@ -557,8 +551,12 @@ mod tests {
         assert_eq!(page.elements[0].id, 1);
         assert_eq!(page.elements[0].selector.as_deref(), Some("p1:1"));
         assert_eq!(page.elements[0].kind.as_deref(), Some("dom"));
+        assert!(!page.elements[0].is_offscreen());
         assert_eq!(find_ref(&page, "1"), Some("p1:1"));
         assert_eq!(find_ref(&page, "p1:1"), Some("p1:1"));
+        assert_eq!(find_ref(&page, "p9:1"), None);
+        assert_eq!(find_ref(&page, ""), None);
+        assert_eq!(find_ref(&page, "#"), None);
         assert_eq!(find_ref(&page, "Smoke Entry"), Some("p1:2"));
     }
 

crates/control/src/cua/client.rs

@@ -1,7 +1,9 @@
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
-use std::time::Instant;
+use std::process::Stdio;
+use std::time::{Duration, Instant};
 
 use serde_json::Value;
+use tokio::io::AsyncWriteExt;
 use tokio::process::Command;
 
 use crate::controller::ControlError;
@@ -42,11 +44,9 @@ impl CuaClient {
     }
 
     pub async fn version(&self) -> Result<String, ControlError> {
-        let output = Command::new(&self.bin)
+        let mut command = Command::new(&self.bin);
-            .arg("--version")
+        command.arg("--version");
-            .output()
+        let output = bounded_output(&mut command, Duration::from_secs(10)).await?;
-            .await
-            .map_err(|_| ControlError::DriverUnavailable)?;
         if !output.status.success() {
             return Err(ControlError::DriverUnavailable);
         }
@@ -58,11 +58,9 @@ impl CuaClient {
         if !socket.exists() {
             return Err(ControlError::DriverUnavailable);
         }
-        let output = Command::new(&self.bin)
+        let mut command = Command::new(&self.bin);
-            .args(["status", "--socket", &socket.to_string_lossy()])
+        command.args(["status", "--socket", &socket.to_string_lossy()]);
-            .output()
+        let output = bounded_output(&mut command, Duration::from_secs(10)).await?;
-            .await
-            .map_err(|_| ControlError::DriverUnavailable)?;
         let text = format!(
             "{}{}",
             String::from_utf8_lossy(&output.stdout),
@@ -90,36 +88,106 @@ impl CuaClient {
             .env("DISPLAY", normalize_display(screen))
             .env(
                 "CUA_DRIVER_RS_HOME",
-                std::env::var("CUA_DRIVER_RS_HOME")
+                format!(
-                    .unwrap_or_else(|_| "/tmp/lazyboy/cua-home".into()),
+                    "/tmp/lazyboy/cua-home-{}",
+                    normalize_display(screen).trim_start_matches(':')
+                ),
             )
             .args(["call", "--socket", &socket.to_string_lossy()]);
         command.args(extra);
         command.arg(tool);
-        command.arg(payload.to_string());
+
         apply_desktop_bus(&mut command, screen);
         let started = Instant::now();
-        let output = command
+        let output = bounded_input_output(&mut command, payload).await?;
-            .output()
-            .await
-            .map_err(|error| ControlError::internal(error.to_string()))?;
         let stdout = String::from_utf8_lossy(&output.stdout);
         let stderr = String::from_utf8_lossy(&output.stderr);
         let combined = format!("{stdout}\n{stderr}");
+        let value =
+            parse_jsonish(&stdout).unwrap_or_else(|| Value::String(stdout.trim().to_string()));
+        let error = if !output.status.success() || stdout.trim_start().starts_with('❌') {
+            Some(classify_cua_failure(&combined))
+        } else {
+            response_error(&value)
+        };
         tracing::info!(
             backend = "cua",
             tool,
             screen,
             duration_ms = started.elapsed().as_millis() as u64,
-            success = output.status.success() && !combined.contains('❌')
+            success = error.is_none()
         );
-        if !output.status.success() || combined.contains('❌') {
+        if let Some(error) = error {
-            return Err(classify_cua_failure(&combined));
+            return Err(error);
         }
-        Ok(parse_jsonish(&stdout).unwrap_or(Value::String(stdout.trim().to_string())))
+
+        Ok(value)
     }
 }
 
+fn response_error(value: &Value) -> Option<ControlError> {
+    if value
+        .get("code")
+        .and_then(Value::as_str)
+        .is_some_and(|code| code != "ok")
+        || value.get("ok").and_then(Value::as_bool) == Some(false)
+        || value.get("isError").and_then(Value::as_bool) == Some(true)
+        || matches!(
+            value.get("status").and_then(Value::as_str),
+            Some("refused" | "error")
+        )
+    {
+        Some(classify_cua_failure(&value.to_string()))
+    } else {
+        None
+    }
+}
+
+async fn bounded_input_output(
+    command: &mut Command,
+    payload: &Value,
+) -> Result<std::process::Output, ControlError> {
+    command
+        .kill_on_drop(true)
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped());
+    tokio::time::timeout(Duration::from_secs(120), async {
+        let mut child = command
+            .spawn()
+            .map_err(|_| ControlError::DriverUnavailable)?;
+        let mut input = child.stdin.take().ok_or(ControlError::DriverUnhealthy)?;
+        let bytes = payload.to_string();
+        // Drain stdout/stderr while writing so large inputs cannot deadlock.
+        let write = async {
+            input.write_all(bytes.as_bytes()).await?;
+            input.shutdown().await?;
+            drop(input);
+            Ok::<(), std::io::Error>(())
+        };
+        let (written, output) = tokio::join!(write, child.wait_with_output());
+        written.map_err(|_| ControlError::DriverUnhealthy)?;
+        output.map_err(|_| ControlError::DriverUnhealthy)
+    })
+    .await
+    .map_err(|_| ControlError::Timeout)?
+}
+
+async fn bounded_output(
+    command: &mut Command,
+    timeout: Duration,
+) -> Result<std::process::Output, ControlError> {
+    command.kill_on_drop(true);
+    tokio::time::timeout(timeout, command.output())
+        .await
+        .map_err(|_| ControlError::Timeout)?
+        .map_err(|error| match error.kind() {
+            std::io::ErrorKind::NotFound => ControlError::DriverUnavailable,
+            std::io::ErrorKind::PermissionDenied => ControlError::PermissionDenied,
+            _ => ControlError::DriverUnhealthy,
+        })
+}
+
 fn apply_desktop_bus(command: &mut Command, display: &str) {
     let dbus = CuaClient::dbus_file(display);
     if let Ok(address) = std::fs::read_to_string(&dbus) {
@@ -194,7 +262,7 @@ pub fn first_array_of_objects<'a>(value: &'a Value, required: &str) -> Vec<&'a V
 
 fn classify_cua_failure(text: &str) -> ControlError {
     let lower = text.to_ascii_lowercase();
-    if lower.contains("stale") {
+    if lower.contains("stale") || (lower.contains("session") && lower.contains("ended")) {
         ControlError::StaleReference
     } else if lower.contains("not_found") || lower.contains("not found") {
         ControlError::TargetNotFound
@@ -202,15 +270,14 @@ fn classify_cua_failure(text: &str) -> ControlError {
         ControlError::Timeout
     } else if lower.contains("permission") || lower.contains("consent") {
         ControlError::PermissionDenied
+    } else if lower.contains("invalid_action_target") {
+        ControlError::InvalidAction("Cua rejected the action target".into())
     } else if lower.contains("unsupported") {
         ControlError::Unsupported
-    } else if Path::new(PRIMARY_SOCKET)
-        .parent()
-        .is_some_and(|dir| !dir.exists())
-    {
-        ControlError::DriverUnavailable
     } else {
-        ControlError::internal(text)
+        // Driver diagnostics can echo typed text or credentials. Keep raw
+        // output out of Agent-visible errors and downstream logs.
+        ControlError::DriverUnhealthy
     }
 }
 
@@ -218,6 +285,46 @@ fn classify_cua_failure(text: &str) -> ControlError {
 mod tests {
     use super::*;
 
+    #[test]
+    fn structured_refusals_are_errors_but_page_text_is_not() {
+        assert!(response_error(&serde_json::json!({"code": "invalid_action_target"})).is_some());
+        assert!(response_error(&serde_json::json!({"outline": "❌ payment declined"})).is_none());
+    }
+
+    #[tokio::test]
+    async fn piped_json_reaches_eof_without_argv_exposure() {
+        let mut command = Command::new("sh");
+        command.args(["-c", "cat"]);
+        let payload = serde_json::json!({"text": "秘密🙂"});
+        let output = tokio::time::timeout(
+            Duration::from_secs(2),
+            bounded_input_output(&mut command, &payload),
+        )
+        .await
+        .unwrap()
+        .unwrap();
+        assert_eq!(
+            serde_json::from_slice::<Value>(&output.stdout).unwrap(),
+            payload
+        );
+    }
+
+    #[tokio::test]
+    async fn hung_driver_is_bounded() {
+        let mut command = Command::new("sh");
+        command.args(["-c", "exec sleep 30"]);
+        assert!(matches!(
+            bounded_output(&mut command, Duration::from_millis(20)).await,
+            Err(ControlError::Timeout)
+        ));
+    }
+
+    #[test]
+    fn unknown_driver_failure_does_not_echo_secret() {
+        let error = classify_cua_failure("failed typing secret-password");
+        assert_eq!(error, ControlError::DriverUnhealthy);
+    }
+
     #[test]
     fn primary_display_uses_well_known_socket() {
         assert_eq!(

crates/control/src/cua/mod.rs

@@ -1,21 +1,24 @@
 mod browser;
 mod client;
+mod native;
 mod record;
 mod translate;
 
 use std::collections::HashMap;
 use std::path::PathBuf;
+use std::sync::Arc;
 use std::time::{SystemTime, UNIX_EPOCH};
 
 use async_trait::async_trait;
-use lazyboy_contracts::{ActiveWindow, ComputerObservation, CursorPosition, UiElement};
+use lazyboy_contracts::{
+    ActiveWindow, ComputerAction, ComputerObservation, CursorPosition, PointerType, UiElement,
+};
 use serde_json::{Value, json};
 use tokio::time::{Duration, sleep};
 
 use crate::controller::{
     ComputerController, ComputerDriver, ControlContext, ControlError, ControllerHealth,
 };
-use crate::legacy::apply_action as legacy_apply_action;
 use crate::process::spawn_detached;
 use crate::{
     ActionRequest, ActionResult, BrowserRequest, CdpPage, RecordingRequest, RecordingResult,
@@ -30,6 +33,8 @@ pub use translate::{TranslatedAction, translate_action};
 #[derive(Debug, Default)]
 pub struct CuaController {
     client: CuaClient,
+    screens: tokio::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
+    native: tokio::sync::Mutex<HashMap<String, HashMap<String, native::NativeTarget>>>,
     browser: tokio::sync::Mutex<HashMap<String, browser::BrowserBind>>,
 }
 
@@ -41,16 +46,36 @@ impl ComputerController for CuaController {
 
     async fn health(&self, ctx: &ControlContext) -> Result<ControllerHealth, ControlError> {
         let version = self.client.version().await.ok();
-        match self.client.status(&ctx.display).await {
+        let report = self
-            Ok(status) => Ok(ControllerHealth {
+            .client
-                backend: ComputerDriver::Cua.as_str().to_string(),
+            .call(&ctx.display, "health_report", &json!({}), &[])
+            .await;
+        match report {
+            Ok(report) => {
+                let compatible = version.as_deref() == Some("cua-driver 0.23.2");
+                let healthy =
+                    compatible && report.get("overall").and_then(Value::as_str) == Some("ok");
+                let mut details: Vec<String> = report
+                    .get("checks")
+                    .and_then(Value::as_array)
+                    .into_iter()
+                    .flatten()
+                    .filter(|check| check["status"] == "fail")
+                    .filter_map(|check| check["message"].as_str().map(str::to_string))
+                    .collect();
+                if !compatible {
+                    details.push("expected pinned Cua Driver 0.23.2".into());
+                }
+                Ok(ControllerHealth {
+                    backend: "cua".into(),
                     version,
-                healthy: true,
+                    healthy,
-                degraded: false,
+                    degraded: !healthy,
-                details: vec![status.trim().to_string()],
+                    details,
-            }),
+                })
+            }
             Err(error) => Ok(ControllerHealth {
-                backend: ComputerDriver::Cua.as_str().to_string(),
+                backend: "cua".into(),
                 version,
                 healthy: false,
                 degraded: true,
@@ -60,6 +85,7 @@ impl ComputerController for CuaController {
     }
 
     async fn observe(&self, ctx: &ControlContext) -> Result<ComputerObservation, ControlError> {
+        let _screen = self.lock_screen(&ctx.display).await;
         self.browser
             .lock()
             .await
@@ -72,29 +98,23 @@ impl ComputerController for CuaController {
         request: &ActionRequest,
         ctx: &ControlContext,
     ) -> Result<ActionResult, ControlError> {
+        let _screen = self.lock_screen(&ctx.display).await;
         let display = ctx.display.as_str();
         let profile = ctx.profile_path.as_deref();
-        let mut completed = 0usize;
+        let key = normalize_display(display).to_string();
-        for action in &request.actions {
+        self.browser.lock().await.remove(&key);
-            match translate_action(action, display, profile) {
+        // One snapshot for the whole batch. Taking the map per action made a
-                Ok(translated) => self.dispatch(display, translated).await?,
+        // leading wait (or a second ref) look stale.
-                Err(ControlError::Unsupported) => {
+        let mut targets = self.native.lock().await.remove(&key).unwrap_or_default();
-                    legacy_apply_action(display, profile, action).await?;
+        match self
-                }
+            .run_actions(request, display, profile, &mut targets)
-                Err(error) => return Err(error),
+            .await
-            }
+        {
-            let pause = action_pause_ms(action);
+            Ok(completed) => {
-            if pause > 0 {
-                sleep(Duration::from_millis(pause)).await;
-            }
-            completed += 1;
-        }
-        if request.settle_ms > 0 {
-            sleep(Duration::from_millis(u64::from(request.settle_ms))).await;
-        }
                 let observation = if request.observe {
                     Some(self.observe_display(display).await?)
                 } else {
+                    self.native.lock().await.insert(key, targets);
                     None
                 };
                 Ok(ActionResult {
@@ -102,12 +122,28 @@ impl ComputerController for CuaController {
                     observation,
                 })
             }
+            Err(error) => {
+                self.native.lock().await.insert(key, targets);
+                Err(error)
+            }
+        }
+    }
 
     async fn browser(
         &self,
         request: &BrowserRequest,
         ctx: &ControlContext,
     ) -> Result<CdpPage, ControlError> {
+        let _screen = self.lock_screen(&ctx.display).await;
+        if !matches!(
+            request.action.as_str(),
+            "snapshot" | "wait" | "probe" | "ensure"
+        ) {
+            self.native
+                .lock()
+                .await
+                .remove(normalize_display(&ctx.display));
+        }
         let key = normalize_display(&ctx.display).to_string();
         let mut windows = self.windows(&ctx.display).await.unwrap_or_default();
         let mut cache = self.browser.lock().await;
@@ -134,6 +170,7 @@ impl ComputerController for CuaController {
                 Err(error)
                     if attempt == 0
                         && had_bind
+                        && matches!(request.action.as_str(), "snapshot" | "wait" | "ensure")
                         && matches!(
                             error,
                             ControlError::BrowserUnavailable
@@ -144,7 +181,16 @@ impl ComputerController for CuaController {
                     windows = self.windows(&ctx.display).await.unwrap_or_default();
                 }
                 other => {
-                    if let Some(current) = bind {
+                    if let Some(mut current) = bind {
+                        current.page = match &other {
+                            Ok(page)
+                                if page.ok
+                                    && !matches!(request.action.as_str(), "probe" | "ensure") =>
+                            {
+                                Some(page.clone())
+                            }
+                            _ => None,
+                        };
                         cache.insert(key, current);
                     }
                     return map_browser_unavailable(other);
@@ -224,7 +270,81 @@ impl ComputerController for CuaController {
 }
 
 impl CuaController {
+    async fn run_actions(
+        &self,
+        request: &ActionRequest,
+        display: &str,
+        profile: Option<&str>,
+        targets: &mut HashMap<String, native::NativeTarget>,
+    ) -> Result<usize, ControlError> {
+        let mut completed = 0usize;
+        while completed < request.actions.len() {
+            let action = &request.actions[completed];
+            if let Some(payload) = drag_payload(&request.actions[completed..]) {
+                self.dispatch(
+                    display,
+                    TranslatedAction::Cua {
+                        tool: "drag",
+                        payload,
+                    },
+                )
+                .await?;
+                completed += 5;
+                continue;
+            }
+            if matches!(
+                action,
+                ComputerAction::Pointer {
+                    pointer_type: PointerType::Down | PointerType::Up,
+                    ..
+                }
+            ) {
+                return Err(ControlError::Unsupported);
+            }
+            if let ComputerAction::Ref {
+                verb,
+                target,
+                ref_kind,
+                text,
+            } = action
+            {
+                if ref_kind != "a11y" {
+                    return Err(ControlError::Unsupported);
+                }
+                let target = targets
+                    .get(target)
+                    .cloned()
+                    .ok_or(ControlError::StaleReference)?;
+                native::act(&self.client, display, target, *verb, text.as_deref()).await?;
+            } else {
+                let translated = translate_action(action, display, profile)?;
+                self.dispatch(display, translated).await?;
+            }
+            let pause = action_pause_ms(action);
+            if pause > 0 {
+                sleep(Duration::from_millis(pause)).await;
+            }
+            completed += 1;
+        }
+        if request.settle_ms > 0 {
+            sleep(Duration::from_millis(u64::from(request.settle_ms))).await;
+        }
+        Ok(completed)
+    }
+
+    async fn lock_screen(&self, display: &str) -> tokio::sync::OwnedMutexGuard<()> {
+        let lock = self
+            .screens
+            .lock()
+            .await
+            .entry(normalize_display(display).into())
+            .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
+            .clone();
+        lock.lock_owned().await
+    }
+
     async fn observe_display(&self, display: &str) -> Result<ComputerObservation, ControlError> {
+        self.native.lock().await.remove(normalize_display(display));
         let png_path = observe_png_path(display);
         let _ = tokio::fs::remove_file(&png_path).await;
         self.client
@@ -252,25 +372,38 @@ impl CuaController {
                 id: window.id.to_string(),
                 title: Some(window.title.clone()).filter(|title| !title.is_empty()),
             });
-        let elements: Vec<UiElement> = windows
+        let (mut elements, targets) =
+            native::observe(&self.client, display, &windows, &png_path.to_string_lossy()).await?;
+        elements.extend(
+            windows
                 .into_iter()
                 .enumerate()
                 .map(|(index, window)| UiElement {
                     id: (index + 1) as u32,
                     title: window.title.chars().take(80).collect(),
-                x: window.x,
+                    x: window.x.max(0) as u32,
-                y: window.y,
+                    y: window.y.max(0) as u32,
                     w: window.w,
                     h: window.h,
                     selector: None,
                     kind: Some("window".into()),
                     role: None,
                 })
-            .collect();
+                .collect::<Vec<_>>(),
-        Ok(observation_with_elements(
+        );
+        for (index, element) in elements.iter_mut().enumerate() {
+            element.id = (index + 1) as u32;
+        }
+        self.native
+            .lock()
+            .await
+            .insert(normalize_display(display).into(), targets);
+        let mut observation = observation_with_elements(
             observation_from_png(png, width, height, cursor, active),
             elements,
-        ))
+        );
+        observation.native_observation_complete = true;
+        Ok(observation)
     }
 
     async fn cursor(&self, display: &str) -> Option<CursorPosition> {
@@ -311,65 +444,39 @@ impl CuaController {
                 sleep(Duration::from_millis(ms)).await;
                 Ok(())
             }
-            TranslatedAction::LegacyArgv { argv, detached } => {
+            TranslatedAction::LegacyArgv { argv } => {
-                if detached {
                 spawn_detached(&argv).await.map_err(ControlError::internal)
-                } else {
-                    crate::process::run_output(&argv)
-                        .await
-                        .map_err(ControlError::internal)
-                        .and_then(|output| {
-                            if output.status.success() {
-                                Ok(())
-                            } else {
-                                Err(ControlError::internal(String::from_utf8_lossy(
-                                    &output.stderr,
-                                )))
-                            }
-                        })
-                }
             }
             TranslatedAction::FocusTitle { title } => self.focus_title(display, &title).await,
-            TranslatedAction::Cua { tool, payload } => {
+            TranslatedAction::Cua { tool, mut payload } => {
-                let payload = self.with_window_target(display, tool, payload).await?;
+                if tool == "drag" {
+                    let x = payload["from_x"].as_f64().unwrap_or(0.0);
+                    let y = payload["from_y"].as_f64().unwrap_or(0.0);
+                    let windows = self.windows(display).await?;
+                    let window = window_containing(&windows, x, y)
+                        .cloned()
+                        .ok_or(ControlError::TargetNotFound)?;
+                    for (key, offset) in [
+                        ("from_x", window.x),
+                        ("to_x", window.x),
+                        ("from_y", window.y),
+                        ("to_y", window.y),
+                    ] {
+                        payload[key] = json!(payload[key].as_f64().unwrap_or(0.0) - offset as f64);
+                    }
+                    payload["pid"] = json!(window.pid);
+                    payload["window_id"] = json!(window.id);
+                    payload["delivery_mode"] = json!("foreground");
+                }
                 self.client.call(display, tool, &payload, &[]).await?;
                 Ok(())
             }
         }
     }
 
-    async fn with_window_target(
-        &self,
-        display: &str,
-        tool: &str,
-        mut payload: Value,
-    ) -> Result<Value, ControlError> {
-        if tool != "mouse_button_down" && tool != "mouse_button_up" {
-            return Ok(payload);
-        }
-        if payload.get("pid").is_some() && payload.get("window_id").is_some() {
-            return Ok(payload);
-        }
-        let window = self
-            .windows(display)
-            .await?
-            .into_iter()
-            .max_by_key(|window| window.z)
-            .ok_or(ControlError::TargetNotFound)?;
-        if let Some(object) = payload.as_object_mut() {
-            object.insert("pid".into(), json!(window.pid));
-            object.insert("window_id".into(), json!(window.id));
-        }
-        Ok(payload)
-    }
-
     async fn focus_title(&self, display: &str, title: &str) -> Result<(), ControlError> {
-        let needle = title.to_ascii_lowercase();
         let windows = self.windows(display).await?;
-        let window = windows
+        let window = window_matching_title(&windows, title).ok_or(ControlError::TargetNotFound)?;
-            .into_iter()
-            .find(|window| window.title.to_ascii_lowercase().contains(&needle))
-            .ok_or(ControlError::TargetNotFound)?;
         self.client
             .call(
                 display,
@@ -382,14 +489,50 @@ impl CuaController {
     }
 }
 
+// The public drag DSL expands into these five actions. Keep the gesture in
+// one Cua call; separate CLI leases do not preserve held-button state.
+fn drag_payload(actions: &[ComputerAction]) -> Option<Value> {
+    let [
+        ComputerAction::Pointer {
+            x,
+            y,
+            pointer_type: PointerType::Down,
+            button,
+        },
+        ComputerAction::Wait { ms: 40 },
+        ComputerAction::Pointer {
+            x: to_x,
+            y: to_y,
+            pointer_type: PointerType::Move,
+            button: move_button,
+        },
+        ComputerAction::Wait { ms: 40 },
+        ComputerAction::Pointer {
+            x: up_x,
+            y: up_y,
+            pointer_type: PointerType::Up,
+            button: up_button,
+        },
+        ..,
+    ] = actions
+    else {
+        return None;
+    };
+    if (to_x, to_y, button) != (up_x, up_y, up_button) || button != move_button {
+        return None;
+    }
+    Some(json!({"from_x": x, "from_y": y, "to_x": to_x, "to_y": to_y,
+        "button": button.unwrap_or(lazyboy_contracts::PointerButton::Left), "duration_ms": 500}))
+}
+
 #[derive(Debug, Clone)]
 pub(crate) struct ListedWindow {
     pub(crate) id: u64,
     pub(crate) pid: u64,
     pub(crate) title: String,
     pub(crate) app_name: String,
-    pub(crate) x: u32,
+    pub(crate) x: i64,
-    pub(crate) y: u32,
+    pub(crate) y: i64,
     pub(crate) w: u32,
     pub(crate) h: u32,
     pub(crate) z: i64,
@@ -433,8 +576,14 @@ fn parse_listed_windows(value: &Value) -> Vec<ListedWindow> {
 
 fn listed_window(value: &Value) -> Option<ListedWindow> {
     let bounds = value.get("bounds");
-    let x = number(value, "x").or_else(|| bounds.and_then(|bounds| number(bounds, "x")))?;
+    let x = value
-    let y = number(value, "y").or_else(|| bounds.and_then(|bounds| number(bounds, "y")))?;
+        .get("x")
+        .or_else(|| bounds.and_then(|b| b.get("x")))?
+        .as_f64()? as i64;
+    let y = value
+        .get("y")
+        .or_else(|| bounds.and_then(|b| b.get("y")))?
+        .as_f64()? as i64;
     let w = number(value, "width")
         .or_else(|| number(value, "w"))
         .or_else(|| bounds.and_then(|bounds| number(bounds, "width")))?;
@@ -457,17 +606,50 @@ fn listed_window(value: &Value) -> Option<ListedWindow> {
             .and_then(Value::as_str)
             .unwrap_or("")
             .to_string(),
-        x: x as u32,
+        x,
-        y: y as u32,
+        y,
         w: w as u32,
         h: h as u32,
         z: number(value, "z_index").unwrap_or(0) as i64,
     })
 }
 
+fn window_containing(windows: &[ListedWindow], x: f64, y: f64) -> Option<&ListedWindow> {
+    windows
+        .iter()
+        .filter(|window| {
+            x >= window.x as f64
+                && y >= window.y as f64
+                && x < window.x as f64 + f64::from(window.w)
+                && y < window.y as f64 + f64::from(window.h)
+        })
+        .min_by_key(|window| u64::from(window.w.saturating_mul(window.h.max(1))))
+}
+
+fn window_matching_title<'a>(windows: &'a [ListedWindow], title: &str) -> Option<&'a ListedWindow> {
+    let needle = title.to_ascii_lowercase();
+    windows
+        .iter()
+        .find(|window| window.title.eq_ignore_ascii_case(title))
+        .or_else(|| {
+            windows.iter().find(|window| {
+                window.title.to_ascii_lowercase().contains(&needle)
+                    && !window.app_name.to_ascii_lowercase().contains("chrom")
+            })
+        })
+        .or_else(|| {
+            windows
+                .iter()
+                .find(|window| window.title.to_ascii_lowercase().contains(&needle))
+        })
+}
+
 fn ignored_window(window: &ListedWindow) -> bool {
     let title = window.title.to_ascii_lowercase();
-    title.is_empty() || title == "desktop" || title == "xfce4-panel" || title.contains("xfdesktop")
+    title.is_empty()
+        || title == "desktop"
+        || title == "xfce4-panel"
+        || window.app_name.to_ascii_lowercase().contains("xfdesktop")
 }
 
 fn number(value: &Value, key: &str) -> Option<u64> {
@@ -501,6 +683,18 @@ fn observe_png_path(display: &str) -> PathBuf {
 mod tests {
     use super::*;
 
+    #[test]
+    fn normalized_drag_uses_one_driver_gesture() {
+        let actions = crate::parse_computer_actions(
+            &json!([{"kind": "drag", "x": 10, "y": 20, "x2": 30, "y2": 40}]),
+        )
+        .unwrap();
+        let payload = drag_payload(&actions).unwrap();
+        assert_eq!(payload["from_x"], 10);
+        assert_eq!(payload["to_y"], 40);
+        assert!(drag_payload(&actions[..4]).is_none());
+    }
+
     #[test]
     fn window_list_skips_panel_and_numbers_from_one() {
         let raw = json!([
@@ -529,4 +723,81 @@ mod tests {
         assert_eq!(windows[0].app_name, "xfce4-terminal");
         assert_eq!(windows[0].w, 753);
     }
+
+    fn listed(
+        title: &str,
+        app: &str,
+        x: i64,
+        y: i64,
+        w: u32,
+        h: u32,
+        z: i64,
+        id: u64,
+    ) -> ListedWindow {
+        ListedWindow {
+            id,
+            pid: id,
+            title: title.into(),
+            app_name: app.into(),
+            x,
+            y,
+            w,
+            h,
+            z,
+        }
+    }
+
+    #[test]
+    fn drag_targets_the_smallest_containing_window() {
+        let gtk = listed(
+            "LazyBoy Cua Smoke",
+            "lazyboy-cua-smoke-gtk",
+            40,
+            40,
+            480,
+            240,
+            1,
+            2,
+        );
+        let chrome = listed(
+            "LazyBoy Cua Smoke - Chromium",
+            "Chromium",
+            0,
+            0,
+            1280,
+            759,
+            4,
+            3,
+        );
+        let windows = [chrome.clone(), gtk.clone()];
+        let hit = window_containing(&windows, 60.0, 70.0).unwrap();
+        assert_eq!(hit.id, gtk.id);
+    }
+
+    #[test]
+    fn focus_prefers_the_exact_native_title_over_chromium() {
+        let gtk = listed(
+            "LazyBoy Cua Smoke",
+            "lazyboy-cua-smoke-gtk",
+            40,
+            40,
+            480,
+            240,
+            1,
+            2,
+        );
+        let chrome = listed(
+            "LazyBoy Cua Smoke - Chromium",
+            "Chromium",
+            0,
+            0,
+            1280,
+            759,
+            4,
+            3,
+        );
+        let windows = [chrome, gtk];
+        let hit = window_matching_title(&windows, "LazyBoy Cua Smoke").unwrap();
+        assert_eq!(hit.app_name, "lazyboy-cua-smoke-gtk");
+    }
 }

crates/control/src/cua/native.rs

@@ -0,0 +1,109 @@
+use std::collections::HashMap;
+
+use lazyboy_contracts::{RefVerb, UiElement};
+use serde_json::{Value, json};
+
+use super::{ListedWindow, client::CuaClient};
+use crate::ControlError;
+
+#[derive(Debug, Clone)]
+pub(super) struct NativeTarget {
+    pid: u64,
+    window_id: u64,
+    token: String,
+}
+
+pub(super) async fn observe(
+    client: &CuaClient,
+    display: &str,
+    windows: &[ListedWindow],
+    generation: &str,
+) -> Result<(Vec<UiElement>, HashMap<String, NativeTarget>), ControlError> {
+    let mut elements = Vec::new();
+    let mut targets = HashMap::new();
+    for window in windows
+        .iter()
+        .filter(|window| !window.app_name.to_ascii_lowercase().contains("chrom"))
+    {
+        let state = client
+            .call(
+                display,
+                "get_window_state",
+                &json!({
+                    "pid": window.pid, "window_id": window.id, "include_screenshot": false,
+                    "max_elements": 200, "max_depth": 16,
+                }),
+                &[],
+            )
+            .await?;
+        for item in state
+            .get("elements")
+            .and_then(Value::as_array)
+            .into_iter()
+            .flatten()
+        {
+            let Some(token) = item.get("element_token").and_then(Value::as_str) else {
+                continue;
+            };
+            if item.get("enabled").and_then(Value::as_bool) == Some(false) {
+                continue;
+            }
+            let selector = format!("cua:{generation}:{}:{token}", window.id);
+            let title = item.get("label").and_then(Value::as_str).unwrap_or("");
+            let role = item.get("role").and_then(Value::as_str).unwrap_or("");
+            let frame = item.get("frame").unwrap_or(&Value::Null);
+            let coordinate = |key| {
+                frame
+                    .get(key)
+                    .and_then(Value::as_f64)
+                    .unwrap_or(0.0)
+                    .clamp(0.0, u32::MAX as f64) as u32
+            };
+            elements.push(UiElement {
+                id: 0,
+                title: if title.is_empty() {
+                    role.into()
+                } else {
+                    title.into()
+                },
+                x: coordinate("x"),
+                y: coordinate("y"),
+                w: coordinate("w"),
+                h: coordinate("h"),
+                selector: Some(selector.clone()),
+                kind: Some("a11y".into()),
+                role: Some(role.into()),
+            });
+            targets.insert(
+                selector,
+                NativeTarget {
+                    pid: window.pid,
+                    window_id: window.id,
+                    token: token.into(),
+                },
+            );
+        }
+    }
+    Ok((elements, targets))
+}
+
+pub(super) async fn act(
+    client: &CuaClient,
+    display: &str,
+    target: NativeTarget,
+    verb: RefVerb,
+    text: Option<&str>,
+) -> Result<(), ControlError> {
+    let mut payload =
+        json!({"pid": target.pid, "window_id": target.window_id, "element_token": target.token});
+    let tool = match verb {
+        RefVerb::Click => "click",
+        RefVerb::SetValue => {
+            payload["value"] = json!(text.unwrap_or(""));
+            "set_value"
+        }
+        RefVerb::Focus => return Err(ControlError::Unsupported),
+    };
+    client.call(display, tool, &payload, &[]).await?;
+    Ok(())
+}

crates/control/src/cua/translate.rs

@@ -8,7 +8,7 @@ use crate::{launch_argv_on, open_argv_on};
 pub enum TranslatedAction {
     Cua { tool: &'static str, payload: Value },
     Sleep { ms: u64 },
-    LegacyArgv { argv: Vec<String>, detached: bool },
+    LegacyArgv { argv: Vec<String> },
     FocusTitle { title: String },
 }
 
@@ -21,15 +21,11 @@ pub fn translate_action(
         ComputerAction::Wait { ms } => Ok(TranslatedAction::Sleep { ms: u64::from(*ms) }),
         ComputerAction::Open { path } => Ok(TranslatedAction::LegacyArgv {
             argv: open_argv_on(display, profile, path),
-            detached: true,
         }),
         ComputerAction::Launch { application, uri } => {
             let argv = launch_argv_on(display, profile, application, uri.as_deref())
                 .ok_or(ControlError::Unsupported)?;
-            Ok(TranslatedAction::LegacyArgv {
+            Ok(TranslatedAction::LegacyArgv { argv })
-                argv,
-                detached: true,
-            })
         }
         ComputerAction::Ref { .. } => Err(ControlError::Unsupported),
         ComputerAction::Focus { title } => Ok(TranslatedAction::FocusTitle {

crates/control/src/observe.rs

@@ -12,6 +12,7 @@ pub fn observation_from_png(
     active_window: Option<ActiveWindow>,
 ) -> ComputerObservation {
     ComputerObservation {
+        native_observation_complete: false,
         frame_id: hex::encode(Sha256::digest(&image)),
         captured_at: Utc::now().to_rfc3339(),
         mime_type: sniff_image_mime(&image).to_string(),
@@ -27,6 +28,7 @@ pub fn observation_from_png(
 pub fn observation_to_control_json(observation: &ComputerObservation) -> Value {
     let mut body = json!({
         "png_base64": base64::engine::general_purpose::STANDARD.encode(&observation.image),
+        "native_observation_complete": observation.native_observation_complete,
     });
     if let Some(cursor) = &observation.cursor {
         body["cursor"] = json!({ "x": cursor.x, "y": cursor.y });

crates/controld/src/main.rs

@@ -2,6 +2,7 @@ use std::sync::Arc;
 
 use axum::extract::State;
 use axum::http::{HeaderMap, StatusCode};
+use axum::response::{IntoResponse, Response};
 use axum::routing::{get, post};
 use axum::{Json, Router};
 use lazyboy_control::{
@@ -26,6 +27,7 @@ async fn main() {
     tracing::info!(backend = driver.as_str(), "computer controller");
     let app = Router::new()
         .route("/health", get(|| async { "ok" }))
+        .route("/controller/health", get(controller_health))
         .route("/observe", post(observe))
         .route("/act", post(act))
         .route("/browser", post(browser))
@@ -76,21 +78,64 @@ fn profile_of(headers: &HeaderMap, fallback: Option<&str>) -> Option<String> {
         .map(str::to_string)
 }
 
-fn status_for(error: &ControlError) -> StatusCode {
+struct ControlFailure(StatusCode, String);
-    if error.is_client_error() {
+
+impl IntoResponse for ControlFailure {
+    fn into_response(self) -> Response {
+        (
+            self.0,
+            Json(serde_json::json!({ "ok": false, "error": self.1 })),
+        )
+            .into_response()
+    }
+}
+
+impl From<StatusCode> for ControlFailure {
+    fn from(status: StatusCode) -> Self {
+        Self(status, status.to_string())
+    }
+}
+
+fn status_for(error: &ControlError) -> ControlFailure {
+    let status = if error.is_client_error() {
         StatusCode::BAD_REQUEST
+    } else if matches!(error, ControlError::Timeout) {
+        StatusCode::GATEWAY_TIMEOUT
     } else {
         tracing::error!(error = %error, "control failed");
         StatusCode::INTERNAL_SERVER_ERROR
+    };
+    ControlFailure(status, error.to_string())
 }
+
+async fn controller_health(
+    State(app): State<App>,
+    headers: HeaderMap,
+) -> Result<Json<serde_json::Value>, ControlFailure> {
+    if !authorized(&headers, &app.token) {
+        return Err(StatusCode::UNAUTHORIZED.into());
+    }
+    let ctx = ControlContext::new(display_of(&headers, None), None);
+    let health = app
+        .controller
+        .health(&ctx)
+        .await
+        .map_err(|error| status_for(&error))?;
+    Ok(Json(serde_json::json!({
+        "backend": health.backend,
+        "version": health.version,
+        "healthy": health.healthy,
+        "degraded": health.degraded,
+        "details": health.details,
+    })))
 }
 
 async fn observe(
     State(app): State<App>,
     headers: HeaderMap,
-) -> Result<Json<serde_json::Value>, StatusCode> {
+) -> Result<Json<serde_json::Value>, ControlFailure> {
     if !authorized(&headers, &app.token) {
-        return Err(StatusCode::UNAUTHORIZED);
+        return Err(StatusCode::UNAUTHORIZED.into());
     }
     let ctx = ControlContext::new(display_of(&headers, None), None);
     match app.controller.observe(&ctx).await {
@@ -103,9 +148,9 @@ async fn act(
     State(app): State<App>,
     headers: HeaderMap,
     Json(request): Json<ActionRequest>,
-) -> Result<Json<serde_json::Value>, StatusCode> {
+) -> Result<Json<serde_json::Value>, ControlFailure> {
     if !authorized(&headers, &app.token) {
-        return Err(StatusCode::UNAUTHORIZED);
+        return Err(StatusCode::UNAUTHORIZED.into());
     }
     let ctx = ControlContext::new(
         display_of(&headers, request.display.as_deref()),
@@ -129,9 +174,9 @@ async fn browser(
     State(app): State<App>,
     headers: HeaderMap,
     Json(request): Json<BrowserRequest>,
-) -> Result<Json<serde_json::Value>, StatusCode> {
+) -> Result<Json<serde_json::Value>, ControlFailure> {
     if !authorized(&headers, &app.token) {
-        return Err(StatusCode::UNAUTHORIZED);
+        return Err(StatusCode::UNAUTHORIZED.into());
     }
     let ctx = ControlContext::new(
         display_of(&headers, request.display.as_deref()),
@@ -156,9 +201,9 @@ async fn recording_start(
     State(app): State<App>,
     headers: HeaderMap,
     Json(request): Json<RecordingRequest>,
-) -> Result<Json<serde_json::Value>, StatusCode> {
+) -> Result<Json<serde_json::Value>, ControlFailure> {
     if !authorized(&headers, &app.token) {
-        return Err(StatusCode::UNAUTHORIZED);
+        return Err(StatusCode::UNAUTHORIZED.into());
     }
     match app
         .controller
@@ -176,9 +221,9 @@ async fn recording_stop(
     State(app): State<App>,
     headers: HeaderMap,
     Json(request): Json<RecordingRequest>,
-) -> Result<Json<serde_json::Value>, StatusCode> {
+) -> Result<Json<serde_json::Value>, ControlFailure> {
     if !authorized(&headers, &app.token) {
-        return Err(StatusCode::UNAUTHORIZED);
+        return Err(StatusCode::UNAUTHORIZED.into());
     }
     match app
         .controller
@@ -194,9 +239,9 @@ async fn recording_collect(
     State(app): State<App>,
     headers: HeaderMap,
     Json(request): Json<RecordingRequest>,
-) -> Result<Json<serde_json::Value>, StatusCode> {
+) -> Result<Json<serde_json::Value>, ControlFailure> {
     if !authorized(&headers, &app.token) {
-        return Err(StatusCode::UNAUTHORIZED);
+        return Err(StatusCode::UNAUTHORIZED.into());
     }
     match app
         .controller

crates/sandbox/src/docker.rs

@@ -546,8 +546,13 @@ fn decode_observation(body: &Value) -> Result<ComputerObservation, SandboxError>
         .get("elements")
         .map(|value| parse_ui_elements(&value.to_string()))
         .unwrap_or_default();
-    Ok(observation_with_elements(
+    let mut observation = observation_with_elements(
         observation_from_png(png, 1280, 800, cursor, window),
         elements,
-    ))
+    );
+    observation.native_observation_complete = body
+        .get("native_observation_complete")
+        .and_then(Value::as_bool)
+        .unwrap_or(false);
+    Ok(observation)
 }

docker-compose.cua.yml

@@ -0,0 +1,9 @@
+# Opt in without changing the default legacy rollout or overwriting its image.
+# docker compose -f docker-compose.yml -f docker-compose.cua.yml up -d --build
+services:
+  computer:
+    image: lazyboy/computer:cua
+  supervisor:
+    environment:
+      LAZYBOY_COMPUTER_DRIVER: cua
+      LAZYBOY_COMPUTER_IMAGE: lazyboy/computer:cua

docker-compose.yml

@@ -51,7 +51,7 @@ services:
       LAZYBOY_COMPUTER_MEMORY_MB: ${LAZYBOY_COMPUTER_MEMORY_MB:-2048}
       LAZYBOY_COMPUTER_PIDS: ${LAZYBOY_COMPUTER_PIDS:-2048}
       LAZYBOY_COMPUTER_SUDO: ${LAZYBOY_COMPUTER_SUDO:-false}
-      LAZYBOY_COMPUTER_DRIVER: ${LAZYBOY_COMPUTER_DRIVER:-cua}
+      LAZYBOY_COMPUTER_DRIVER: ${LAZYBOY_COMPUTER_DRIVER:-legacy}
       LAZYBOY_LXCFS_ROOT: /var/lib/lxcfs
       SUPERVISOR_BIND: 0.0.0.0:7091
       DATA_DIR: /data

docs/cua-benchmark.md

@@ -0,0 +1,17 @@
+# Cua adapter timings (2026-09-07)
+
+Measured on Apple Silicon, `lazyboy/computer:local`, Cua Driver 0.23.2, inside `make cua-smoke` (`--repeat 10`). These are LazyBoy `controld` HTTP calls, not raw `cua-driver` CLI.
+
+| Call | n | median ms | max ms |
+| --- | ---: | ---: | ---: |
+| `GET /controller/health` | 1 | 11 | 11 |
+| `POST /observe` | 30 | 101 | 125 |
+| `POST /act` | 40 | 403 | 744 |
+| `POST /browser` snapshot | 10 | 48 | 51 |
+| `POST /browser` click | 20 | 332 | 337 |
+| `POST /browser` type | 10 | 391 | 399 |
+| `POST /browser` navigate | 10 | 927 | 941 |
+
+`/act` includes native click, batched setvalue+click, focus, and one Cua `drag`. Navigate includes the 800 ms settle in the adapter.
+
+There is no legacy control-plane comparison in this run. Do not treat these numbers as a ship gate against CDP/AT-SPI.

docs/cua-compatibility.md

@@ -4,16 +4,16 @@ This report answers one question, from a real `make cua-smoke` run on 2026-09-07
 
 > Can Cua Driver reliably control the existing LazyBoy XFCE + Xvfb desktop container?
 
-**Yes.** Five core actions succeeded 10 consecutive times. Production `computer_observe` / `computer_act` / `browser` still use the legacy controld path; this only proves Cua as a driver behind those abstractions.
+**Yes, as an opt-in backend.** A disposable `lazyboy/computer:local` desktop on 2026-09-07 (linux/arm64, Cua Driver 0.23.2) passed raw Driver smoke 10/10, the LazyBoy adapter 10/10, dual-display isolation, and Chromium cookie persistence across `docker pause` and `docker restart`. Production still defaults to `legacy`. See [cua-review.md](cua-review.md) and [cua-benchmark.md](cua-benchmark.md).
 
 ## Environment
 
 - Image: `lazyboy/computer:local` (`image/computer/Dockerfile`)
-- Distro: Debian bookworm, x86_64
+- Distro: Debian bookworm; Driver binary follows `TARGETARCH` (`linux-arm64` or `linux-x86_64`)
 - Display: Xvfb `DISPLAY=:1` at 1280×800, XFCE (`xfwm4` compositor off, `xfce4-panel`, `xfdesktop`)
 - Accessibility: AT-SPI 2 per screen (`at-spi-bus-launcher` + `at-spi2-registryd`)
 - Browser: Debian `chromium` via `lazyboy-browser` (persistent profile, `--remote-debugging-port=9221+display`, `--force-renderer-accessibility`, `--lang=zh-TW`)
-- Cua Driver: **0.23.2** (`cua-driver-rs-v0.23.2` linux-x86_64-binary, SHA256 `01bf8339ec129cc00f4b4b2c6056ef1a7c5b52df39ff83ad17c9b16818aec500`)
+- Cua Driver: **0.23.2** (`cua-driver-rs-v0.23.2`; linux-arm64 SHA256 `be22768a207796a4bc1de50c52f32f9ef680b5e86e58c059e02eec2caba2e7bb`, linux-x86_64 SHA256 `01bf8339ec129cc00f4b4b2c6056ef1a7c5b52df39ff83ad17c9b16818aec500`)
 - Install path: `/usr/local/lib/cua-driver` + `/usr/local/bin/cua-driver` (not under the persisted `/home/lazyboy` bind)
 - Daemon: `cua-driver serve --grant existing-profile --socket /tmp/lazyboy/cua.sock --no-overlay` on the primary display only
 - Telemetry: disabled
@@ -82,6 +82,15 @@ Not used here: recording, isolated `launch_app` browsers, Wayland helpers. These
 
 Cua Driver 0.23.2 **can** control the existing LazyBoy XFCE + Xvfb container: screenshot, window/AT-SPI observation, native click/type, and Chromium semantic click/type, 10/10, without replacing the browser profile or breaking noVNC.
 
-`ComputerController` is in place. Production now defaults to `cua`. Set `LAZYBOY_COMPUTER_DRIVER=legacy` on the supervisor (passed into each desktop container) to roll back to CDP/AT-SPI/xdotool. Recreate desktop containers after changing the flag.
+`ComputerController` is in place. Production defaults to `legacy` pending the full acceptance suite and benchmark. Set `LAZYBOY_COMPUTER_DRIVER=cua` only for explicit testing. Set `LAZYBOY_COMPUTER_DRIVER=legacy` on the supervisor (passed into each desktop container) to roll back to CDP/AT-SPI/xdotool. Recreate desktop containers after changing the flag.
 
 With `cua`: `POST /observe`, `POST /act`, and `POST /browser` go through Cua Driver. The Agent-facing `browser` schema is unchanged (`snapshot` / `click` / `type` / `press` / `navigate` / `wait`); Cua attaches with `existing_profile` and maps `semantic_v2` refs (`pN:M`) onto the existing element list. After human takeover ends, the run forces a fresh `computer_observe` and drops pre-handoff ids/refs. Skill teaching starts Cua `start_recording` (no video) plus the existing CDP DOM recorder so a human noVNC demo still yields semantic click/type/navigate events; Cua trajectory turns are ingested as extra evidence and password-labelled typing is masked. `use_saved_login` still fills via CDP stdin so passwords never appear on argv. `cdp.py` / AT-SPI remain for login fill, human browser recording, and the `legacy` rollback.
+
+
+## Review of the current checkout (2026-09-07)
+
+The locally tagged `lazyboy/computer:local` image now installs Cua Driver 0.23.2
+for the build architecture (`cua-driver 0.23.2` on linux/arm64 in this run).
+`make cua-smoke` is the acceptance entry: raw Driver smoke, adapter E2E,
+isolation, and pause/restart persistence. Production defaults remain `legacy`.
+See [the migration audit](cua-review.md).

docs/cua-review.md

@@ -0,0 +1,53 @@
+# Cua 遷移檢查（2026-09-07）
+
+結論：`a.md` Phase 1 規格尚未全部勾完，但 **opt-in Cua 已可在現有 XFCE + Xvfb 桌面容器使用**。生產預設仍是 `legacy`。
+
+本次在 Apple Silicon（linux/arm64）上以 `lazyboy/computer:local` + Cua Driver **0.23.2** 重跑隔離桌面驗收。
+
+## 現況
+
+| 規格 | 狀態 |
+| --- | --- |
+| 雙後端、Agent schema 不變 | 完成。`LAZYBOY_COMPUTER_DRIVER=legacy`（預設）或 `cua`。 |
+| Docker 安裝 | 完成。Dockerfile 依 `TARGETARCH` 安裝 linux-arm64 / linux-x86_64，checksum 固定。 |
+| `computer_observe` | 完成。截圖 + native AT-SPI 元素（`kind=a11y`、snapshot-scoped `cua:…` handle）+ 視窗列表。 |
+| `computer_act` | 完成。pointer / type / key / scroll / focus / 單次 `drag`；native ref 走 Cua `click` / `set_value`。不支援的動作明確 `Unsupported`，不再偷偷回退 legacy。 |
+| Browser | 完成。`existing_profile` attach、`semantic_v2` refs、navigate 限 http/https/about。 |
+| 多螢幕隔離 | 完成。display `:1` 的 handle 送到 `:2` 會被拒絕；各自點擊只改自己的測試程式。 |
+| pause / restart 後 Chromium cookie | 完成。adapter `--check-persistence` 在 `docker pause` 與 `docker restart` 後都通過。 |
+| noVNC | smoke 確認 `:6080` 仍可連；完整 human takeover 端到端未另開測試。 |
+| 示範錄製 | Cua `start_recording` + 既有 CDP recorder 仍在；未做真人 noVNC 示範驗收。 |
+| Benchmark 文件 | 見 [cua-benchmark.md](cua-benchmark.md)。尚未對 legacy 做對照。 |
+| 預設切到 Cua | **未做。** 完整 DoD（takeover、錄製、生產 metrics）未過前維持 legacy。 |
+
+## 這輪修正
+
+- Native 同一批 `computer_act` 共用一份觀察快照；先前每個動作都把 handle map 拿掉，導致 `wait` 後面的 ref 或連續兩個 ref 被當成過期。
+- 拒絕的過期 ref 不再清掉該螢幕上其他仍有效的 handle。
+- JSON `code != ok` 即使行程成功碼為 0 也當失敗（避免 drag 誤報成功）。
+- CLI JSON 改走 stdin，避免輸入文字出現在 argv。
+- 拖曳改打最小包含該點的視窗（避免點到覆蓋其上的 Chromium），focus 優先精確標題（避免 `LazyBoy Cua Smoke - Chromium` 搶走 GTK 視窗）。
+- GTK 測資把 drawing area 座標轉成螢幕座標。
+- Cua 選到但 binary 不在或 daemon 起不來會明確失敗。
+
+## 驗證（本機 2026-09-07）
+
+- `cargo test --locked -p lazyboy-control`：84 通過。
+- `cargo check --locked -p lazyboy-api -p lazyboy-sandbox -p lazyboy-controld`：通過。
+- `make cua-smoke`（`--repeat 10`）：
+  - 原始 Driver smoke **10/10**（截圖、視窗、native click/type、Chromium attach、noVNC）
+  - LazyBoy adapter **10/10**（GTK click / 中文 setvalue、過期 handle 拒絕、批次 native、drag、Chromium 表單）
+  - 雙 display 隔離通過
+  - pause/unpause 與 `docker restart` 後 cookie 仍在
+
+## 如何啟用
+
+預設不要改。要在本機明確跑 Cua：
+
+```bash
+make cua-smoke
+# 或
+docker compose -f docker-compose.yml -f docker-compose.cua.yml up -d --build
+```
+
+overlay 把 supervisor 的 `LAZYBOY_COMPUTER_DRIVER` 設成 `cua`，桌面映像標成 `lazyboy/computer:cua`，不會覆寫預設的 `lazyboy/computer:local` legacy 映像。改 flag 後必須重建桌面容器。

docs/cua-schemas/0.23.2/schema-bring_to_front.txt

@@ -0,0 +1,22 @@
+name: bring_to_front
+
+description:
+Persistently activate a window so subsequent input lands on it. This deliberately breaks the no-foreground contract and is not part of the normal input ladder. For an ordinary `background_unavailable` response, retry only the refused action with `delivery_mode:"foreground"`; the input tool performs its own activate, act, and restore sequence. Use `bring_to_front` only for a focus-proxy surface that must remain foreground across multiple calls, such as a remote desktop session, or when repeated action-scoped activation prevents the remote surface from accepting input. X11: EWMH _NET_ACTIVE_WINDOW activation (the `wmctrl -a` equivalent, with proper timestamp handling to beat focus-stealing prevention). Wayland: activates through a target-addressable compositor adapter (wlroots foreign-toplevel or the GNOME Shell helper) and refuses when the compositor offers no safe adapter. Matches the macOS / Windows bring_to_front rung.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "pid": {
+      "type": "integer"
+    },
+    "window_id": {
+      "description": "X11 window id (xid) to activate. If omitted, the first window of `pid` is used.",
+      "type": "integer"
+    }
+  },
+  "required": [
+    "pid"
+  ],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-browser_click.txt

@@ -0,0 +1,48 @@
+name: browser_click
+
+description:
+Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture. input_route="dom_event" (synthetic el.click(), ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. Refused for heuristic bindings.
+
+input_schema:
+{
+  "additionalProperties": true,
+  "properties": {
+    "input_route": {
+      "description": "\"trusted\" (default): Input.dispatchMouseEvent. It refuses rather than foregrounding a standalone browser. \"dom_event\": synthetic full-background DOM click, only when explicitly requested. Dispatch does not prove the control activated; refresh page state and verify the expected postcondition.",
+      "enum": [
+        "trusted",
+        "dom_event"
+      ],
+      "type": "string"
+    },
+    "ref": {
+      "description": "Page element ref in the p<snapshot>:<index> namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.",
+      "type": "string"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.",
+      "type": "string"
+    },
+    "tab_id": {
+      "description": "Opaque tab id from get_browser_state (session-scoped).",
+      "type": "string"
+    },
+    "target_id": {
+      "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).",
+      "type": "string"
+    },
+    "x": {
+      "description": "Viewport x (CSS px) — alternative to ref.",
+      "type": "number"
+    },
+    "y": {
+      "description": "Viewport y (CSS px) — alternative to ref.",
+      "type": "number"
+    }
+  },
+  "required": [
+    "target_id",
+    "tab_id"
+  ],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-browser_navigate.txt

@@ -0,0 +1,33 @@
+name: browser_navigate
+
+description:
+Navigate one tab of an exactly-bound browser target to a new URL (http/https/about only). Refused for heuristic bindings. Navigation invalidates all p<snapshot>:<index> refs for the tab.
+
+input_schema:
+{
+  "additionalProperties": true,
+  "properties": {
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.",
+      "type": "string"
+    },
+    "tab_id": {
+      "description": "Opaque tab id from get_browser_state (session-scoped).",
+      "type": "string"
+    },
+    "target_id": {
+      "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).",
+      "type": "string"
+    },
+    "url": {
+      "description": "Destination URL (http:, https:, or about:).",
+      "type": "string"
+    }
+  },
+  "required": [
+    "target_id",
+    "tab_id",
+    "url"
+  ],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-browser_prepare.txt

@@ -0,0 +1,95 @@
+name: browser_prepare
+
+description:
+Explicitly prepare an owned DevTools endpoint for a browser. pid is required for an existing process or existing-profile attachment, and optional only for allow_launch=true with an isolated profile. Existing endpoints are detected without side effects. Acting setup for an isolated profile follows the runtime permission mode and optional capability manifest. It requires allow_launch=true, launches a separate browser, and never copies, modifies, or terminates the requested user profile. Without pid, only a platform-attested system Chrome/Edge installation (or a root-owned package payload on Linux) is eligible; redirects and user-controlled locations fail closed. Existing-profile attachment is explicit and follows the runtime's immutable permission mode: standard requires an explicit --grant existing-profile launch grant or an embedding authorization host, bounded requires a launch-approved exact resource manifest, and unrestricted requires explicit trusted startup risk acceptance. Ordinary MCP transport approval never proves profile authorization. On proven platforms, an authorized request also permits one bounded exact-window setup: open the recognized browser product's fixed remote-debugging page, toggle its uniquely matched per-instance checkbox, prove the PID-owned loopback endpoint, and close the temporary tab. Every visible effect is reported; ambiguity is refused.
+
+input_schema:
+{
+  "additionalProperties": true,
+  "anyOf": [
+    {
+      "required": [
+        "pid"
+      ]
+    },
+    {
+      "properties": {
+        "allow_launch": {
+          "const": true
+        },
+        "profile": {
+          "properties": {
+            "mode": {
+              "enum": [
+                "isolated_new",
+                "isolated_named"
+              ]
+            }
+          },
+          "required": [
+            "mode"
+          ]
+        }
+      },
+      "required": [
+        "allow_launch",
+        "profile"
+      ]
+    }
+  ],
+  "properties": {
+    "allow_launch": {
+      "description": "Allow a separate driver-owned isolated Chromium process to be launched (default false).",
+      "type": "boolean"
+    },
+    "pid": {
+      "description": "Browser process id to prepare. Required except for a driver-owned isolated_new/isolated_named launch with allow_launch=true.",
+      "type": "integer"
+    },
+    "profile": {
+      "additionalProperties": false,
+      "properties": {
+        "mode": {
+          "enum": [
+            "isolated_new",
+            "isolated_named"
+          ],
+          "type": "string"
+        },
+        "name": {
+          "description": "Required only for isolated_named; 1-64 path-safe ASCII characters.",
+          "type": "string"
+        }
+      },
+      "required": [
+        "mode"
+      ],
+      "type": "object"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.",
+      "type": "string"
+    },
+    "strategy": {
+      "additionalProperties": false,
+      "properties": {
+        "kind": {
+          "enum": [
+            "existing_profile"
+          ],
+          "type": "string"
+        }
+      },
+      "required": [
+        "kind"
+      ],
+      "type": "object"
+    },
+    "window_id": {
+      "description": "Exact native window approval anchor; required for strategy.kind=existing_profile.",
+      "type": "integer"
+    }
+  },
+  "required": [],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-browser_type.txt

@@ -0,0 +1,50 @@
+name: browser_type
+
+description:
+Type text into an exactly-bound tab via the Input domain. mode="insert_text" (default) uses Input.insertText; mode="keystrokes" dispatches per-character key events. Both insert at the caret, so typing into a field that already holds text appends to it; pass replace=true to set the field instead, or to clear it by typing an empty string. Pass a ref to an editable element from the latest snapshot. A ref is required; heuristic bindings are refused.
+
+input_schema:
+{
+  "additionalProperties": true,
+  "properties": {
+    "mode": {
+      "description": "insert_text (default): bulk Input.insertText. keystrokes: per-character Input.dispatchKeyEvent.",
+      "enum": [
+        "insert_text",
+        "keystrokes"
+      ],
+      "type": "string"
+    },
+    "ref": {
+      "description": "Page element ref in the p<snapshot>:<index> namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.",
+      "type": "string"
+    },
+    "replace": {
+      "description": "false (default): insert at the caret, appending to whatever the field already holds. true: select the element's whole content first so the text replaces it — with an empty text this clears the field. Replacement goes through the selection, so beforeinput/input still fire and framework state stays consistent.",
+      "type": "boolean"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.",
+      "type": "string"
+    },
+    "tab_id": {
+      "description": "Opaque tab id from get_browser_state (session-scoped).",
+      "type": "string"
+    },
+    "target_id": {
+      "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).",
+      "type": "string"
+    },
+    "text": {
+      "description": "Text to type.",
+      "type": "string"
+    }
+  },
+  "required": [
+    "target_id",
+    "tab_id",
+    "ref",
+    "text"
+  ],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-click.txt

@@ -0,0 +1,146 @@
+name: click
+
+description:
+Click against a target pid. **Prefer `element_index` over pixel coordinates** — element_index works on backgrounded / hidden windows, surfaces a stable handle, and tells you what you're clicking via the cached AT-SPI element's role + label. Reach for `x, y` only when the target is a canvas / custom-drawn surface that doesn't appear in the AT-SPI tree.
+
+Provide either (window_id + x/y) or (pid + element_index). Routes via XSendEvent (no focus steal). element_index cache is scoped per (pid, window_id) and is replaced by the next get_window_state of the same window — re-snapshot every turn before clicking.
+
+After a zoom call, pass from_zoom=true to auto-translate zoom-image coords back to full-window space.
+
+button: "left" (default), "right", or "middle". Defaults to left so the field is fully back-compat. X11: routes through XSendEvent ButtonPress/Release with the matching button code. Native Wayland: only left-button is supported via the virtual-pointer protocol — right/middle return an error rather than silently degrading to left. `modifier` holds ctrl/shift/alt/super for the click on X11. Native Wayland refuses modified pointer clicks until its input protocol can carry keyboard modifier state.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "button": {
+      "description": "Mouse button. Default: \"left\" (legacy back-compat). X11: routed via ButtonPress/Release with the matching evdev code. Native Wayland: only left-button is supported via the virtual-pointer protocol; right/middle return an error.",
+      "enum": [
+        "left",
+        "right",
+        "middle"
+      ],
+      "type": "string"
+    },
+    "count": {
+      "type": "integer"
+    },
+    "cursor_id": {
+      "description": "Optional multi-cursor instance id. Default: 'default'.",
+      "type": "string"
+    },
+    "delivery_mode": {
+      "default": "background",
+      "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.",
+      "enum": [
+        "background",
+        "foreground"
+      ],
+      "type": "string"
+    },
+    "element_index": {
+      "description": "Element index from get_window_state. Requires the matching `snapshot_id` alongside it. Prefer `element_token`, which carries both values.",
+      "type": "integer"
+    },
+    "element_token": {
+      "description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. If element_index, snapshot_id, or window_id are also supplied they must agree. Returns an explicit stale error once a newer snapshot supersedes it.",
+      "type": "string"
+    },
+    "from_zoom": {
+      "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.",
+      "type": "boolean"
+    },
+    "modifier": {
+      "description": "Modifier keys held during the action: cmd, shift, option/alt, ctrl.",
+      "items": {
+        "type": "string"
+      },
+      "type": "array"
+    },
+    "pid": {
+      "type": "integer"
+    },
+    "scope": {
+      "default": "window",
+      "enum": [
+        "window",
+        "desktop"
+      ],
+      "type": "string"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.",
+      "type": "string"
+    },
+    "snapshot_id": {
+      "description": "Snapshot handle from get_window_state. Required when targeting by element_index; stale snapshots fail closed.",
+      "pattern": "^s[0-9a-f]{8}$",
+      "type": "string"
+    },
+    "target": {
+      "anyOf": [
+        {
+          "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.",
+          "oneOf": [
+            {
+              "additionalProperties": true,
+              "properties": {
+                "kind": {
+                  "const": "window",
+                  "type": "string"
+                },
+                "pid": {
+                  "format": "uint32",
+                  "minimum": 0,
+                  "type": "integer"
+                },
+                "window_id": {
+                  "format": "uint64",
+                  "minimum": 0,
+                  "type": "integer"
+                }
+              },
+              "required": [
+                "kind",
+                "pid",
+                "window_id"
+              ],
+              "type": "object"
+            },
+            {
+              "additionalProperties": true,
+              "properties": {
+                "display_id": {
+                  "type": "string"
+                },
+                "kind": {
+                  "const": "desktop",
+                  "type": "string"
+                }
+              },
+              "required": [
+                "kind",
+                "display_id"
+              ],
+              "type": "object"
+            }
+          ]
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "window_id": {
+      "type": "integer"
+    },
+    "x": {
+      "type": "number"
+    },
+    "y": {
+      "type": "number"
+    }
+  },
+  "required": [],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-get_browser_state.txt

@@ -0,0 +1,57 @@
+name: get_browser_state
+
+description:
+Read-only browser inspection. Mode 1 (bind): pass pid + window_id of a native browser window to classify it, correlate it to a CDP target (exact-or-refuse), and mint a session-scoped target id plus tab ids. Mode 2 (snapshot): pass target_id + tab_id. The dom_refs_v1 compatibility format returns composed DOM refs. semantic_v2 joins accessibility, DOM, layout, and viewport state; ranks visible content before retained/offscreen state; and returns a semantic outline, typed action refs, content refs, scoped reads, and opaque continuation. Never performs setup — a missing endpoint is a structured browser_requires_setup refusal pointing at browser_prepare.
+
+input_schema:
+{
+  "additionalProperties": true,
+  "properties": {
+    "continuation": {
+      "description": "Opaque continuation minted by an earlier semantic_v2 response.",
+      "type": "string"
+    },
+    "include_screenshot": {
+      "default": false,
+      "description": "Capture the exact tab viewport as PNG through CDP without selecting the tab or foregrounding its native window. The request refuses if capture cannot be completed.",
+      "type": "boolean"
+    },
+    "pid": {
+      "description": "Native browser process id (bind mode).",
+      "type": "integer"
+    },
+    "query": {
+      "description": "Read-only semantic match over role, accessible name, and visible text.",
+      "type": "string"
+    },
+    "scope_ref": {
+      "description": "Current semantic/content ref whose subtree should be observed.",
+      "type": "string"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.",
+      "type": "string"
+    },
+    "snapshot_format": {
+      "description": "Versioned snapshot contract. dom_refs_v1 remains the compatibility default.",
+      "enum": [
+        "dom_refs_v1",
+        "semantic_v2"
+      ],
+      "type": "string"
+    },
+    "tab_id": {
+      "description": "Opaque tab id from get_browser_state (session-scoped).",
+      "type": "string"
+    },
+    "target_id": {
+      "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).",
+      "type": "string"
+    },
+    "window_id": {
+      "description": "Native window id owned by pid (bind mode).",
+      "type": "integer"
+    }
+  },
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-get_cursor_position.txt

@@ -0,0 +1,16 @@
+name: get_cursor_position
+
+description:
+Return the current mouse cursor position in screen points (origin top-left).
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-get_desktop_state.txt

@@ -0,0 +1,20 @@
+name: get_desktop_state
+
+description:
+Capture the full display in the desktop action coordinate frame. Use the returned PNG directly as the coordinate source for actions whose target is {kind:"desktop",display_id:"primary"}. No AT-SPI walk.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "screenshot_out_file": {
+      "description": "Write PNG here instead of base64.",
+      "type": "string"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-get_window_state.txt

@@ -0,0 +1,63 @@
+name: get_window_state
+
+description:
+Walk a running app's AT-SPI tree and return BOTH a structured `elements` array (preferred) AND a Markdown rendering of the same tree (back-compat). Every actionable element is tagged with [element_index N] in the markdown and as `element_index` in the structured array.
+
+PREFERRED CONSUMERS read `structuredContent.elements` (one entry per indexed row with `element_index`, `role`, `label`, `value`, `enabled`, `selected`, `frame: {x,y,w,h}` when AT-SPI reports usable bounds, `parent_index`, `depth`). The markdown `tree_markdown` stays available and unchanged in shape for existing text-parsing callers — but new fields will only be added to the structured side. Set `query` to project BOTH representations to matching rows plus their ancestor chain while preserving original indices. `total_element_count` reports the complete snapshot and `returned_element_count` reports the projection.
+
+Always returns BOTH the element tree AND a screenshot — ground on both and cross-check (the tree lies on some surfaces). Choose the modality at ACTION time: an element ax action (element_index/element_token → accessibility rung) or an element px action (x,y → pixel rung off this screenshot). capture_mode is deprecated and ignored. On Wayland, where output capture cannot prove the requested surface's identity, the truthful tree is returned without a screenshot and `screenshot_error.code` is `surface_identity_unproven`.
+
+Optional `max_elements` / `max_depth` bound the AT-SPI walk to mitigate context-window blow-up on Electron / large web apps that produce 10k+ element trees. When applied, BOTH the markdown and the structured elements are truncated identically. Omit both for current default behaviour.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "capture_mode": {
+      "description": "DEPRECATED and ignored. get_window_state always returns BOTH the element tree and a screenshot — ground on both. The modality is chosen at action time by how you address the target: an element ax action (element_index/element_token) or an element px action (x,y). Any value (including the old \"som\"/\"screenshot\" aliases) is accepted but has no effect.",
+      "enum": [
+        "ax",
+        "vision"
+      ],
+      "type": "string"
+    },
+    "include_screenshot": {
+      "description": "Default true — returns a grounding screenshot alongside the tree. Set false to skip the grab and return tree only (the cheap path for re-indexing before an element ax action).",
+      "type": "boolean"
+    },
+    "max_depth": {
+      "description": "Cap on the AT-SPI tree walk depth. Omit for the default (uncapped). Lower for deeply nested apps.",
+      "minimum": 1,
+      "type": "integer"
+    },
+    "max_elements": {
+      "description": "Cap on total AT-SPI nodes walked. Omit for the default (5 000). Lower for huge web/Electron trees.",
+      "minimum": 1,
+      "type": "integer"
+    },
+    "pid": {
+      "type": "integer"
+    },
+    "query": {
+      "description": "Optional case-insensitive substring. Projects both tree_markdown and structured elements to matches plus ancestors while preserving original indices. Compare total_element_count with returned_element_count.",
+      "type": "string"
+    },
+    "screenshot_out_file": {
+      "description": "When set, write the PNG to this file path (~ expanded) instead of embedding base64 in the response. The structured output carries screenshot_file_path instead.",
+      "type": "string"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.",
+      "type": "string"
+    },
+    "window_id": {
+      "description": "Native window identifier from list_windows.",
+      "type": "integer"
+    }
+  },
+  "required": [
+    "pid",
+    "window_id"
+  ],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-health_report.txt

@@ -0,0 +1,67 @@
+name: health_report
+
+description:
+Single-call end-to-end driver diagnostics. Designed to let downstream consumers ship one stable call instead of stitching together check_permissions, doctor, version, bundle attribution, and platform capability status. On macOS, prompt-capable direct capture is deliberately skipped; use `cua-driver permissions grant` to verify it explicitly. cua-driver owns the health model; consumers stay thin.
+
+Input — all optional:
+  {
+    "include": ["<check_name>", ...],   // run only these
+    "skip":    ["<check_name>", ...]    // skip these
+  }
+If both are given, `include` wins.
+
+Canonical check names:
+  macOS  : binary_version, platform_supported, session_active,
+           bundle_identity, tcc_accessibility, tcc_screen_recording,
+           ax_capability, screen_capture_capability
+  Windows: binary_version, platform_supported, session_active,
+           ax_capability (via UIA), screen_capture_capability (via DXGI)
+  Linux  : binary_version, platform_supported, session_active,
+           ax_capability (via AT-SPI), screen_capture_capability (via X11)
+
+Output — stable contract, schema_version="1":
+  {
+    "schema_version": "1",
+    "platform": "darwin" | "win32" | "linux",
+    "driver_version": "<semver>",
+    "overall": "ok" | "degraded" | "failed",
+    "checks": [
+      {
+        "name": "<one of the canonical names above>",
+        "status": "pass" | "fail" | "skip",
+        "message": "<one-line summary, always present>",
+        "hint": "<remediation step, present when status=fail>",
+        "data": { /* check-specific structured fields */ }
+      },
+      ...
+    ]
+  }
+
+`overall` rules:
+  - `ok`       — every non-skipped check passes
+  - `degraded` — at least one non-core check fails (binary is still usable)
+  - `failed`   — any core check fails (binary_version, platform_supported, session_active)
+
+Stability: schema_version="1" is the contract. Future breaking changes will be `"2"`. Adding new check names under the same schema_version is non-breaking; consumers must tolerate unknown check names.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "include": {
+      "description": "Only run these checks (canonical names). Wins over `skip`.",
+      "items": {
+        "type": "string"
+      },
+      "type": "array"
+    },
+    "skip": {
+      "description": "Skip these checks (canonical names). Ignored when `include` is set.",
+      "items": {
+        "type": "string"
+      },
+      "type": "array"
+    }
+  },
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-hotkey.txt

@@ -0,0 +1,125 @@
+name: hotkey
+
+description:
+Press a combination of keys simultaneously, e.g. ["ctrl","c"] for Copy. Sent via XSendEvent directly to the target pid; target does NOT need to be frontmost.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "delivery_mode": {
+      "default": "background",
+      "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.",
+      "enum": [
+        "background",
+        "foreground"
+      ],
+      "type": "string"
+    },
+    "element_index": {
+      "description": "Element index from get_window_state. Requires the matching `snapshot_id` alongside it. Prefer `element_token`, which carries both values.",
+      "type": "integer"
+    },
+    "element_token": {
+      "description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. If element_index, snapshot_id, or window_id are also supplied they must agree. Returns an explicit stale error once a newer snapshot supersedes it.",
+      "type": "string"
+    },
+    "keys": {
+      "description": "Modifier(s) + one non-modifier key, e.g. [\"ctrl\",\"c\"].",
+      "items": {
+        "type": "string"
+      },
+      "minItems": 2,
+      "type": "array"
+    },
+    "pid": {
+      "type": "integer"
+    },
+    "scope": {
+      "default": "window",
+      "enum": [
+        "window",
+        "desktop"
+      ],
+      "type": "string"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.",
+      "type": "string"
+    },
+    "snapshot_id": {
+      "description": "Snapshot handle from get_window_state. Required when targeting by element_index; stale snapshots fail closed.",
+      "pattern": "^s[0-9a-f]{8}$",
+      "type": "string"
+    },
+    "target": {
+      "anyOf": [
+        {
+          "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.",
+          "oneOf": [
+            {
+              "additionalProperties": true,
+              "properties": {
+                "kind": {
+                  "const": "window",
+                  "type": "string"
+                },
+                "pid": {
+                  "format": "uint32",
+                  "minimum": 0,
+                  "type": "integer"
+                },
+                "window_id": {
+                  "format": "uint64",
+                  "minimum": 0,
+                  "type": "integer"
+                }
+              },
+              "required": [
+                "kind",
+                "pid",
+                "window_id"
+              ],
+              "type": "object"
+            },
+            {
+              "additionalProperties": true,
+              "properties": {
+                "display_id": {
+                  "type": "string"
+                },
+                "kind": {
+                  "const": "desktop",
+                  "type": "string"
+                }
+              },
+              "required": [
+                "kind",
+                "display_id"
+              ],
+              "type": "object"
+            }
+          ]
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "window_id": {
+      "type": "integer"
+    },
+    "x": {
+      "description": "Screenshot-pixel X — the element px action form: pixel-click there to focus, then send the combo (so e.g. Ctrl+V pastes into that field). Pass with y. Use for Chromium/Electron surfaces the background combo can't reach.",
+      "type": "number"
+    },
+    "y": {
+      "description": "Screenshot-pixel Y (see x).",
+      "type": "number"
+    }
+  },
+  "required": [
+    "keys"
+  ],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-list_windows.txt

@@ -0,0 +1,19 @@
+name: list_windows
+
+description:
+List top-level windows. Each record includes z_index (integer or null; higher values are closer to the front; null means stacking order is unavailable and callers must not infer one). To select a frontmost candidate, take the maximum integer z_index; if every value is null, use an explicit fallback instead of relying on array order.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "on_screen_only": {
+      "description": "When true, filter to visible windows only. Default false.",
+      "type": "boolean"
+    },
+    "pid": {
+      "type": "integer"
+    }
+  },
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-mouse_button_down.txt

@@ -0,0 +1,51 @@
+name: mouse_button_down
+
+description:
+Press and hold a mouse button at (x,y) via background X11 delivery. Does not release the button; pair with mouse_drag / mouse_button_up. Returns the current held-button state.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "button": {
+      "description": "Mouse button. Default \"left\".",
+      "enum": [
+        "left",
+        "right",
+        "middle"
+      ],
+      "type": "string"
+    },
+    "cursor_id": {
+      "description": "Optional multi-cursor instance id. Default: 'default'.",
+      "type": "string"
+    },
+    "from_zoom": {
+      "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.",
+      "type": "boolean"
+    },
+    "pid": {
+      "type": "integer"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. When both are present, session takes precedence over cursor_id.",
+      "type": "string"
+    },
+    "window_id": {
+      "type": "integer"
+    },
+    "x": {
+      "type": "number"
+    },
+    "y": {
+      "type": "number"
+    }
+  },
+  "required": [
+    "pid",
+    "window_id",
+    "x",
+    "y"
+  ],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-mouse_button_up.txt

@@ -0,0 +1,36 @@
+name: mouse_button_up
+
+description:
+Release a previously-held mouse button via background X11 delivery. If x/y are omitted, releases at the last held position. Returns the current held-button state.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "cursor_id": {
+      "description": "Optional multi-cursor instance id. Default: 'default'.",
+      "type": "string"
+    },
+    "from_zoom": {
+      "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.",
+      "type": "boolean"
+    },
+    "pid": {
+      "type": "integer"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. When both are present, session takes precedence over cursor_id.",
+      "type": "string"
+    },
+    "window_id": {
+      "type": "integer"
+    },
+    "x": {
+      "type": "number"
+    },
+    "y": {
+      "type": "number"
+    }
+  },
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-mouse_drag.txt

@@ -0,0 +1,52 @@
+name: mouse_drag
+
+description:
+Move a previously-held mouse button to a new point via background X11 delivery. Requires an active mouse_button_down state; does not release the button. Returns the updated held-button state.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "cursor_id": {
+      "description": "Optional multi-cursor instance id. Default: 'default'.",
+      "type": "string"
+    },
+    "duration_ms": {
+      "description": "Total drag duration. Default: 500.",
+      "maximum": 10000,
+      "minimum": 0,
+      "type": "integer"
+    },
+    "from_zoom": {
+      "description": "Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space.",
+      "type": "boolean"
+    },
+    "pid": {
+      "type": "integer"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. When both are present, session takes precedence over cursor_id.",
+      "type": "string"
+    },
+    "steps": {
+      "description": "Intermediate MotionNotify events. Default: 20.",
+      "maximum": 200,
+      "minimum": 1,
+      "type": "integer"
+    },
+    "window_id": {
+      "type": "integer"
+    },
+    "x": {
+      "type": "number"
+    },
+    "y": {
+      "type": "number"
+    }
+  },
+  "required": [
+    "x",
+    "y"
+  ],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-move_cursor.txt

@@ -0,0 +1,92 @@
+name: move_cursor
+
+description:
+Move the synthetic agent cursor without changing the user's pointer. Only an explicit scope=desktop request moves the real OS pointer in get_desktop_state coordinates.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "cursor_id": {
+      "type": "string"
+    },
+    "scope": {
+      "default": "window",
+      "enum": [
+        "window",
+        "desktop"
+      ],
+      "type": "string"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.",
+      "type": "string"
+    },
+    "target": {
+      "anyOf": [
+        {
+          "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.",
+          "oneOf": [
+            {
+              "additionalProperties": true,
+              "properties": {
+                "kind": {
+                  "const": "window",
+                  "type": "string"
+                },
+                "pid": {
+                  "format": "uint32",
+                  "minimum": 0,
+                  "type": "integer"
+                },
+                "window_id": {
+                  "format": "uint64",
+                  "minimum": 0,
+                  "type": "integer"
+                }
+              },
+              "required": [
+                "kind",
+                "pid",
+                "window_id"
+              ],
+              "type": "object"
+            },
+            {
+              "additionalProperties": true,
+              "properties": {
+                "display_id": {
+                  "type": "string"
+                },
+                "kind": {
+                  "const": "desktop",
+                  "type": "string"
+                }
+              },
+              "required": [
+                "kind",
+                "display_id"
+              ],
+              "type": "object"
+            }
+          ]
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "Preferred per-call target. New callers should set this field."
+    },
+    "x": {
+      "type": "number"
+    },
+    "y": {
+      "type": "number"
+    }
+  },
+  "required": [
+    "x",
+    "y"
+  ],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-press_key.txt

@@ -0,0 +1,126 @@
+name: press_key
+
+description:
+Press a key via XSendEvent to a window. No focus steal.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "delivery_mode": {
+      "default": "background",
+      "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.",
+      "enum": [
+        "background",
+        "foreground"
+      ],
+      "type": "string"
+    },
+    "element_index": {
+      "description": "Element index from get_window_state. Requires the matching `snapshot_id` alongside it. Prefer `element_token`, which carries both values.",
+      "type": "integer"
+    },
+    "element_token": {
+      "description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. If element_index, snapshot_id, or window_id are also supplied they must agree. Returns an explicit stale error once a newer snapshot supersedes it.",
+      "type": "string"
+    },
+    "key": {
+      "type": "string"
+    },
+    "modifiers": {
+      "items": {
+        "type": "string"
+      },
+      "type": "array"
+    },
+    "pid": {
+      "type": "integer"
+    },
+    "scope": {
+      "default": "window",
+      "enum": [
+        "window",
+        "desktop"
+      ],
+      "type": "string"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.",
+      "type": "string"
+    },
+    "snapshot_id": {
+      "description": "Snapshot handle from get_window_state. Required when targeting by element_index; stale snapshots fail closed.",
+      "pattern": "^s[0-9a-f]{8}$",
+      "type": "string"
+    },
+    "target": {
+      "anyOf": [
+        {
+          "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.",
+          "oneOf": [
+            {
+              "additionalProperties": true,
+              "properties": {
+                "kind": {
+                  "const": "window",
+                  "type": "string"
+                },
+                "pid": {
+                  "format": "uint32",
+                  "minimum": 0,
+                  "type": "integer"
+                },
+                "window_id": {
+                  "format": "uint64",
+                  "minimum": 0,
+                  "type": "integer"
+                }
+              },
+              "required": [
+                "kind",
+                "pid",
+                "window_id"
+              ],
+              "type": "object"
+            },
+            {
+              "additionalProperties": true,
+              "properties": {
+                "display_id": {
+                  "type": "string"
+                },
+                "kind": {
+                  "const": "desktop",
+                  "type": "string"
+                }
+              },
+              "required": [
+                "kind",
+                "display_id"
+              ],
+              "type": "object"
+            }
+          ]
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "window_id": {
+      "type": "integer"
+    },
+    "x": {
+      "description": "Screenshot-pixel X — the element px action form: pixel-click there to focus, then send the key. Use when the key must go to a Chromium/Electron surface the AX path can't focus. Pass with y, no element_index.",
+      "type": "number"
+    },
+    "y": {
+      "description": "Screenshot-pixel Y (see x).",
+      "type": "number"
+    }
+  },
+  "required": [
+    "key"
+  ],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-scroll.txt

@@ -0,0 +1,142 @@
+name: scroll
+
+description:
+Scroll the target pid's focused region via XSendEvent Button4/5. direction required; by defaults to line, amount defaults to 3.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "amount": {
+      "maximum": 50,
+      "minimum": 1,
+      "type": "integer"
+    },
+    "by": {
+      "enum": [
+        "line",
+        "page"
+      ],
+      "type": "string"
+    },
+    "cursor_id": {
+      "description": "Optional multi-cursor instance id. Default: 'default'.",
+      "type": "string"
+    },
+    "delivery_mode": {
+      "default": "background",
+      "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.",
+      "enum": [
+        "background",
+        "foreground"
+      ],
+      "type": "string"
+    },
+    "direction": {
+      "enum": [
+        "up",
+        "down",
+        "left",
+        "right"
+      ],
+      "type": "string"
+    },
+    "element_index": {
+      "description": "Element index from get_window_state. Requires the matching `snapshot_id` alongside it. Prefer `element_token`, which carries both values.",
+      "type": "integer"
+    },
+    "element_token": {
+      "description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. If element_index, snapshot_id, or window_id are also supplied they must agree. Returns an explicit stale error once a newer snapshot supersedes it.",
+      "type": "string"
+    },
+    "pid": {
+      "type": "integer"
+    },
+    "scope": {
+      "default": "window",
+      "enum": [
+        "window",
+        "desktop"
+      ],
+      "type": "string"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.",
+      "type": "string"
+    },
+    "snapshot_id": {
+      "description": "Snapshot handle from get_window_state. Required when targeting by element_index; stale snapshots fail closed.",
+      "pattern": "^s[0-9a-f]{8}$",
+      "type": "string"
+    },
+    "target": {
+      "anyOf": [
+        {
+          "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.",
+          "oneOf": [
+            {
+              "additionalProperties": true,
+              "properties": {
+                "kind": {
+                  "const": "window",
+                  "type": "string"
+                },
+                "pid": {
+                  "format": "uint32",
+                  "minimum": 0,
+                  "type": "integer"
+                },
+                "window_id": {
+                  "format": "uint64",
+                  "minimum": 0,
+                  "type": "integer"
+                }
+              },
+              "required": [
+                "kind",
+                "pid",
+                "window_id"
+              ],
+              "type": "object"
+            },
+            {
+              "additionalProperties": true,
+              "properties": {
+                "display_id": {
+                  "type": "string"
+                },
+                "kind": {
+                  "const": "desktop",
+                  "type": "string"
+                }
+              },
+              "required": [
+                "kind",
+                "display_id"
+              ],
+              "type": "object"
+            }
+          ]
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "window_id": {
+      "type": "integer"
+    },
+    "x": {
+      "description": "Window-local screenshot-pixel X of the scroll target. Pass with y and without element_index.",
+      "type": "number"
+    },
+    "y": {
+      "description": "Window-local screenshot-pixel Y of the scroll target. Pass with x and without element_index.",
+      "type": "number"
+    }
+  },
+  "required": [
+    "direction"
+  ],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-set_value.txt

@@ -0,0 +1,43 @@
+name: set_value
+
+description:
+Set value of an AT-SPI element via SetValue action.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "element_index": {
+      "description": "Element index from get_window_state. Requires the matching `snapshot_id` alongside it. Prefer `element_token`, which carries both values.",
+      "type": "integer"
+    },
+    "element_token": {
+      "description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. If element_index, snapshot_id, or window_id are also supplied they must agree. Returns an explicit stale error once a newer snapshot supersedes it.",
+      "type": "string"
+    },
+    "pid": {
+      "type": "integer"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.",
+      "type": "string"
+    },
+    "snapshot_id": {
+      "description": "Snapshot handle from get_window_state. Required when targeting by element_index; stale snapshots fail closed.",
+      "pattern": "^s[0-9a-f]{8}$",
+      "type": "string"
+    },
+    "value": {
+      "type": "string"
+    },
+    "window_id": {
+      "description": "Required when element_index is used; optional when element_token is supplied (the token carries it).",
+      "type": "integer"
+    }
+  },
+  "required": [
+    "pid",
+    "value"
+  ],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-start_recording.txt

@@ -0,0 +1,41 @@
+name: start_recording
+
+description:
+Start trajectory recording. Every subsequent action-tool invocation (click, right_click, scroll, type_text, press_key, hotkey, set_value) writes a turn folder under `output_dir`:
+
+- `before_state.json` / `after_state.json` — application AX/UIA/AT-SPI state immediately before and after the action.
+- `before.png` / `after.png` — target-window screenshots immediately before and after the action.
+- `evidence.json` — capture status and a stable classification when an expected artifact could not be captured.
+- `app_state.json` — post-action AX/UIA snapshot for the target pid.
+- `screenshot.png` — compatibility alias of `after.png`.
+- `action.json` — tool name, full input arguments, result summary, result-error flag, pid, click point (when applicable), ISO-8601 timestamp.
+- `click.png` — for dispatched click-family actions only, `before.png` with a red marker at the click point. A call refused before target resolution is explicitly not applicable instead.
+
+Turn folders are named `turn-00001/`, `turn-00002/`, etc.  Turn numbering restarts at 1 each time recording is (re-)started.
+
+**Video is off by default.** Pass `record_video: true` to also capture the main display to `<output_dir>/recording.mp4` (H.264 / 30 fps) for the lifetime of the session. The recording is torn down automatically when the MCP client disconnects.
+
+**macOS uses native ScreenCaptureKit** (daemon-owned SCStream + SCRecordingOutput) so video inherits the daemon's Screen Recording grant — no extra TCC prompt, no ffmpeg subprocess. Requires macOS 15.0+.
+
+**Windows + Linux use an ffmpeg subprocess** (`gdigrab` / `x11grab` + libx264). Requires ffmpeg on PATH (winget install Gyan.FFmpeg / apt install ffmpeg); when ffmpeg is missing or fails on startup the per-turn capture (screenshots + action.json) still runs and the session's `last_error` field carries the diagnostic.
+
+State persists for the life of the daemon; a restart resets to disabled with no on-disk state. Call `stop_recording` to disable + finalize the mp4.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "output_dir": {
+      "description": "Absolute or ~-rooted directory where turn folders and (when enabled) the video file are written.",
+      "type": "string"
+    },
+    "record_video": {
+      "description": "Capture the main display to <output_dir>/recording.mp4. Default: false. Set to true to also capture the main display to recording.mp4 (otherwise only the per-turn screenshots + JSON are recorded). On macOS this uses native ScreenCaptureKit (no extra TCC prompt, macOS 15.0+); on Windows + Linux it requires ffmpeg on PATH.",
+      "type": "boolean"
+    }
+  },
+  "required": [
+    "output_dir"
+  ],
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-stop_recording.txt

@@ -0,0 +1,13 @@
+name: stop_recording
+
+description:
+Stop trajectory recording. Disables further per-turn capture and, when video was enabled, gracefully terminates the ffmpeg subprocess so the mp4's moov atom is finalized (the file is playable). Calling stop on an already-stopped session is a no-op. The response carries `last_video_path` pointing at the finalized mp4 (when video was on).
+
+A manual `stop_recording` is **unconditional** — it stops whatever recording is active regardless of which session started it. Ownership-scoped teardown (so one client disconnecting can't stop a recording a later client started) is handled by the registry's `session_end` lifecycle hook, not by this tool.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {},
+  "type": "object"
+}

docs/cua-schemas/0.23.2/schema-type_text.txt

@@ -0,0 +1,120 @@
+name: type_text
+
+description:
+Type text to a window via XSendEvent (KeyPress/KeyRelease). No focus steal.
+
+input_schema:
+{
+  "additionalProperties": false,
+  "properties": {
+    "delivery_mode": {
+      "default": "background",
+      "description": "Input delivery mode. 'background' (default) never activates or raises the target window. On X11 it injects via XTEST / the XInput2 master pointer (no focus steal). On Wayland it goes through libei + xdg-desktop-portal, which injects to the compositor's input focus — Wayland's security model has no per-window background targeting, so a specific non-focused window cannot be aimed at; when no libei backend is available the tool returns a structured background_unavailable error. 'foreground' is the explicit escalation: activate the target (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then restore the prior active window — a brief focus swap unless the target was already active. Matches the macOS / Windows delivery_mode surface.",
+      "enum": [
+        "background",
+        "foreground"
+      ],
+      "type": "string"
+    },
+    "element_index": {
+      "description": "Element index from get_window_state. Requires the matching `snapshot_id` alongside it. Prefer `element_token`, which carries both values.",
+      "type": "integer"
+    },
+    "element_token": {
+      "description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. If element_index, snapshot_id, or window_id are also supplied they must agree. Returns an explicit stale error once a newer snapshot supersedes it.",
+      "type": "string"
+    },
+    "pid": {
+      "type": "integer"
+    },
+    "scope": {
+      "default": "window",
+      "enum": [
+        "window",
+        "desktop"
+      ],
+      "type": "string"
+    },
+    "session": {
+      "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.",
+      "type": "string"
+    },
+    "snapshot_id": {
+      "description": "Snapshot handle from get_window_state. Required when targeting by element_index; stale snapshots fail closed.",
+      "pattern": "^s[0-9a-f]{8}$",
+      "type": "string"
+    },
+    "target": {
+      "anyOf": [
+        {
+          "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.",
+          "oneOf": [
+            {
+              "additionalProperties": true,
+              "properties": {
+                "kind": {
+                  "const": "window",
+                  "type": "string"
+                },
+                "pid": {
+                  "format": "uint32",
+                  "minimum": 0,
+                  "type": "integer"
+                },
+                "window_id": {
+                  "format": "uint64",
+                  "minimum": 0,
+                  "type": "integer"
+                }
+              },
+              "required": [
+                "kind",
+                "pid",
+                "window_id"
+              ],
+              "type": "object"
+            },
+            {
+              "additionalProperties": true,
+              "properties": {
+                "display_id": {
+                  "type": "string"
+                },
+                "kind": {
+                  "const": "desktop",
+                  "type": "string"
+                }
+              },
+              "required": [
+                "kind",
+                "display_id"
+              ],
+              "type": "object"
+            }
+          ]
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "text": {
+      "type": "string"
+    },
+    "window_id": {
+      "type": "integer"
+    },
+    "x": {
+      "description": "Screenshot-pixel X of the field to type into — the element px action form. Pass x,y (no element_index) and the tool pixel-clicks there to establish real renderer focus, then types. Use for Chromium/Electron inputs the AX path can't reach. Read straight off the get_window_state PNG, same convention as click.",
+      "type": "number"
+    },
+    "y": {
+      "description": "Screenshot-pixel Y of the field (see x).",
+      "type": "number"
+    }
+  },
+  "required": [
+    "text"
+  ],
+  "type": "object"
+}

docs/development.md

@@ -40,10 +40,12 @@ python3 tests/shell-session.test.py   # 持久終端機腳本，只需要 tmux
 
 # Cua Driver 能否控制現有 XFCE + Xvfb 桌面（會建 computer image）
 make cua-smoke
-# 結果摘要見 docs/cua-compatibility.md
+# 結果摘要見 docs/cua-compatibility.md、docs/cua-review.md
-# 生產路徑預設是 cua。要回退 CDP/xdotool：
+# 生產路徑預設仍是 legacy。要在本機明確跑 Cua：
-#   LAZYBOY_COMPUTER_DRIVER=legacy
+#   make cua-smoke
-# 寫進 .env 後重建 supervisor 與桌面容器。Agent 工具 schema 不變。
+#   docker compose -f docker-compose.yml -f docker-compose.cua.yml up -d --build
+# 或 LAZYBOY_COMPUTER_DRIVER=cua 寫進 .env 後重建 supervisor 與桌面容器。
+# Agent 工具 schema 不變。
 
 # Python 整合測試用 docker compose exec 連進 Postgres，自己建一次性資料庫後清掉
 python3 tests/retention.test.py

docs/operations.md

@@ -57,7 +57,7 @@
 | `LAZYBOY_COMPUTER_MEMORY_MB` | 每台電腦記憶體 | `2048` |
 | `LAZYBOY_COMPUTER_PIDS` | 每台電腦 PID 上限 | `2048` |
 | `LAZYBOY_COMPUTER_SUDO` | 容器內免密碼 sudo；重建桌面容器後生效 | `false` |
-| `LAZYBOY_COMPUTER_DRIVER` | 桌面控制後端：`cua`（預設）或 `legacy`（observe / act / browser / 示範錄製）；重建桌面容器後生效 | `cua` |
+| `LAZYBOY_COMPUTER_DRIVER` | 桌面控制後端：`legacy`（預設，CDP/AT-SPI/xdotool）或 `cua`（opt-in Cua Driver）。改完需重建桌面容器。本機可用 `docker compose -f docker-compose.yml -f docker-compose.cua.yml up -d --build` | `legacy` |
 | `LAZYBOY_MEMORY_ENABLED` | 長期記憶 | `true` |
 
 完整清單與保留政策請見 [`.env.example`](../.env.example)。

image/computer/Dockerfile

@@ -141,12 +141,19 @@ COPY --chmod=755 image/computer/chromium /usr/local/bin/chromium
 RUN sed -i 's|^Exec=/usr/bin/chromium|Exec=/usr/local/bin/lazyboy-browser|' /usr/share/applications/chromium.desktop || true
 
 # Pin Cua Driver outside the persisted /home/lazyboy bind-mount.
-# SHA256 is the linux-x86_64-binary tarball from the matching GitHub release.
+# Checksums are from the matching official release; select the target architecture.
 ARG CUA_DRIVER_RS_VERSION=0.23.2
-ARG CUA_DRIVER_RS_SHA256=01bf8339ec129cc00f4b4b2c6056ef1a7c5b52df39ff83ad17c9b16818aec500
+ARG TARGETARCH
-RUN curl -fsSL -o /tmp/cua-driver.tar.gz \
+ARG CUA_DRIVER_RS_SHA256_AMD64=01bf8339ec129cc00f4b4b2c6056ef1a7c5b52df39ff83ad17c9b16818aec500
-      "https://github.com/trycua/cua/releases/download/cua-driver-rs-v${CUA_DRIVER_RS_VERSION}/cua-driver-rs-${CUA_DRIVER_RS_VERSION}-linux-x86_64-binary.tar.gz" \
+ARG CUA_DRIVER_RS_SHA256_ARM64=be22768a207796a4bc1de50c52f32f9ef680b5e86e58c059e02eec2caba2e7bb
-  && echo "${CUA_DRIVER_RS_SHA256}  /tmp/cua-driver.tar.gz" | sha256sum -c \
+RUN case "$TARGETARCH" in \
+      amd64) cua_arch=x86_64; cua_sha="$CUA_DRIVER_RS_SHA256_AMD64" ;; \
+      arm64) cua_arch=arm64; cua_sha="$CUA_DRIVER_RS_SHA256_ARM64" ;; \
+      *) echo "Unsupported Cua architecture: $TARGETARCH" >&2; exit 1 ;; \
+    esac \
+  && curl -fsSL -o /tmp/cua-driver.tar.gz \
+      "https://github.com/trycua/cua/releases/download/cua-driver-rs-v${CUA_DRIVER_RS_VERSION}/cua-driver-rs-${CUA_DRIVER_RS_VERSION}-linux-${cua_arch}-binary.tar.gz" \
+  && echo "${cua_sha}  /tmp/cua-driver.tar.gz" | sha256sum -c \
   && mkdir -p /usr/local/lib/cua-driver \
   && tar -xzf /tmp/cua-driver.tar.gz -C /usr/local/lib/cua-driver \
   && chmod 755 /usr/local/lib/cua-driver/cua-driver \
@@ -159,6 +166,8 @@ COPY --chmod=755 image/computer/entrypoint.sh /usr/local/bin/lazyboy-entrypoint
 COPY --chmod=644 image/computer/cua-smoke.html /usr/share/lazyboy/cua-smoke.html
 COPY --chmod=755 image/computer/cua-smoke-gtk.py /usr/local/bin/lazyboy-cua-smoke-gtk
 COPY --chmod=755 scripts/cua-smoke-inner.py /usr/local/bin/lazyboy-cua-smoke
+COPY --chmod=755 scripts/cua-adapter-test.py /usr/local/bin/lazyboy-cua-adapter-test
+COPY --chmod=755 scripts/cua-isolation-test.py /usr/local/bin/lazyboy-cua-isolation-test
 COPY --chmod=755 scripts/cua-smoke-test.sh /usr/local/bin/lazyboy-cua-smoke-host
 
 USER root

image/computer/cua-smoke-gtk.py

@@ -2,20 +2,22 @@
 """Tiny GTK window used by the Cua smoke test to verify native click/type."""
 
 from pathlib import Path
+import json
+import os
 
 import gi
 
 gi.require_version("Gtk", "3.0")
-from gi.repository import Gtk
+from gi.repository import Gtk, Gdk, GLib
 
-ROOT = Path("/tmp/lazyboy")
+ROOT = Path(os.environ.get("LAZYBOY_SMOKE_ROOT", "/tmp/lazyboy"))
 CLICKED = ROOT / "cua-smoke-clicked"
 TYPED = ROOT / "cua-smoke-typed"
 
 
 class SmokeWindow(Gtk.Window):
     def __init__(self) -> None:
-        super().__init__(title="LazyBoy Cua Smoke")
+        super().__init__(title=os.environ.get("LAZYBOY_SMOKE_TITLE", "LazyBoy Cua Smoke"))
         self.set_default_size(480, 240)
         box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=8)
         box.set_margin_top(16)
@@ -36,12 +38,37 @@ class SmokeWindow(Gtk.Window):
         box.pack_start(self.entry, False, False, 0)
         box.pack_start(save, False, False, 0)
 
+        self.drag_events = []
+        self.drag = Gtk.DrawingArea()
+        self.drag.set_size_request(400, 80)
+        self.drag.add_events(Gdk.EventMask.BUTTON_PRESS_MASK | Gdk.EventMask.BUTTON_RELEASE_MASK | Gdk.EventMask.POINTER_MOTION_MASK | Gdk.EventMask.SCROLL_MASK)
+        for signal in ("button-press-event", "button-release-event", "motion-notify-event", "scroll-event"):
+            self.drag.connect(signal, self.on_drag_event, signal)
+        box.pack_start(self.drag, False, False, 0)
+        GLib.idle_add(self.save_drag_geometry)
+
         click.get_accessible().set_name("Smoke Click")
         self.entry.get_accessible().set_name("Smoke Entry")
         save.get_accessible().set_name("Smoke Save")
         self.status.get_accessible().set_name("Smoke Status")
         self.connect("destroy", Gtk.main_quit)
 
+    def save_drag_geometry(self):
+        # Root coords of the drawing area, not widget-local origin.
+        top = self.get_window().get_origin()
+        alloc = self.drag.get_allocation()
+        ROOT.mkdir(parents=True, exist_ok=True)
+        (ROOT / "cua-smoke-drag-geometry.json").write_text(json.dumps({
+            "x": int(top[-2]) + int(alloc.x),
+            "y": int(top[-1]) + int(alloc.y),
+        }))
+        return False
+
+    def on_drag_event(self, _widget, event, signal):
+        self.drag_events.append(signal)
+        (ROOT / "cua-smoke-drag.json").write_text(json.dumps(self.drag_events))
+        return False
+
     def on_click(self, _button: Gtk.Button) -> None:
         ROOT.mkdir(parents=True, exist_ok=True)
         CLICKED.write_text("clicked\n", encoding="utf-8")

image/computer/cua-smoke.html

@@ -11,6 +11,7 @@
 <body>
   <h1>LazyBoy Cua Smoke</h1>
   <p id="result">ready</p>
+  <p id="persistent"></p>
   <p>
     <button id="click" type="button">Smoke Click</button>
   </p>
@@ -19,11 +20,17 @@
     <button id="save" type="button">Smoke Save</button>
   </p>
   <script>
+    var retained = document.cookie.includes("cuaSmoke=retained");
+    document.getElementById("persistent").textContent = retained ? "session-retained" : "fresh-session";
+    if (retained) {
+      document.title = "LazyBoy Cua Smoke (session-retained)";
+    }
     document.getElementById("click").onclick = function () {
       document.getElementById("result").textContent = "clicked-ok";
     };
     document.getElementById("save").onclick = function () {
       var value = document.getElementById("name").value;
+      document.cookie = "cuaSmoke=retained; Max-Age=86400; SameSite=Lax; path=/";
       document.getElementById("result").textContent = "typed:" + value;
     };
   </script>

image/computer/lazyboy-screen

@@ -215,19 +215,27 @@ start_cua_driver() {
   local display="$1"
   local log="$2"
   local number="${display#:}"
+  [[ "${LAZYBOY_COMPUTER_DRIVER:-legacy}" == "cua" ]] || return 0
   if ! command -v cua-driver >/dev/null 2>&1; then
-    return 0
+    echo "Cua backend selected but cua-driver is not installed" >&2
+    return 1
   fi
+  local sock="$ROOT/cua-${number}.sock"
+  if [[ "$number" == "1" ]]; then sock="$ROOT/cua.sock"; fi
   if alive_pidfile "${log}-cua.pid"; then
-    return 0
+    if [[ -S "$sock" ]]; then return 0; fi
+    echo "Cua process exists but its display socket is missing" >&2
+    return 1
+  fi
+  export CUA_DRIVER_RS_HOME="$ROOT/cua-home-${number}"
+  if [[ -s "$ROOT/screen-${number}.dbus" ]]; then
+    export DBUS_SESSION_BUS_ADDRESS="$(cat "$ROOT/screen-${number}.dbus")"
+  fi
+  if [[ -s "$ROOT/screen-${number}.runtime" ]]; then
+    export XDG_RUNTIME_DIR="$(cat "$ROOT/screen-${number}.runtime")"
   fi
-  export CUA_DRIVER_RS_HOME="${CUA_DRIVER_RS_HOME:-$ROOT/cua-home}"
   mkdir -p "$CUA_DRIVER_RS_HOME"
   cua-driver telemetry disable >>"${log}-cua.log" 2>&1 || true
-  local sock="$ROOT/cua-${number}.sock"
-  if [[ "$number" == "1" ]]; then
-    sock="$ROOT/cua.sock"
-  fi
   rm -f "$sock"
   DISPLAY="$display" cua-driver serve \
     --grant existing-profile \
@@ -302,6 +310,7 @@ ensure_slot() {
     printf '%s\n' "$profile" >"$ROOT/screen-${number}.profile"
   fi
   if xdpyinfo -display "$display" >/dev/null 2>&1 && port_open "$vnc_port" && port_open "$view_port"; then
+    start_cua_driver "$display" "$log" || return 1
     if [[ -n "$profile" ]]; then
       start_browser "$display" "$profile" "$log"
     fi
@@ -315,6 +324,7 @@ ensure_slot() {
       exit 1
     }
     if xdpyinfo -display "$display" >/dev/null 2>&1 && port_open "$vnc_port" && port_open "$view_port"; then
+      start_cua_driver "$display" "$log" || exit 1
       if [[ -n "$profile" ]]; then
         start_browser "$display" "$profile" "$log"
       fi
@@ -328,7 +338,7 @@ ensure_slot() {
       exit 1
     }
     start_desktop "$display" "$xfce_home" "$log"
-    start_cua_driver "$display" "$log" || true
+    start_cua_driver "$display" "$log" || exit 1
     start_xterm "$display" "$log"
     start_vnc "$display" "$vnc_port" "$view_port" "$log" || exit 1
     if [[ -n "$profile" ]]; then

scripts/cua-adapter-test.py

@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+"""Exercise the real controld adapter in a disposable desktop, not raw Cua calls."""
+import argparse
+import base64
+import importlib.machinery
+import importlib.util
+import json
+import os
+import statistics
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+loader = importlib.machinery.SourceFileLoader("smoke", "/usr/local/bin/lazyboy-cua-smoke")
+spec = importlib.util.spec_from_loader(loader.name, loader)
+smoke = importlib.util.module_from_spec(spec)
+loader.exec_module(smoke)
+TIMINGS = {}
+
+
+def api(path, body=None, display=":1", expect_error=False):
+    request = urllib.request.Request(
+        "http://127.0.0.1:7070" + path,
+        data=None if body is None else json.dumps(body).encode(),
+        headers={"Authorization": "Bearer " + os.environ["LAZYBOY_CONTROL_TOKEN"],
+                 "Content-Type": "application/json", "x-lazyboy-display": display},
+    )
+    started = time.monotonic()
+    try:
+        with urllib.request.urlopen(request, timeout=150) as response:
+            result = json.load(response)
+    except urllib.error.HTTPError as error:
+        if expect_error:
+            assert error.code == 400, error.code
+            return None
+        raise AssertionError(f"{path}: HTTP {error.code}: {error.read()[:300]!r}") from error
+    finally:
+        if not expect_error:
+            TIMINGS.setdefault(path + ":" + str((body or {}).get("action", "")), []).append(
+                (time.monotonic() - started) * 1000
+            )
+    if expect_error:
+        assert not result.get("ok", True), result
+    else:
+        assert result.get("ok", True), result
+    return result
+
+
+def wait_health():
+    deadline = time.monotonic() + 120
+    last = None
+    while time.monotonic() < deadline:
+        try:
+            health = api("/controller/health")
+            if health.get("backend") == "cua" and health.get("healthy"):
+                return health
+            last = health
+        except Exception as error:
+            last = str(error)
+        time.sleep(0.5)
+    raise AssertionError(f"controld/cua health was not ready: {last}")
+
+
+def observe():
+    result = api("/observe", {})
+    png = base64.b64decode(result["png_base64"])
+    assert png.startswith(b"\x89PNG"), "no desktop screenshot"
+    return result
+
+
+def element(result, label):
+    found = [el for el in result.get("elements", []) if el["title"] == label]
+    assert len(found) == 1, (label, result.get("elements"))
+    return found[0]
+
+
+def act(action, expect_error=False):
+    return act_many([action], expect_error=expect_error)
+
+
+def act_many(actions, expect_error=False):
+    return api("/act", {"actions": actions, "observe": True, "settle_ms": 100},
+               expect_error=expect_error)
+
+
+def native_round():
+    gtk = smoke.launch_gtk()
+    try:
+        before = observe()
+        # Agent enrichment requests browser state after native observation.
+        api("/browser", {"action": "snapshot", "ensure": True})
+        target = element(before, "Smoke Click")
+        assert target["kind"] == "a11y", target
+        click = {"kind": "ref", "verb": "click", "refKind": "a11y", "target": target["selector"]}
+        after = act(click)
+        smoke.wait_file(smoke.CLICKED, "clicked")
+        # A successful mutation invalidates the previous observation's handles.
+        act(click, expect_error=True)
+        after = observe()
+        entry = element(after, "Smoke Entry")
+        save = element(after, "Smoke Save")
+        act_many([
+            {"kind": "ref", "verb": "setvalue", "refKind": "a11y",
+             "target": entry["selector"], "text": "中文 hello-cua"},
+            {"kind": "ref", "verb": "click", "refKind": "a11y", "target": save["selector"]},
+        ])
+        smoke.wait_file(smoke.TYPED, "中文 hello-cua")
+        act({"kind": "focus", "title": "LazyBoy Cua Smoke"})
+        page = observe()
+        gtk_windows = [el for el in page.get("elements", [])
+                       if el.get("kind") == "window"
+                       and "Cua Smoke" in el.get("title", "")
+                       and "Chromium" not in el.get("title", "")]
+        assert gtk_windows, page.get("elements")
+        win = gtk_windows[0]
+        start_x = win["x"] + 40
+        start_y = win["y"] + max(int(win["h"]), 80) - 40
+        last_error = None
+        for _ in range(3):
+            (smoke.ROOT / "cua-smoke-drag.json").unlink(missing_ok=True)
+            actions = []
+            for kind, offset in (("down", 0), ("move", 160), ("up", 160)):
+                if actions:
+                    actions.append({"kind": "wait", "ms": 40})
+                actions.append({"kind": "pointer", "type": kind, "button": "left",
+                                "x": start_x + offset, "y": start_y})
+            api("/act", {"actions": actions, "observe": False, "settle_ms": 150})
+            try:
+                events = json.loads(smoke.wait_file(
+                    smoke.ROOT / "cua-smoke-drag.json", "button-press-event", timeout=3))
+                break
+            except smoke.SmokeError as error:
+                last_error = error
+                act({"kind": "focus", "title": "LazyBoy Cua Smoke"})
+        else:
+            raise last_error
+        assert "button-press-event" in events and "motion-notify-event" in events and "button-release-event" in events, events
+
+    finally:
+        gtk.terminate()
+        gtk.wait(timeout=5)
+
+
+def browser_round():
+    page = api("/browser", {"action": "navigate", "url": smoke.FIXTURE_URL, "ensure": True})
+    target = element(page, "Smoke Click")
+    page = api("/browser", {"action": "click", "selector": target["selector"]})
+    assert "clicked-ok" in page["text"], page
+    entry = element(page, "Smoke Entry")
+    page = api("/browser", {"action": "type", "selector": entry["selector"], "text": "中文 hello-cua"})
+    save = element(page, "Smoke Save")
+    page = api("/browser", {"action": "click", "selector": save["selector"]})
+    assert "typed:中文 hello-cua" in page["text"], page
+    api("/browser", {"action": "click", "selector": "p999999:999999"}, expect_error=True)
+    # The browser and human view are the same existing profile/window.
+    assert smoke.find_browser_window(smoke.list_windows())
+    assert smoke.run(["xdpyinfo", "-display", ":1"]).returncode == 0
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--repeat", type=int, default=10)
+    parser.add_argument("--check-persistence", action="store_true")
+    args = parser.parse_args()
+    assert args.repeat > 0
+    smoke.REPORT.mkdir(parents=True, exist_ok=True)
+    smoke.wait_ready(120)
+    health = wait_health()
+    server = smoke.start_fixture_server()
+    summary = {"backend": health, "requested": args.repeat, "passed": 0}
+    try:
+        if args.check_persistence:
+            page = api("/browser", {"action": "navigate", "url": smoke.FIXTURE_URL, "ensure": True})
+            blob = (page.get("title") or "") + "\n" + (page.get("text") or "")
+            assert "session-retained" in blob, page
+            summary.update(requested=1, passed=1, persistent_cookie=True)
+            print("persistent browser cookie survived lifecycle change", flush=True)
+            return
+        for i in range(args.repeat):
+            native_round()
+            browser_round()
+            summary["passed"] += 1
+            print(f"adapter iteration {i + 1}/{args.repeat} passed", flush=True)
+    finally:
+        server.terminate()
+        server.wait(timeout=5)
+        summary["timings_ms"] = {key: {"count": len(values), "median": statistics.median(values),
+                                        "max": max(values)} for key, values in TIMINGS.items()}
+        (smoke.REPORT / ("persistence.json" if args.check_persistence else "adapter.json")).write_text(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()

scripts/cua-isolation-test.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""Verify Cua native handles cannot cross two LazyBoy display sessions."""
+import importlib.machinery
+import importlib.util
+import json
+import os
+import subprocess
+import time
+from pathlib import Path
+
+loader = importlib.machinery.SourceFileLoader("adapter", "/usr/local/bin/lazyboy-cua-adapter-test")
+spec = importlib.util.spec_from_loader(loader.name, loader)
+adapter = importlib.util.module_from_spec(spec)
+loader.exec_module(adapter)
+
+
+def main():
+    subprocess.run(["lazyboy-screen", "ensure", "1"], check=True, timeout=60)
+    adapter.smoke.REPORT.mkdir(parents=True, exist_ok=True)
+    roots = [Path("/tmp/lazyboy/isolation-1"), Path("/tmp/lazyboy/isolation-2")]
+    processes = []
+    try:
+        for number, root in enumerate(roots, 1):
+            root.mkdir(exist_ok=True)
+            (root / "cua-smoke-clicked").unlink(missing_ok=True)
+            env = os.environ.copy()
+            env.update(DISPLAY=f":{number}", LAZYBOY_SMOKE_ROOT=str(root),
+                       LAZYBOY_SMOKE_TITLE=f"Isolation {number}",
+                       DBUS_SESSION_BUS_ADDRESS=Path(f"/tmp/lazyboy/screen-{number}.dbus").read_text().strip())
+            processes.append(subprocess.Popen(["/usr/local/bin/lazyboy-cua-smoke-gtk"], env=env))
+        deadline = time.monotonic() + 10
+        while time.monotonic() < deadline and not Path("/tmp/lazyboy/cua-2.sock").is_socket():
+            time.sleep(0.2)
+        time.sleep(1)
+        pages = [adapter.api("/observe", {}, display=f":{n}") for n in (1, 2)]
+        targets = [adapter.element(page, "Smoke Click")["selector"] for page in pages]
+        assert targets[0] != targets[1]
+        assert all(adapter.api("/controller/health", display=f":{n}")["healthy"] for n in (1, 2))
+        body = lambda target: {"actions": [{"kind": "ref", "verb": "click", "refKind": "a11y", "target": target}], "observe": False, "settle_ms": 100}
+        adapter.api("/act", body(targets[0]), display=":2", expect_error=True)
+        assert not any((root / "cua-smoke-clicked").exists() for root in roots)
+        adapter.api("/act", body(targets[0]), display=":1")
+        assert (roots[0] / "cua-smoke-clicked").read_text().strip() == "clicked"
+        assert not (roots[1] / "cua-smoke-clicked").exists()
+        adapter.api("/act", body(targets[1]), display=":2")
+        assert (roots[1] / "cua-smoke-clicked").read_text().strip() == "clicked"
+        result = {"displays": 2, "cross_display_ref_rejected": True, "independent_clicks": True}
+        (adapter.smoke.REPORT / "isolation.json").write_text(json.dumps(result, indent=2))
+        print(json.dumps(result), flush=True)
+    finally:
+        for process in processes:
+            process.terminate()
+            process.wait(timeout=5)
+
+
+if __name__ == "__main__":
+    main()

scripts/cua-smoke-inner.py

@@ -522,6 +522,7 @@ def browser_prepare(pid: int, window_id: int) -> dict[str, Any]:
 
 
 def browser_round(iteration: int) -> dict[str, Any]:
+    cua_call("start_session", {"session": "lazyboy-smoke"})
     windows = list_windows()
     window = find_browser_window(windows)
     if not window:

scripts/cua-smoke-test.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 repeat=10
 ready_timeout=120
 image="${COMPUTER_IMAGE:-lazyboy/computer:local}"
-name="lazyboy-cua-smoke"
+name=""
 docker_mode=0
 
 usage() {
@@ -18,14 +18,17 @@ usage() {
 while [[ $# -gt 0 ]]; do
   case "$1" in
     --docker) docker_mode=1; shift ;;
-    --repeat) repeat="${2:-}"; shift 2 ;;
+    --repeat) [[ $# -ge 2 ]] || usage; repeat="$2"; shift 2 ;;
-    --image) image="${2:-}"; shift 2 ;;
+    --image) [[ $# -ge 2 ]] || usage; image="$2"; shift 2 ;;
-    --ready-timeout) ready_timeout="${2:-}"; shift 2 ;;
+    --ready-timeout) [[ $# -ge 2 ]] || usage; ready_timeout="$2"; shift 2 ;;
     -h|--help) usage ;;
     *) usage ;;
   esac
 done
 
+[[ "$repeat" =~ ^[1-9][0-9]*$ ]] || usage
+[[ "$ready_timeout" =~ ^[1-9][0-9]*$ ]] || usage
+
 if [[ "$docker_mode" -eq 0 ]] && [[ -x /usr/local/bin/lazyboy-cua-smoke ]] && [[ -S /tmp/lazyboy/cua.sock || -f /tmp/lazyboy/ready ]]; then
   exec /usr/local/bin/lazyboy-cua-smoke --repeat "$repeat" --ready-timeout "$ready_timeout"
 fi
@@ -47,14 +50,14 @@ if ! docker image inspect "$image" >/dev/null 2>&1; then
 fi
 
 cleanup() {
-  docker rm -f "$name" >/dev/null 2>&1 || true
+  if [[ -n "$name" ]]; then docker rm -f "$name" >/dev/null 2>&1 || true; fi
 }
 trap cleanup EXIT
 
-cleanup
+name=$(docker run -d --shm-size=512m \
-docker run -d --name "$name" --shm-size=512m \
+  -e DISPLAY=:1 -e LAZYBOY_COMPUTER_DRIVER=cua \
-  -e DISPLAY=:1 \
+  -e LAZYBOY_CONTROL_TOKEN=cua-smoke-local-only \
-  "$image" >/dev/null
+  "$image")
 
 echo "waiting for desktop + cua-driver in $name"
 for _ in $(seq 1 "$ready_timeout"); do
@@ -80,6 +83,38 @@ echo "running $repeat Cua smoke iterations"
 set +e
 docker exec -u 1000:1000 "$name" /usr/local/bin/lazyboy-cua-smoke --repeat "$repeat" --ready-timeout "$ready_timeout"
 code=$?
+if [[ "$code" -eq 0 ]]; then
+  docker exec -u 1000:1000 "$name" python3 /usr/local/bin/lazyboy-cua-adapter-test --repeat "$repeat"
+  code=$?
+fi
+if [[ "$code" -eq 0 ]]; then
+  docker exec -u 1000:1000 "$name" python3 /usr/local/bin/lazyboy-cua-isolation-test
+  code=$?
+fi
+if [[ "$code" -eq 0 ]]; then
+  docker pause "$name" >/dev/null && docker unpause "$name" >/dev/null
+  code=$?
+fi
+if [[ "$code" -eq 0 ]]; then
+  docker exec -u 1000:1000 "$name" python3 /usr/local/bin/lazyboy-cua-adapter-test --check-persistence
+  code=$?
+fi
+if [[ "$code" -eq 0 ]]; then
+  docker restart "$name" >/dev/null
+  code=$?
+fi
+if [[ "$code" -eq 0 ]]; then
+  echo "waiting for desktop after restart"
+  for _ in $(seq 1 "$ready_timeout"); do
+    if docker exec -u 1000:1000 "$name" test -f /tmp/lazyboy/ready \
+      && docker exec -u 1000:1000 "$name" test -S /tmp/lazyboy/cua.sock; then
+      break
+    fi
+    sleep 1
+  done
+  docker exec -u 1000:1000 "$name" python3 /usr/local/bin/lazyboy-cua-adapter-test --check-persistence
+  code=$?
+fi
 set -e
 
 out="${CUA_SMOKE_OUT:-/tmp/lazyboy-cua-smoke-last}"