//! Optional Playwright DOM browser tools (P3) + human handoff (P4). //! //! Spawns `tools/playwright/browser_helper.mjs` (JSONL over stdin/stdout). //! Fail closed with an install hint when Node / Playwright / Chromium is missing. //! Prefer selectors / roles — not screenshot-first control. //! //! Env: //! - `GROKBOY_BROWSER_HEADED=1` — launch Chromium headed (visible) by default //! - `GROKBOY_HANDOFF_AUTO=1` — auto-resume handoff (tests); `abort` to auto-abort use anyhow::{Context, Result, anyhow}; use serde_json::{Value, json}; use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; use std::sync::{Arc, Mutex, OnceLock}; use std::sync::mpsc; use std::time::{Duration, Instant}; pub const INSTALL_HINT: &str = "Playwright browser tools unavailable. Install with: \ cd tools/playwright && npm install && npx playwright install chromium"; const HELPER_REL: &str = "tools/playwright/browser_helper.mjs"; const HELPER_TIMEOUT: Duration = Duration::from_secs(60); /// Shared last navigated URL for session metadata. pub type LastUrlSlot = Arc>>; static HELPER: OnceLock> = OnceLock::new(); enum HelperState { /// Not started yet. Idle, /// Running JSONL child. Running { #[allow(dead_code)] child: Child, stdin: ChildStdin, stdout: BufReader, next_id: u64, }, /// Permanently unavailable this process (missing node/helper). Unavailable(String), } fn helper_lock() -> &'static Mutex { HELPER.get_or_init(|| Mutex::new(HelperState::Idle)) } /// Locate the helper script relative to cwd, then walk parents, then exe-relative. pub fn find_helper_script(cwd: &Path) -> Option { let mut dir = cwd.to_path_buf(); for _ in 0..8 { let candidate = dir.join(HELPER_REL); if candidate.is_file() { return Some(candidate); } if !dir.pop() { break; } } // Also try next to the running binary's ancestors (dev: target/debug). if let Ok(exe) = std::env::current_exe() { let mut dir = exe.parent().map(|p| p.to_path_buf()); for _ in 0..6 { let Some(d) = dir else { break }; let candidate = d.join(HELPER_REL); if candidate.is_file() { return Some(candidate); } // target/debug -> repo root if let Some(parent) = d.parent() { let up2 = parent.parent().map(|p| p.join(HELPER_REL)); if let Some(c) = up2 { if c.is_file() { return Some(c); } } } dir = d.parent().map(|p| p.to_path_buf()); } } None } fn which_node() -> Option { // Prefer PATH lookup. if let Ok(output) = Command::new("sh") .arg("-c") .arg("command -v node") .output() { if output.status.success() { let p = String::from_utf8_lossy(&output.stdout).trim().to_string(); if !p.is_empty() { return Some(PathBuf::from(p)); } } } None } fn spawn_helper(script: &Path) -> Result<(Child, ChildStdin, BufReader)> { let node = which_node().ok_or_else(|| { anyhow!("{INSTALL_HINT} (node not found on PATH)") })?; let mut child = Command::new(&node) .arg(script) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .with_context(|| format!("spawn node {}", script.display()))?; let stdin = child .stdin .take() .ok_or_else(|| anyhow!("helper stdin missing"))?; let stdout = child .stdout .take() .ok_or_else(|| anyhow!("helper stdout missing"))?; Ok((child, stdin, BufReader::new(stdout))) } fn ensure_running<'a>(state: &'a mut HelperState, cwd: &Path) -> Result<(&'a mut ChildStdin, &'a mut BufReader, &'a mut u64)> { match state { HelperState::Unavailable(msg) => Err(anyhow!("{msg}")), HelperState::Running { stdin, stdout, next_id, .. } => Ok((stdin, stdout, next_id)), HelperState::Idle => { let script = match find_helper_script(cwd) { Some(p) => p, None => { let msg = format!( "{INSTALL_HINT} (helper not found at {HELPER_REL} from {})", cwd.display() ); *state = HelperState::Unavailable(msg.clone()); return Err(anyhow!("{msg}")); } }; match spawn_helper(&script) { Ok((child, stdin, stdout)) => { *state = HelperState::Running { child, stdin, stdout, next_id: 1, }; match state { HelperState::Running { stdin, stdout, next_id, .. } => Ok((stdin, stdout, next_id)), _ => unreachable!(), } } Err(e) => { let msg = format!("{INSTALL_HINT} ({e:#})"); *state = HelperState::Unavailable(msg.clone()); Err(anyhow!("{msg}")) } } } } } fn read_json_line(stdout: &mut BufReader, deadline: Instant) -> Result { let mut line = String::new(); loop { if Instant::now() > deadline { return Err(anyhow!("browser helper timed out waiting for response")); } // Blocking read — browser ops are infrequent; keep it simple. line.clear(); let n = stdout .read_line(&mut line) .context("read helper stdout")?; if n == 0 { return Err(anyhow!("browser helper exited unexpectedly")); } let trimmed = line.trim(); if trimmed.is_empty() { continue; } let v: Value = serde_json::from_str(trimmed) .with_context(|| format!("helper returned non-JSON: {trimmed}"))?; return Ok(v); } } /// Send one JSON request to the helper; returns the parsed response object. pub fn browser_request(cwd: &Path, mut req: Value) -> Result { let mut guard = helper_lock() .lock() .map_err(|_| anyhow!("browser helper lock poisoned"))?; let (stdin, stdout, next_id) = ensure_running(&mut guard, cwd)?; let id = *next_id; *next_id += 1; if req.get("id").is_none() { req["id"] = json!(id.to_string()); } let line = serde_json::to_string(&req)? + "\n"; stdin .write_all(line.as_bytes()) .context("write to browser helper")?; stdin.flush().context("flush browser helper")?; let deadline = Instant::now() + HELPER_TIMEOUT; let resp = read_json_line(stdout, deadline)?; // If helper reported permanent missing playwright, mark unavailable for clearer retries. if resp.get("ok") == Some(&json!(false)) { if let Some(code) = resp.get("code").and_then(|c| c.as_str()) { if code == "PLAYWRIGHT_MISSING" || code == "CHROMIUM_MISSING" { // Keep process; user may install mid-session — don't mark Unavailable. } } } Ok(resp) } /// One-shot `--cmd` invocation (no daemon). Useful for offline protocol tests. pub fn browser_oneshot(cwd: &Path, req: &Value) -> Result { let script = find_helper_script(cwd).ok_or_else(|| { anyhow!("{INSTALL_HINT} (helper not found)") })?; let node = which_node().ok_or_else(|| anyhow!("{INSTALL_HINT} (node not found)"))?; let cmd_json = serde_json::to_string(req)?; let output = Command::new(node) .arg(&script) .arg("--cmd") .arg(&cmd_json) .current_dir(cwd) .output() .context("oneshot browser helper")?; let stdout = String::from_utf8_lossy(&output.stdout); let line = stdout .lines() .rev() .find(|l| !l.trim().is_empty()) .unwrap_or(""); if line.is_empty() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(anyhow!( "browser helper produced no JSON (stderr: {})", stderr.trim() )); } let v: Value = serde_json::from_str(line) .with_context(|| format!("oneshot non-JSON: {line}"))?; Ok(v) } /// Run helper `--self-test` (no Chromium). Returns parsed summary JSON. pub fn browser_self_test(cwd: &Path) -> Result { let script = find_helper_script(cwd).ok_or_else(|| { anyhow!("{INSTALL_HINT} (helper not found)") })?; let node = which_node().ok_or_else(|| anyhow!("{INSTALL_HINT} (node not found)"))?; let output = Command::new(node) .arg(&script) .arg("--self-test") .current_dir( script .parent() .unwrap_or(cwd), ) .output() .context("browser helper --self-test")?; let stdout = String::from_utf8_lossy(&output.stdout); let line = stdout .lines() .rev() .find(|l| !l.trim().is_empty()) .unwrap_or(""); if line.is_empty() { return Err(anyhow!( "self-test empty stdout; stderr={}", String::from_utf8_lossy(&output.stderr) )); } let v: Value = serde_json::from_str(line)?; if !output.status.success() || v.get("ok") != Some(&json!(true)) { return Err(anyhow!("self-test failed: {v}")); } Ok(v) } pub fn update_last_url(slot: &LastUrlSlot, resp: &Value) { if let Some(url) = resp.get("url").and_then(|u| u.as_str()) { if let Ok(mut g) = slot.lock() { *g = Some(url.to_string()); } } } pub fn response_to_tool_json(resp: Value) -> Value { if resp.get("ok") == Some(&json!(true)) { let mut out = resp; if let Some(obj) = out.as_object_mut() { obj.remove("ok"); obj.remove("id"); } out } else { let err = resp .get("error") .and_then(|e| e.as_str()) .unwrap_or("browser tool failed"); json!({ "error": err, "blocked": true, "install_hint": INSTALL_HINT, "detail": resp, }) } } /// OpenAI tool definitions for browser ops (always registered; fail closed if missing). pub fn browser_tool_definitions() -> Vec { vec![ json!({ "type": "function", "function": { "name": "browser_navigate", "description": "Navigate the optional Playwright browser to a URL (DOM path). Fails closed with install hint if Playwright is missing.", "parameters": { "type": "object", "properties": { "url": { "type": "string", "description": "URL to open" } }, "required": ["url"] } } }), json!({ "type": "function", "function": { "name": "browser_snapshot", "description": "Structured text snapshot of the page (roles/names/CSS selectors). Prefer this over screenshots for control.", "parameters": { "type": "object", "properties": {}, "additionalProperties": false } } }), json!({ "type": "function", "function": { "name": "browser_click", "description": "Click an element by CSS selector or role/name (Playwright DOM). Not pixel/XY click.", "parameters": { "type": "object", "properties": { "selector": { "type": "string", "description": "CSS selector" }, "role": { "type": "string", "description": "ARIA role, e.g. button, link" }, "name": { "type": "string", "description": "Accessible name when using role" }, "text": { "type": "string", "description": "Visible text alternative" }, "label": { "type": "string" }, "placeholder": { "type": "string" } } } } }), json!({ "type": "function", "function": { "name": "browser_type", "description": "Type/fill text into an element by CSS selector or role/name.", "parameters": { "type": "object", "properties": { "text": { "type": "string", "description": "Text to type" }, "selector": { "type": "string" }, "role": { "type": "string" }, "name": { "type": "string" }, "label": { "type": "string" }, "placeholder": { "type": "string" }, "clear": { "type": "boolean", "description": "Fill (clear first) vs append; default true" } }, "required": ["text"] } } }), json!({ "type": "function", "function": { "name": "browser_eval", "description": "Evaluate a JavaScript expression in the page context (use sparingly; prefer snapshot/click/type).", "parameters": { "type": "object", "properties": { "expression": { "type": "string", "description": "JS expression to eval in page" } }, "required": ["expression"] } } }), json!({ "type": "function", "function": { "name": "browser_handoff", "description": "Pause for human help on auth walls (login, OTP, captcha). Shows a headed Chromium window, prints terminal instructions (ZH+EN), waits for Enter (or abort), then returns a DOM snapshot so the agent can continue. Prefer when automation cannot pass the wall.", "parameters": { "type": "object", "properties": { "reason": { "type": "string", "description": "Why human help is needed (e.g. login page, OTP, captcha)" }, "timeout_secs": { "type": "integer", "description": "Seconds to wait for user (default 300)" } }, "required": ["reason"] } } }), ] } pub fn is_browser_tool(name: &str) -> bool { matches!( name, "browser_navigate" | "browser_snapshot" | "browser_dom" | "browser_click" | "browser_type" | "browser_eval" | "browser_handoff" ) } pub async fn execute_browser_tool( cwd: &Path, last_url: &LastUrlSlot, name: &str, args: &Value, ) -> Result { if name == "browser_handoff" { return execute_browser_handoff(cwd, last_url, args).await; } // Run blocking helper I/O off the async runtime. let cwd = cwd.to_path_buf(); let name = name.to_string(); let args = args.clone(); let last_url = last_url.clone(); tokio::task::spawn_blocking(move || { let op = match name.as_str() { "browser_navigate" => "navigate", "browser_snapshot" | "browser_dom" => "snapshot", "browser_click" => "click", "browser_type" => "type", "browser_eval" => "eval", other => return Err(anyhow!("unknown browser tool: {other}")), }; let mut req = args; if let Some(obj) = req.as_object_mut() { obj.insert("op".into(), json!(op)); } else { req = json!({ "op": op }); } // Normalize eval field if op == "eval" { if req.get("expression").is_none() { if let Some(js) = req.get("js").cloned() { req["expression"] = js; } } } let resp = browser_request(&cwd, req)?; update_last_url(&last_url, &resp); Ok(response_to_tool_json(resp)) }) .await .map_err(|e| anyhow!("browser task join: {e}"))? } /// Outcome of waiting for the human during handoff. #[derive(Debug, Clone, PartialEq, Eq)] pub enum HandoffWait { Resumed, Aborted(String), TimedOut, } const DEFAULT_HANDOFF_TIMEOUT_SECS: u64 = 300; /// Wait for stdin line (Enter to continue, `abort` to cancel) or timeout. /// /// `GROKBOY_HANDOFF_AUTO=1|resume` skips the wait (tests). /// `GROKBOY_HANDOFF_AUTO=abort` auto-aborts. pub fn wait_for_handoff_resume(timeout_secs: u64) -> HandoffWait { match std::env::var("GROKBOY_HANDOFF_AUTO") { Ok(v) => { let v = v.trim().to_ascii_lowercase(); if v == "1" || v == "true" || v == "yes" || v == "resume" || v == "continue" { return HandoffWait::Resumed; } if v == "abort" || v == "0" || v == "false" || v == "no" { return HandoffWait::Aborted(format!("GROKBOY_HANDOFF_AUTO={v}")); } } Err(_) => {} } let (tx, rx) = mpsc::channel::>(); std::thread::spawn(move || { let stdin = std::io::stdin(); let mut line = String::new(); match stdin.lock().read_line(&mut line) { Ok(0) => { let _ = tx.send(Err("stdin closed (EOF)".into())); } Ok(_) => { let _ = tx.send(Ok(line)); } Err(e) => { let _ = tx.send(Err(format!("stdin read error: {e}"))); } } }); match rx.recv_timeout(Duration::from_secs(timeout_secs.max(1))) { Ok(Ok(line)) => { let t = line.trim().to_ascii_lowercase(); if t == "abort" || t == "q" || t == "quit" || t == "cancel" { HandoffWait::Aborted(format!("user typed {t}")) } else { // Empty line (Enter) or any other input → resume. HandoffWait::Resumed } } Ok(Err(msg)) => HandoffWait::Aborted(msg), Err(mpsc::RecvTimeoutError::Timeout) => HandoffWait::TimedOut, Err(mpsc::RecvTimeoutError::Disconnected) => { HandoffWait::Aborted("handoff wait thread disconnected".into()) } } } fn print_handoff_instructions(reason: &str, timeout_secs: u64) { let banner = format!( " ╔══════════════════════════════════════════════════════════════╗ ║ GrokBoy P4 — Human handoff / 人工接手 ║ ╚══════════════════════════════════════════════════════════════╝ 【為什麼暫停 / Why paused】 {reason} 【請你做什麼 / What to do】 1. 看著已開啟的 Chromium 視窗(headed / visible)。 Look at the visible Chromium window. 2. 完成登入、OTP、驗證碼或其他真人操作。 Complete login / OTP / captcha (or whatever is blocking). 3. 完成後回到這個終端機,按 Enter 繼續。 When done, return here and press Enter to continue. 4. 若要放棄,輸入 abort 再按 Enter。 To give up, type abort then Enter. Timeout / 逾時: {timeout_secs}s (tests: GROKBOY_HANDOFF_AUTO=1 to auto-resume) ─── waiting for Enter / 等待 Enter ─── " ); eprintln!("{banner}"); let _ = std::io::Write::flush(&mut std::io::stderr()); } /// `browser_handoff`: prepare headed browser, wait for human, return snapshot. pub async fn execute_browser_handoff( cwd: &Path, last_url: &LastUrlSlot, args: &Value, ) -> Result { let reason = args .get("reason") .and_then(|v| v.as_str()) .map(str::trim) .filter(|s| !s.is_empty()) .ok_or_else(|| anyhow!("browser_handoff: missing 'reason'"))? .to_string(); let timeout_secs = args .get("timeout_secs") .and_then(|v| v.as_u64()) .filter(|&n| n > 0) .unwrap_or(DEFAULT_HANDOFF_TIMEOUT_SECS); let cwd = cwd.to_path_buf(); let last_url = last_url.clone(); tokio::task::spawn_blocking(move || { // 1) Ensure headed Chromium (JSONL daemon keeps page state when already headed). let prep = browser_request( &cwd, json!({ "op": "handoff_prepare", "reason": reason, }), )?; if prep.get("ok") != Some(&json!(true)) { return Ok(response_to_tool_json(prep)); } update_last_url(&last_url, &prep); // 2) Instruct the human (bilingual). print_handoff_instructions(&reason, timeout_secs); // 3) Block until Enter / abort / timeout. let wait = wait_for_handoff_resume(timeout_secs); match wait { HandoffWait::Aborted(msg) => { return Ok(json!({ "status": "blocked", "blocked": true, "handoff": "aborted", "reason": format!("human handoff aborted: {msg}"), "original_reason": reason, })); } HandoffWait::TimedOut => { return Ok(json!({ "status": "blocked", "blocked": true, "handoff": "timeout", "reason": format!( "human handoff timed out after {timeout_secs}s (fail-closed)" ), "original_reason": reason, })); } HandoffWait::Resumed => {} } // 4) Snapshot so the model can continue from post-login DOM. let snap = browser_request(&cwd, json!({ "op": "snapshot" }))?; update_last_url(&last_url, &snap); let mut out = response_to_tool_json(snap); if let Some(obj) = out.as_object_mut() { obj.insert("handoff".into(), json!("resumed")); obj.insert("handoff_reason".into(), json!(reason)); obj.insert( "message".into(), json!("Human handoff resumed; DOM snapshot attached."), ); } Ok(out) }) .await .map_err(|e| anyhow!("browser handoff join: {e}"))? } #[cfg(test)] mod tests { use super::*; use std::time::{SystemTime, UNIX_EPOCH}; fn repo_cwd() -> PathBuf { // Crate manifest dir is crates/grokboy-core → repo root is ../.. PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../..") .canonicalize() .unwrap_or_else(|_| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..")) } #[test] fn finds_helper_from_repo_root() { let root = repo_cwd(); let helper = find_helper_script(&root); assert!( helper.is_some(), "expected helper under {}/{HELPER_REL}", root.display() ); } #[test] fn helper_self_test_protocol() { let root = repo_cwd(); // Skip soft if node missing (should still usually be present on Mac). if which_node().is_none() { eprintln!("skip: node not on PATH"); return; } let summary = browser_self_test(&root).expect("self-test"); assert_eq!(summary["ok"], true); } #[test] fn oneshot_ping_json_protocol() { let root = repo_cwd(); if which_node().is_none() { eprintln!("skip: node not on PATH"); return; } let resp = browser_oneshot(&root, &json!({"op": "ping", "id": "u1"})).unwrap(); assert_eq!(resp["ok"], true); assert_eq!(resp["pong"], true); assert_eq!(resp["protocol"], 1); } #[test] fn response_maps_errors_to_blocked() { let v = response_to_tool_json(json!({ "ok": false, "blocked": true, "error": "nope", "id": "1" })); assert!(v.get("error").is_some()); assert_eq!(v["blocked"], true); assert!(v["install_hint"].as_str().unwrap().contains("npm install")); } #[test] fn browser_defs_count() { assert_eq!(browser_tool_definitions().len(), 6); assert!(is_browser_tool("browser_navigate")); assert!(is_browser_tool("browser_snapshot")); assert!(is_browser_tool("browser_handoff")); assert!(!is_browser_tool("shell")); } #[test] fn handoff_auto_resume_and_abort() { // SAFETY: tests run serially for this env in practice; restore after. let prev = std::env::var("GROKBOY_HANDOFF_AUTO").ok(); unsafe { std::env::set_var("GROKBOY_HANDOFF_AUTO", "1") }; assert_eq!(wait_for_handoff_resume(1), HandoffWait::Resumed); unsafe { std::env::set_var("GROKBOY_HANDOFF_AUTO", "abort") }; match wait_for_handoff_resume(1) { HandoffWait::Aborted(msg) => assert!(msg.contains("abort"), "{msg}"), other => panic!("expected aborted, got {other:?}"), } match prev { Some(v) => unsafe { std::env::set_var("GROKBOY_HANDOFF_AUTO", v) }, None => unsafe { std::env::remove_var("GROKBOY_HANDOFF_AUTO") }, } } #[test] fn oneshot_ping_lists_handoff_prepare() { let root = repo_cwd(); if which_node().is_none() { return; } let resp = browser_oneshot(&root, &json!({"op": "ping", "id": "h1"})).unwrap(); let ops = resp["ops"].as_array().expect("ops"); assert!( ops.iter().any(|o| o.as_str() == Some("handoff_prepare")), "{resp}" ); } #[test] fn install_hint_constant() { assert!(INSTALL_HINT.contains("npx playwright install chromium")); } #[tokio::test] async fn execute_ping_via_navigate_missing_url_style() { // Exercise execute path with type missing text → blocked JSON, no chromium needed // if helper starts. Use browser_type without text through execute_browser_tool args check. let root = repo_cwd(); if which_node().is_none() || find_helper_script(&root).is_none() { return; } let slot: LastUrlSlot = Arc::new(Mutex::new(None)); // Direct request ping through oneshot already covered; here ensure spawn works with status via request. let resp = browser_oneshot(&root, &json!({"op":"status"})).unwrap(); assert_eq!(resp["ok"], true); let _ = slot; let _ = SystemTime::now().duration_since(UNIX_EPOCH); } }