diff --git a/.gitignore b/.gitignore index 4027b13..6bda689 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ .DS_Store .env *.swp + +# Optional Playwright helper +tools/playwright/node_modules/ +tools/playwright/package-lock.json diff --git a/README.md b/README.md index b0d67d4..d085849 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # GrokBoy -Minimal local **GrokBot-like** CLI agent. Phase **P2**: completion contract, loop guard, context truncation. +Minimal local **GrokBot-like** CLI agent. Phase **P3**: optional Playwright DOM browser tools. ## Status @@ -9,7 +9,7 @@ Minimal local **GrokBot-like** CLI agent. Phase **P2**: completion contract, loo | P0 streaming chat | done | | P1 shell / files + ReAct | done | | P2 completion / loop guard / truncation | done | -| P3 browser (Playwright) | later | +| P3 browser (Playwright DOM) | done (optional) | No Docker desktop, no Codex/LazyBoy fork. @@ -26,36 +26,52 @@ cd ~/GrokBoy cargo run -p grokboy -- chat ``` +### Optional: Playwright browser tools + +Browser tools are always registered but **fail closed** until you install the helper: + +```bash +cd ~/GrokBoy/tools/playwright +npm install +npx playwright install chromium +``` + +Then the agent can use `browser_navigate`, `browser_snapshot`, `browser_click`, `browser_type`, `browser_eval` (DOM/selector/role — not screenshot-first). + ## Commands - `grokboy chat` — interactive streaming chat (no tools) - `grokboy run ""` — one-shot agent with tools - `grokboy run --session ""` — continue a saved session -- `grokboy smoke` — offline checks (no API key required) +- `grokboy smoke` — offline checks (no API key / no Chromium required) - `grokboy help` -Tools: `shell`, `list_dir`, `read_file`, `write_file`, `report_done`, `report_blocked`. +Tools: `shell`, `list_dir`, `read_file`, `write_file`, `report_done`, `report_blocked`, +`browser_navigate`, `browser_snapshot`, `browser_click`, `browser_type`, `browser_eval`. -Sessions are stored under `~/.grokboy/sessions/.json`. +Sessions are stored under `~/.grokboy/sessions/.json` (may include `last_browser_url`). The agent stops on `report_done` / `report_blocked`, blocks identical tool rounds (×3), and truncates old context when over budget. ## 繁體中文 -本機終端機 coding assistant。P2 已支援完成合約(`report_done` / `report_blocked`)、迴圈守衛與上下文截斷。 +本機終端機 coding assistant。P3 可選 Playwright DOM 瀏覽器工具(非截圖優先)。 ```bash export GROKBOY_API_KEY=你的金鑰 cd ~/GrokBoy cargo run -p grokboy -- smoke -cargo run -p grokboy -- run "列出目前目錄並讀 README.md" +# 可選瀏覽器: +cd tools/playwright && npm install && npx playwright install chromium +cargo run -p grokboy -- run "打開 example.com 並 snapshot" cargo run -p grokboy -- chat ``` ## Layout ``` -crates/grokboy-core/ # config, model, tools, agent, session -crates/grokboy/ # CLI binary +crates/grokboy-core/ # config, model, tools, browser, agent, session +crates/grokboy/ # CLI binary +tools/playwright/ # optional Node Playwright helper (JSONL) docs/ACCEPTANCE.md ``` diff --git a/crates/grokboy-core/src/agent.rs b/crates/grokboy-core/src/agent.rs index d8ad7e4..2b04f1e 100644 --- a/crates/grokboy-core/src/agent.rs +++ b/crates/grokboy-core/src/agent.rs @@ -15,7 +15,8 @@ pub const AGENT_SYSTEM: &str = "\ You are GrokBoy, a concise local coding assistant with tools. Use tools when they help solve the task; otherwise answer directly. Prefer short, clear answers. Traditional Chinese is welcome when the user writes in Chinese. -Available tools: shell, list_dir, read_file, write_file, report_done, report_blocked. +Available tools: shell, list_dir, read_file, write_file, report_done, report_blocked, and optional browser_* (Playwright DOM: browser_navigate, browser_snapshot, browser_click, browser_type, browser_eval). +For web pages prefer DOM snapshot + selector/role click/type — not screenshots or pixel XY clicks. When the task is finished, call report_done with a short summary. If you are stuck or cannot proceed, call report_blocked with the reason — do not invent results or loop. Do not invent tool results — call the tools."; @@ -312,6 +313,7 @@ mod tests { assert!(AGENT_SYSTEM.contains("shell")); assert!(AGENT_SYSTEM.contains("report_done")); assert!(AGENT_SYSTEM.contains("report_blocked")); + assert!(AGENT_SYSTEM.contains("browser_navigate") || AGENT_SYSTEM.contains("browser_")); assert!(AGENT_SYSTEM.contains("Traditional Chinese") || AGENT_SYSTEM.contains("Chinese")); } diff --git a/crates/grokboy-core/src/browser.rs b/crates/grokboy-core/src/browser.rs new file mode 100644 index 0000000..77623da --- /dev/null +++ b/crates/grokboy-core/src/browser.rs @@ -0,0 +1,553 @@ +//! Optional Playwright DOM browser tools (P3). +//! +//! Spawns `tools/playwright/browser_helper.mjs` (JSONL over stdin/stdout). +//! Fail closed with an install hint when Node / Playwright / Chromium is missing. +//! Prefer selectors / roles — not screenshot-first control. + +use anyhow::{Context, Result, anyhow}; +use serde_json::{Value, json}; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::time::{Duration, Instant}; + +pub const INSTALL_HINT: &str = "Playwright browser tools unavailable. Install with: \ +cd tools/playwright && npm install && npx playwright install chromium"; + +const HELPER_REL: &str = "tools/playwright/browser_helper.mjs"; +const HELPER_TIMEOUT: Duration = Duration::from_secs(60); + +/// Shared last navigated URL for session metadata. +pub type LastUrlSlot = Arc>>; + +static HELPER: OnceLock> = OnceLock::new(); + +enum HelperState { + /// Not started yet. + Idle, + /// Running JSONL child. + Running { + #[allow(dead_code)] + child: Child, + stdin: ChildStdin, + stdout: BufReader, + next_id: u64, + }, + /// Permanently unavailable this process (missing node/helper). + Unavailable(String), +} + +fn helper_lock() -> &'static Mutex { + HELPER.get_or_init(|| Mutex::new(HelperState::Idle)) +} + +/// Locate the helper script relative to cwd, then walk parents, then exe-relative. +pub fn find_helper_script(cwd: &Path) -> Option { + let mut dir = cwd.to_path_buf(); + for _ in 0..8 { + let candidate = dir.join(HELPER_REL); + if candidate.is_file() { + return Some(candidate); + } + if !dir.pop() { + break; + } + } + // Also try next to the running binary's ancestors (dev: target/debug). + if let Ok(exe) = std::env::current_exe() { + let mut dir = exe.parent().map(|p| p.to_path_buf()); + for _ in 0..6 { + let Some(d) = dir else { break }; + let candidate = d.join(HELPER_REL); + if candidate.is_file() { + return Some(candidate); + } + // target/debug -> repo root + if let Some(parent) = d.parent() { + let up2 = parent.parent().map(|p| p.join(HELPER_REL)); + if let Some(c) = up2 { + if c.is_file() { + return Some(c); + } + } + } + dir = d.parent().map(|p| p.to_path_buf()); + } + } + None +} + +fn which_node() -> Option { + // Prefer PATH lookup. + if let Ok(output) = Command::new("sh") + .arg("-c") + .arg("command -v node") + .output() + { + if output.status.success() { + let p = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if !p.is_empty() { + return Some(PathBuf::from(p)); + } + } + } + None +} + +fn spawn_helper(script: &Path) -> Result<(Child, ChildStdin, BufReader)> { + let node = which_node().ok_or_else(|| { + anyhow!("{INSTALL_HINT} (node not found on PATH)") + })?; + + let mut child = Command::new(&node) + .arg(script) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("spawn node {}", script.display()))?; + + let stdin = child + .stdin + .take() + .ok_or_else(|| anyhow!("helper stdin missing"))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| anyhow!("helper stdout missing"))?; + Ok((child, stdin, BufReader::new(stdout))) +} + +fn ensure_running<'a>(state: &'a mut HelperState, cwd: &Path) -> Result<(&'a mut ChildStdin, &'a mut BufReader, &'a mut u64)> { + match state { + HelperState::Unavailable(msg) => Err(anyhow!("{msg}")), + HelperState::Running { stdin, stdout, next_id, .. } => Ok((stdin, stdout, next_id)), + HelperState::Idle => { + let script = match find_helper_script(cwd) { + Some(p) => p, + None => { + let msg = format!( + "{INSTALL_HINT} (helper not found at {HELPER_REL} from {})", + cwd.display() + ); + *state = HelperState::Unavailable(msg.clone()); + return Err(anyhow!("{msg}")); + } + }; + match spawn_helper(&script) { + Ok((child, stdin, stdout)) => { + *state = HelperState::Running { + child, + stdin, + stdout, + next_id: 1, + }; + match state { + HelperState::Running { + stdin, + stdout, + next_id, + .. + } => Ok((stdin, stdout, next_id)), + _ => unreachable!(), + } + } + Err(e) => { + let msg = format!("{INSTALL_HINT} ({e:#})"); + *state = HelperState::Unavailable(msg.clone()); + Err(anyhow!("{msg}")) + } + } + } + } +} + +fn read_json_line(stdout: &mut BufReader, deadline: Instant) -> Result { + let mut line = String::new(); + loop { + if Instant::now() > deadline { + return Err(anyhow!("browser helper timed out waiting for response")); + } + // Blocking read — browser ops are infrequent; keep it simple. + line.clear(); + let n = stdout + .read_line(&mut line) + .context("read helper stdout")?; + if n == 0 { + return Err(anyhow!("browser helper exited unexpectedly")); + } + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + let v: Value = serde_json::from_str(trimmed) + .with_context(|| format!("helper returned non-JSON: {trimmed}"))?; + return Ok(v); + } +} + +/// Send one JSON request to the helper; returns the parsed response object. +pub fn browser_request(cwd: &Path, mut req: Value) -> Result { + let mut guard = helper_lock() + .lock() + .map_err(|_| anyhow!("browser helper lock poisoned"))?; + + let (stdin, stdout, next_id) = ensure_running(&mut guard, cwd)?; + let id = *next_id; + *next_id += 1; + if req.get("id").is_none() { + req["id"] = json!(id.to_string()); + } + + let line = serde_json::to_string(&req)? + "\n"; + stdin + .write_all(line.as_bytes()) + .context("write to browser helper")?; + stdin.flush().context("flush browser helper")?; + + let deadline = Instant::now() + HELPER_TIMEOUT; + let resp = read_json_line(stdout, deadline)?; + + // If helper reported permanent missing playwright, mark unavailable for clearer retries. + if resp.get("ok") == Some(&json!(false)) { + if let Some(code) = resp.get("code").and_then(|c| c.as_str()) { + if code == "PLAYWRIGHT_MISSING" || code == "CHROMIUM_MISSING" { + // Keep process; user may install mid-session — don't mark Unavailable. + } + } + } + + Ok(resp) +} + +/// One-shot `--cmd` invocation (no daemon). Useful for offline protocol tests. +pub fn browser_oneshot(cwd: &Path, req: &Value) -> Result { + let script = find_helper_script(cwd).ok_or_else(|| { + anyhow!("{INSTALL_HINT} (helper not found)") + })?; + let node = which_node().ok_or_else(|| anyhow!("{INSTALL_HINT} (node not found)"))?; + let cmd_json = serde_json::to_string(req)?; + let output = Command::new(node) + .arg(&script) + .arg("--cmd") + .arg(&cmd_json) + .current_dir(cwd) + .output() + .context("oneshot browser helper")?; + let stdout = String::from_utf8_lossy(&output.stdout); + let line = stdout + .lines() + .rev() + .find(|l| !l.trim().is_empty()) + .unwrap_or(""); + if line.is_empty() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(anyhow!( + "browser helper produced no JSON (stderr: {})", + stderr.trim() + )); + } + let v: Value = serde_json::from_str(line) + .with_context(|| format!("oneshot non-JSON: {line}"))?; + Ok(v) +} + +/// Run helper `--self-test` (no Chromium). Returns parsed summary JSON. +pub fn browser_self_test(cwd: &Path) -> Result { + let script = find_helper_script(cwd).ok_or_else(|| { + anyhow!("{INSTALL_HINT} (helper not found)") + })?; + let node = which_node().ok_or_else(|| anyhow!("{INSTALL_HINT} (node not found)"))?; + let output = Command::new(node) + .arg(&script) + .arg("--self-test") + .current_dir( + script + .parent() + .unwrap_or(cwd), + ) + .output() + .context("browser helper --self-test")?; + let stdout = String::from_utf8_lossy(&output.stdout); + let line = stdout + .lines() + .rev() + .find(|l| !l.trim().is_empty()) + .unwrap_or(""); + if line.is_empty() { + return Err(anyhow!( + "self-test empty stdout; stderr={}", + String::from_utf8_lossy(&output.stderr) + )); + } + let v: Value = serde_json::from_str(line)?; + if !output.status.success() || v.get("ok") != Some(&json!(true)) { + return Err(anyhow!("self-test failed: {v}")); + } + Ok(v) +} + +pub fn update_last_url(slot: &LastUrlSlot, resp: &Value) { + if let Some(url) = resp.get("url").and_then(|u| u.as_str()) { + if let Ok(mut g) = slot.lock() { + *g = Some(url.to_string()); + } + } +} + +pub fn response_to_tool_json(resp: Value) -> Value { + if resp.get("ok") == Some(&json!(true)) { + let mut out = resp; + if let Some(obj) = out.as_object_mut() { + obj.remove("ok"); + obj.remove("id"); + } + out + } else { + let err = resp + .get("error") + .and_then(|e| e.as_str()) + .unwrap_or("browser tool failed"); + json!({ + "error": err, + "blocked": true, + "install_hint": INSTALL_HINT, + "detail": resp, + }) + } +} + +/// OpenAI tool definitions for browser ops (always registered; fail closed if missing). +pub fn browser_tool_definitions() -> Vec { + vec![ + json!({ + "type": "function", + "function": { + "name": "browser_navigate", + "description": "Navigate the optional Playwright browser to a URL (DOM path). Fails closed with install hint if Playwright is missing.", + "parameters": { + "type": "object", + "properties": { + "url": { "type": "string", "description": "URL to open" } + }, + "required": ["url"] + } + } + }), + json!({ + "type": "function", + "function": { + "name": "browser_snapshot", + "description": "Structured text snapshot of the page (roles/names/CSS selectors). Prefer this over screenshots for control.", + "parameters": { + "type": "object", + "properties": {}, + "additionalProperties": false + } + } + }), + json!({ + "type": "function", + "function": { + "name": "browser_click", + "description": "Click an element by CSS selector or role/name (Playwright DOM). Not pixel/XY click.", + "parameters": { + "type": "object", + "properties": { + "selector": { "type": "string", "description": "CSS selector" }, + "role": { "type": "string", "description": "ARIA role, e.g. button, link" }, + "name": { "type": "string", "description": "Accessible name when using role" }, + "text": { "type": "string", "description": "Visible text alternative" }, + "label": { "type": "string" }, + "placeholder": { "type": "string" } + } + } + } + }), + json!({ + "type": "function", + "function": { + "name": "browser_type", + "description": "Type/fill text into an element by CSS selector or role/name.", + "parameters": { + "type": "object", + "properties": { + "text": { "type": "string", "description": "Text to type" }, + "selector": { "type": "string" }, + "role": { "type": "string" }, + "name": { "type": "string" }, + "label": { "type": "string" }, + "placeholder": { "type": "string" }, + "clear": { "type": "boolean", "description": "Fill (clear first) vs append; default true" } + }, + "required": ["text"] + } + } + }), + json!({ + "type": "function", + "function": { + "name": "browser_eval", + "description": "Evaluate a JavaScript expression in the page context (use sparingly; prefer snapshot/click/type).", + "parameters": { + "type": "object", + "properties": { + "expression": { "type": "string", "description": "JS expression to eval in page" } + }, + "required": ["expression"] + } + } + }), + ] +} + +pub fn is_browser_tool(name: &str) -> bool { + matches!( + name, + "browser_navigate" + | "browser_snapshot" + | "browser_dom" + | "browser_click" + | "browser_type" + | "browser_eval" + ) +} + +pub async fn execute_browser_tool( + cwd: &Path, + last_url: &LastUrlSlot, + name: &str, + args: &Value, +) -> Result { + // Run blocking helper I/O off the async runtime. + let cwd = cwd.to_path_buf(); + let name = name.to_string(); + let args = args.clone(); + let last_url = last_url.clone(); + + tokio::task::spawn_blocking(move || { + let op = match name.as_str() { + "browser_navigate" => "navigate", + "browser_snapshot" | "browser_dom" => "snapshot", + "browser_click" => "click", + "browser_type" => "type", + "browser_eval" => "eval", + other => return Err(anyhow!("unknown browser tool: {other}")), + }; + + let mut req = args; + if let Some(obj) = req.as_object_mut() { + obj.insert("op".into(), json!(op)); + } else { + req = json!({ "op": op }); + } + + // Normalize eval field + if op == "eval" { + if req.get("expression").is_none() { + if let Some(js) = req.get("js").cloned() { + req["expression"] = js; + } + } + } + + let resp = browser_request(&cwd, req)?; + update_last_url(&last_url, &resp); + Ok(response_to_tool_json(resp)) + }) + .await + .map_err(|e| anyhow!("browser task join: {e}"))? +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn repo_cwd() -> PathBuf { + // Crate manifest dir is crates/grokboy-core → repo root is ../.. + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .unwrap_or_else(|_| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..")) + } + + #[test] + fn finds_helper_from_repo_root() { + let root = repo_cwd(); + let helper = find_helper_script(&root); + assert!( + helper.is_some(), + "expected helper under {}/{HELPER_REL}", + root.display() + ); + } + + #[test] + fn helper_self_test_protocol() { + let root = repo_cwd(); + // Skip soft if node missing (should still usually be present on Mac). + if which_node().is_none() { + eprintln!("skip: node not on PATH"); + return; + } + let summary = browser_self_test(&root).expect("self-test"); + assert_eq!(summary["ok"], true); + } + + #[test] + fn oneshot_ping_json_protocol() { + let root = repo_cwd(); + if which_node().is_none() { + eprintln!("skip: node not on PATH"); + return; + } + let resp = browser_oneshot(&root, &json!({"op": "ping", "id": "u1"})).unwrap(); + assert_eq!(resp["ok"], true); + assert_eq!(resp["pong"], true); + assert_eq!(resp["protocol"], 1); + } + + #[test] + fn response_maps_errors_to_blocked() { + let v = response_to_tool_json(json!({ + "ok": false, + "blocked": true, + "error": "nope", + "id": "1" + })); + assert!(v.get("error").is_some()); + assert_eq!(v["blocked"], true); + assert!(v["install_hint"].as_str().unwrap().contains("npm install")); + } + + #[test] + fn browser_defs_count() { + assert_eq!(browser_tool_definitions().len(), 5); + assert!(is_browser_tool("browser_navigate")); + assert!(is_browser_tool("browser_snapshot")); + assert!(!is_browser_tool("shell")); + } + + #[test] + fn install_hint_constant() { + assert!(INSTALL_HINT.contains("npx playwright install chromium")); + } + + #[tokio::test] + async fn execute_ping_via_navigate_missing_url_style() { + // Exercise execute path with type missing text → blocked JSON, no chromium needed + // if helper starts. Use browser_type without text through execute_browser_tool args check. + let root = repo_cwd(); + if which_node().is_none() || find_helper_script(&root).is_none() { + return; + } + let slot: LastUrlSlot = Arc::new(Mutex::new(None)); + // Direct request ping through oneshot already covered; here ensure spawn works with status via request. + let resp = browser_oneshot(&root, &json!({"op":"status"})).unwrap(); + assert_eq!(resp["ok"], true); + let _ = slot; + let _ = SystemTime::now().duration_since(UNIX_EPOCH); + } +} diff --git a/crates/grokboy-core/src/lib.rs b/crates/grokboy-core/src/lib.rs index 5d15197..bbbf59a 100644 --- a/crates/grokboy-core/src/lib.rs +++ b/crates/grokboy-core/src/lib.rs @@ -1,6 +1,7 @@ //! GrokBoy core: config, streaming chat, tools, ReAct agent, sessions. mod agent; +mod browser; mod config; mod model; mod session; @@ -11,6 +12,7 @@ pub use agent::{ context_char_budget, message_char_len, messages_char_len, round_signature, run_agent, run_agent_with, tool_call_signature, truncate_messages, }; +pub use browser::{INSTALL_HINT as BROWSER_INSTALL_HINT, browser_oneshot, browser_self_test, find_helper_script}; pub use config::Config; pub use model::{ChatMessage, FunctionCall, Role, ToolCall, chat_completion, stream_chat}; pub use session::{Session, load_or_create, load_session, save_session, sessions_dir}; diff --git a/crates/grokboy-core/src/session.rs b/crates/grokboy-core/src/session.rs index f09e90f..b4e7b9d 100644 --- a/crates/grokboy-core/src/session.rs +++ b/crates/grokboy-core/src/session.rs @@ -15,6 +15,9 @@ pub struct Session { pub updated_at: DateTime, pub cwd: PathBuf, pub messages: Vec, + /// Last page URL from optional Playwright browser tools (P3). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_browser_url: Option, } impl Session { @@ -26,6 +29,7 @@ impl Session { updated_at: now, cwd: cwd.into(), messages: Vec::new(), + last_browser_url: None, } } diff --git a/crates/grokboy-core/src/tools.rs b/crates/grokboy-core/src/tools.rs index b271d48..713a49a 100644 --- a/crates/grokboy-core/src/tools.rs +++ b/crates/grokboy-core/src/tools.rs @@ -1,4 +1,4 @@ -//! Built-in tools: shell, list_dir, read_file, write_file, report_done, report_blocked. +//! Built-in tools: shell, files, completion, and optional Playwright browser (P3). use anyhow::{Context, Result, anyhow}; use serde_json::{Value, json}; @@ -7,6 +7,8 @@ use std::process::Stdio; use std::time::Duration; use tokio::process::Command; +use crate::browser::{self, LastUrlSlot}; + pub const MAX_READ_BYTES: usize = 256 * 1024; pub const SHELL_TIMEOUT_SECS: u64 = 30; @@ -17,6 +19,8 @@ pub struct ToolContext { pub cwd: PathBuf, /// Optional workspace root; paths outside it are rejected when set. pub workspace_root: Option, + /// Last navigated browser URL (shared across clones). + pub last_browser_url: LastUrlSlot, } impl ToolContext { @@ -25,6 +29,7 @@ impl ToolContext { Self { cwd: cwd.clone(), workspace_root: Some(cwd), + last_browser_url: std::sync::Arc::new(std::sync::Mutex::new(None)), } } @@ -32,11 +37,15 @@ impl ToolContext { self.workspace_root = root; self } + + pub fn last_browser_url_value(&self) -> Option { + self.last_browser_url.lock().ok().and_then(|g| g.clone()) + } } /// OpenAI-compatible tool definitions for chat completions. pub fn tool_definitions() -> Value { - json!([ + let mut defs = json!([ { "type": "function", "function": { @@ -123,7 +132,12 @@ pub fn tool_definitions() -> Value { } } } - ]) + ]); + // Optional Playwright DOM tools (always registered; fail closed with install hint if missing). + if let Some(arr) = defs.as_array_mut() { + arr.extend(browser::browser_tool_definitions()); + } + defs } /// True if the tool name ends the ReAct loop (completion contract). @@ -191,6 +205,9 @@ async fn execute_tool_inner(ctx: &ToolContext, name: &str, arguments_json: &str) "write_file" => tool_write_file(ctx, &args).await, "report_done" => tool_report_done(&args), "report_blocked" => tool_report_blocked(&args), + name if browser::is_browser_tool(name) => { + browser::execute_browser_tool(&ctx.cwd, &ctx.last_browser_url, name, &args).await + } other => Err(anyhow!("unknown tool: {other}")), } } @@ -476,10 +493,10 @@ mod tests { } #[test] - fn tool_defs_include_six() { + fn tool_defs_include_core_and_browser() { let defs = tool_definitions(); let arr = defs.as_array().unwrap(); - assert_eq!(arr.len(), 6); + assert_eq!(arr.len(), 11); // 6 core + 5 browser let names: Vec<&str> = arr .iter() .map(|t| t["function"]["name"].as_str().unwrap()) @@ -490,5 +507,33 @@ mod tests { assert!(names.contains(&"write_file")); assert!(names.contains(&"report_done")); assert!(names.contains(&"report_blocked")); + assert!(names.contains(&"browser_navigate")); + assert!(names.contains(&"browser_snapshot")); + assert!(names.contains(&"browser_click")); + assert!(names.contains(&"browser_type")); + assert!(names.contains(&"browser_eval")); + } + + #[tokio::test] + async fn browser_tool_fails_closed_without_chromium_ok_with_helper() { + // Missing required args should fail closed via helper protocol (no Chromium needed). + let (ctx, dir) = temp_ctx(); + // Point cwd at repo root so helper is found when running under cargo test. + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let root = root.canonicalize().unwrap_or(root); + let ctx = ToolContext { + cwd: root.clone(), + workspace_root: ctx.workspace_root, + last_browser_url: ctx.last_browser_url, + }; + let out = execute_tool( + &ctx, + "browser_type", + &json!({"selector": "#x"}).to_string(), + ) + .await; + let v: Value = serde_json::from_str(&out).unwrap(); + assert!(v.get("error").is_some(), "{v}"); + let _ = std::fs::remove_dir_all(&dir); } } diff --git a/crates/grokboy/src/main.rs b/crates/grokboy/src/main.rs index 8b68408..aecca8f 100644 --- a/crates/grokboy/src/main.rs +++ b/crates/grokboy/src/main.rs @@ -6,6 +6,7 @@ use grokboy_core::{ }; use serde_json::json; use std::io::{self, Write}; +use std::path::PathBuf; use std::process::ExitCode; const CHAT_SYSTEM: &str = "You are GrokBoy, a concise local coding assistant. Prefer clear, short answers. Traditional Chinese is welcome when the user writes in Chinese."; @@ -49,7 +50,7 @@ async fn run() -> Result<()> { fn print_help() { println!( "\ -GrokBoy — minimal local CLI agent (P2: completion + loop guard + truncation) +GrokBoy — minimal local CLI agent (P3: optional Playwright DOM browser) USAGE: grokboy chat Interactive streaming chat (no tools) @@ -65,8 +66,10 @@ ENV: GROKBOY_MODEL default grok-4.6 GROKBOY_CONTEXT_CHARS context budget (default 100000) -Tools: shell, list_dir, read_file, write_file, report_done, report_blocked +Tools: shell, list_dir, read_file, write_file, report_done, report_blocked, + browser_navigate, browser_snapshot, browser_click, browser_type, browser_eval Sessions: ~/.grokboy/sessions/.json +Browser (optional): cd tools/playwright && npm i && npx playwright install chromium " ); } @@ -159,6 +162,9 @@ async fn cmd_run(args: &[String]) -> Result<()> { ) .await?; + if let Some(url) = tool_ctx.last_browser_url_value() { + session.last_browser_url = Some(url); + } session.touch(); let path = save_session(&session)?; println!("{}", verdict.message()); @@ -176,7 +182,7 @@ async fn cmd_run(args: &[String]) -> Result<()> { } async fn cmd_smoke() -> Result<()> { - println!("GrokBoy smoke (offline P2)…"); + println!("GrokBoy smoke (offline P3)…"); let stamp = uuid_like(); let dir = std::env::temp_dir().join(format!("grokboy-smoke-{stamp}")); std::fs::create_dir_all(&dir).context("temp dir")?; @@ -275,14 +281,39 @@ async fn cmd_smoke() -> Result<()> { } println!(" completion ok"); - // tool definitions present (6) + // tool definitions present (6 core + 5 browser) let defs = tool_definitions(); let n_tools = defs.as_array().map(|a| a.len()).unwrap_or(0); - if n_tools != 6 { - return Err(anyhow!("expected 6 tool defs, got {n_tools}")); + if n_tools != 11 { + return Err(anyhow!("expected 11 tool defs, got {n_tools}")); } println!(" tool defs ok"); + // browser helper protocol (no Chromium required) + let repo = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); + match grokboy_core::browser_self_test(&repo) { + Ok(_) => println!(" browser proto ok"), + Err(e) => { + // Soft: smoke still passes if helper missing in weird cwd; warn. + println!(" browser proto skip ({e:#})"); + } + } + // Fail-closed install hint when navigating without ensuring chromium — oneshot missing url already covered in helper self-test. + let nav = execute_tool( + &ToolContext::new(repo.clone()), + "browser_navigate", + &json!({"url": ""}).to_string(), + ) + .await; + let nav_v: serde_json::Value = serde_json::from_str(&nav)?; + // empty url should error from helper; if helper missing, also error with install hint + if nav_v.get("error").is_none() && nav_v.get("blocked").is_none() { + // empty string might still "succeed" oddly — accept either error or blocked + println!(" browser fail note: {nav_v}"); + } else { + println!(" browser fail ok"); + } + // context truncation let mut msgs = vec![ ChatMessage::system(AGENT_SYSTEM), diff --git a/docs/ACCEPTANCE.md b/docs/ACCEPTANCE.md index 12664c0..7718529 100644 --- a/docs/ACCEPTANCE.md +++ b/docs/ACCEPTANCE.md @@ -21,4 +21,8 @@ - [x] Offline smoke/tests cover P2 without API key ## P3 — browser -- [ ] Playwright DOM path (no screenshot-first) +- [x] Playwright DOM path (no screenshot-first) +- [x] Tools: `browser_navigate`, `browser_snapshot`, `browser_click`, `browser_type`, `browser_eval` +- [x] Thin Node helper under `tools/playwright/` (JSONL / one-shot JSON) +- [x] Fail closed with install hint when Node/Playwright/Chromium missing +- [x] `cargo test` / `grokboy smoke` pass without Playwright browsers installed diff --git a/tools/playwright/README.md b/tools/playwright/README.md new file mode 100644 index 0000000..1214eb0 --- /dev/null +++ b/tools/playwright/README.md @@ -0,0 +1,21 @@ +# GrokBoy Playwright helper (optional) + +Thin Node script used by Rust `browser_*` tools. + +## Install + +```bash +cd tools/playwright +npm install +npx playwright install chromium +``` + +## Protocol + +- **JSONL daemon** (default): one JSON request per stdin line → one JSON response on stdout. +- **One-shot**: `node browser_helper.mjs --cmd '{"op":"ping"}'` +- **Self-test** (no Chromium): `npm run self-test` + +Ops: `ping`, `navigate`, `snapshot`/`dom`, `click`, `type`, `eval`, `close`, `status`. + +Prefer CSS selector or `role`+`name` over screenshots. diff --git a/tools/playwright/browser_helper.mjs b/tools/playwright/browser_helper.mjs new file mode 100755 index 0000000..0e2a79c --- /dev/null +++ b/tools/playwright/browser_helper.mjs @@ -0,0 +1,453 @@ +#!/usr/bin/env node +/** + * GrokBoy Playwright DOM helper (optional). + * + * Modes: + * JSONL daemon (default when no --cmd): read JSON lines from stdin, write JSON lines to stdout. + * One-shot: node browser_helper.mjs --cmd '' + * Self-test: node browser_helper.mjs --self-test (no Chromium required) + * + * Ops: ping, navigate, snapshot, click, type, eval, close, status + * Prefer CSS selector or role+name. Screenshots are NOT the primary control surface. + */ + +import { createInterface } from "node:readline"; +import { dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +const INSTALL_HINT = + "Playwright not available. From the GrokBoy repo run: " + + "cd tools/playwright && npm install && npx playwright install chromium"; + +let browser = null; +let context = null; +let page = null; +let lastUrl = null; + +function ok(id, extra = {}) { + return { id: id ?? null, ok: true, ...extra }; +} + +function fail(id, error, extra = {}) { + return { + id: id ?? null, + ok: false, + blocked: true, + error: String(error), + ...extra, + }; +} + +async function loadPlaywright() { + try { + return await import("playwright"); + } catch (e) { + const err = new Error(`${INSTALL_HINT} (import failed: ${e.message})`); + err.code = "PLAYWRIGHT_MISSING"; + throw err; + } +} + +async function ensurePage() { + if (page) return page; + const { chromium } = await loadPlaywright(); + try { + browser = await chromium.launch({ headless: true }); + } catch (e) { + const err = new Error( + `${INSTALL_HINT} (chromium launch failed: ${e.message})` + ); + err.code = "CHROMIUM_MISSING"; + throw err; + } + context = await browser.newContext(); + page = await context.newPage(); + return page; +} + +/** Accessibility-ish text snapshot: roles, names, and useful selectors — not pixels. */ +async function buildSnapshot(p) { + const nodes = await p.evaluate(() => { + const out = []; + const max = 400; + const interesting = new Set([ + "a", + "button", + "input", + "textarea", + "select", + "option", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "label", + "li", + "summary", + "nav", + "main", + "form", + "img", + "table", + "th", + "td", + ]); + + function cssPath(el) { + if (el.id) return `#${CSS.escape(el.id)}`; + const parts = []; + let cur = el; + while (cur && cur.nodeType === 1 && parts.length < 5) { + let part = cur.tagName.toLowerCase(); + if (cur.id) { + parts.unshift(`#${CSS.escape(cur.id)}`); + break; + } + const parent = cur.parentElement; + if (parent) { + const siblings = [...parent.children].filter( + (c) => c.tagName === cur.tagName + ); + if (siblings.length > 1) { + const idx = siblings.indexOf(cur) + 1; + part += `:nth-of-type(${idx})`; + } + } + parts.unshift(part); + cur = parent; + } + return parts.join(" > "); + } + + const walker = document.createTreeWalker( + document.body || document.documentElement, + NodeFilter.SHOW_ELEMENT + ); + let node = walker.currentNode; + while (node && out.length < max) { + const tag = node.tagName ? node.tagName.toLowerCase() : ""; + const role = + node.getAttribute("role") || + (tag === "a" + ? "link" + : tag === "button" + ? "button" + : tag === "input" + ? node.getAttribute("type") === "submit" + ? "button" + : "textbox" + : tag === "textarea" + ? "textbox" + : tag === "img" + ? "img" + : tag.startsWith("h") && tag.length === 2 + ? "heading" + : null); + const name = + node.getAttribute("aria-label") || + node.getAttribute("alt") || + node.getAttribute("placeholder") || + node.getAttribute("title") || + (node.innerText || node.textContent || "").trim().slice(0, 120); + const href = node.getAttribute && node.getAttribute("href"); + const type = node.getAttribute && node.getAttribute("type"); + const value = + "value" in node && typeof node.value === "string" + ? String(node.value).slice(0, 80) + : null; + + if (interesting.has(tag) || node.getAttribute("role") || node.id) { + const entry = { + tag, + role: role || tag, + name: name || "", + selector: cssPath(node), + }; + if (href) entry.href = href; + if (type) entry.type = type; + if (value) entry.value = value; + if (node.id) entry.id = node.id; + out.push(entry); + } + node = walker.nextNode(); + } + return out; + }); + + const title = await p.title(); + const url = p.url(); + const lines = nodes.map((n, i) => { + const bits = [`[${i}]`, n.role || n.tag]; + if (n.name) bits.push(`name=${JSON.stringify(n.name)}`); + if (n.selector) bits.push(`sel=${n.selector}`); + if (n.href) bits.push(`href=${n.href}`); + if (n.type) bits.push(`type=${n.type}`); + if (n.value) bits.push(`value=${JSON.stringify(n.value)}`); + return bits.join(" "); + }); + + return { + url, + title, + count: nodes.length, + text: lines.join("\n"), + nodes, + }; +} + +async function resolveLocator(p, args) { + if (args.selector) { + return p.locator(String(args.selector)).first(); + } + if (args.role) { + const opts = {}; + if (args.name) opts.name = String(args.name); + return p.getByRole(String(args.role), opts).first(); + } + if (args.text) { + return p.getByText(String(args.text), { exact: !!args.exact }).first(); + } + if (args.label) { + return p.getByLabel(String(args.label)).first(); + } + if (args.placeholder) { + return p.getByPlaceholder(String(args.placeholder)).first(); + } + throw new Error( + "click/type requires selector, role(+name), text, label, or placeholder" + ); +} + +async function handle(req) { + const id = req.id ?? null; + const op = req.op || req.command; + if (!op) return fail(id, "missing op"); + + try { + switch (op) { + case "ping": + return ok(id, { + pong: true, + protocol: 1, + ops: [ + "ping", + "navigate", + "snapshot", + "click", + "type", + "eval", + "close", + "status", + ], + }); + + case "status": { + return ok(id, { + browser_open: !!browser, + last_url: lastUrl, + page_url: page ? page.url() : null, + }); + } + + case "navigate": { + const url = req.url; + if (!url) return fail(id, "navigate: missing url"); + const p = await ensurePage(); + const resp = await p.goto(String(url), { + waitUntil: "domcontentloaded", + timeout: req.timeout_ms ?? 30000, + }); + lastUrl = p.url(); + return ok(id, { + url: lastUrl, + title: await p.title(), + status: resp ? resp.status() : null, + }); + } + + case "snapshot": + case "dom": { + const p = await ensurePage(); + const snap = await buildSnapshot(p); + lastUrl = snap.url; + return ok(id, { + url: snap.url, + title: snap.title, + count: snap.count, + snapshot: snap.text, + nodes: snap.nodes.slice(0, 200), + }); + } + + case "click": { + const p = await ensurePage(); + const loc = await resolveLocator(p, req); + await loc.click({ timeout: req.timeout_ms ?? 10000 }); + lastUrl = p.url(); + return ok(id, { clicked: true, url: lastUrl }); + } + + case "type": { + const p = await ensurePage(); + const text = req.text ?? req.value; + if (text == null) return fail(id, "type: missing text"); + const loc = await resolveLocator(p, req); + if (req.clear !== false) { + await loc.fill(String(text), { timeout: req.timeout_ms ?? 10000 }); + } else { + await loc.type(String(text), { timeout: req.timeout_ms ?? 10000 }); + } + lastUrl = p.url(); + return ok(id, { typed: true, url: lastUrl }); + } + + case "eval": { + const p = await ensurePage(); + const expression = req.expression ?? req.js ?? req.code; + if (!expression) return fail(id, "eval: missing expression"); + // Evaluate as expression body; fail closed on throw. + const result = await p.evaluate((expr) => { + // eslint-disable-next-line no-eval + return eval(expr); + }, String(expression)); + let serialized; + try { + serialized = JSON.parse(JSON.stringify(result)); + } catch { + serialized = String(result); + } + return ok(id, { result: serialized, url: p.url() }); + } + + case "close": { + if (browser) { + await browser.close().catch(() => {}); + } + browser = null; + context = null; + page = null; + return ok(id, { closed: true, last_url: lastUrl }); + } + + default: + return fail(id, `unknown op: ${op}`); + } + } catch (e) { + const extra = {}; + if (e && e.code) extra.code = e.code; + return fail(id, e.message || e, extra); + } +} + +function parseArgs(argv) { + const out = { cmd: null, selfTest: false, daemon: true }; + for (let i = 2; i < argv.length; i++) { + const a = argv[i]; + if (a === "--self-test") out.selfTest = true; + else if (a === "--cmd" || a === "-c") { + out.cmd = argv[++i]; + out.daemon = false; + } else if (a === "--daemon") out.daemon = true; + else if (a === "--help" || a === "-h") out.help = true; + } + return out; +} + +async function runSelfTest() { + const cases = []; + + // Protocol: ping without playwright + let r = await handle({ id: "t1", op: "ping" }); + cases.push(["ping", r.ok === true && r.pong === true]); + + // Missing fields fail closed + r = await handle({ id: "t2", op: "navigate" }); + cases.push(["navigate_missing_url", r.ok === false && r.blocked === true]); + + r = await handle({ id: "t3", op: "type", selector: "#x" }); + // ensurePage may fail if no playwright — either blocked missing text first + // type checks text before ensurePage... actually type checks text then ensurePage then resolveLocator + // missing text → fail before playwright + cases.push(["type_missing_text", r.ok === false]); + + r = await handle({ id: "t4", op: "nope" }); + cases.push(["unknown_op", r.ok === false]); + + r = await handle({ id: "t5", op: "status" }); + cases.push(["status", r.ok === true && r.browser_open === false]); + + const failed = cases.filter(([, okv]) => !okv); + const summary = { + ok: failed.length === 0, + cases: Object.fromEntries(cases), + failed: failed.map(([n]) => n), + }; + console.log(JSON.stringify(summary)); + process.exit(failed.length === 0 ? 0 : 1); +} + +async function main() { + const args = parseArgs(process.argv); + if (args.help) { + console.log( + JSON.stringify({ + ok: true, + usage: + "browser_helper.mjs [--self-test] | [--cmd JSON] | (JSONL on stdin)", + install: INSTALL_HINT, + }) + ); + return; + } + if (args.selfTest) { + await runSelfTest(); + return; + } + if (args.cmd) { + let req; + try { + req = JSON.parse(args.cmd); + } catch (e) { + console.log(JSON.stringify(fail(null, `invalid --cmd JSON: ${e.message}`))); + process.exit(1); + return; + } + const res = await handle(req); + console.log(JSON.stringify(res)); + process.exit(res.ok ? 0 : 1); + return; + } + + // JSONL daemon + const rl = createInterface({ input: process.stdin, crlfDelay: Infinity }); + for await (const line of rl) { + const trimmed = line.trim(); + if (!trimmed) continue; + let req; + try { + req = JSON.parse(trimmed); + } catch (e) { + console.log( + JSON.stringify(fail(null, `invalid JSON line: ${e.message}`)) + ); + continue; + } + const res = await handle(req); + console.log(JSON.stringify(res)); + if (req.op === "close" || req.command === "close") { + // keep process alive unless parent closes stdin + } + } + if (browser) { + await browser.close().catch(() => {}); + } +} + +main().catch((e) => { + console.error(JSON.stringify(fail(null, e.message || e))); + process.exit(1); +}); diff --git a/tools/playwright/package.json b/tools/playwright/package.json new file mode 100644 index 0000000..37e97c4 --- /dev/null +++ b/tools/playwright/package.json @@ -0,0 +1,14 @@ +{ + "name": "grokboy-playwright-helper", + "version": "0.1.0", + "private": true, + "description": "Optional Playwright DOM helper for GrokBoy (JSONL / one-shot CLI)", + "type": "module", + "main": "browser_helper.mjs", + "scripts": { + "self-test": "node browser_helper.mjs --self-test" + }, + "dependencies": { + "playwright": "^1.49.0" + } +}