#!/usr/bin/env node /** * GrokBoy Playwright DOM helper (optional). * * Modes: * JSONL daemon (default when no --cmd): read JSON lines from stdin, write JSON lines to stdout. * One-shot: node browser_helper.mjs --cmd '' * Self-test: node browser_helper.mjs --self-test (no Chromium required) * * Ops: ping, navigate, snapshot, click, type, eval, close, status, handoff_prepare * Prefer CSS selector or role+name. Screenshots are NOT the primary control surface. */ import { createInterface } from "node:readline"; import { dirname, resolve, relative, isAbsolute } from "node:path"; import { mkdir, stat, realpath, lstat, readFile, chmod } from "node:fs/promises"; import { fileURLToPath } from "node:url"; const __dirname = dirname(fileURLToPath(import.meta.url)); const INSTALL_HINT = "Playwright not available. From the GrokBoy repo run: " + "cd tools/playwright && npm install && npx playwright install chromium"; let browser = null; let context = null; let page = null; let lastUrl = null; /** Whether the current browser was launched headed (visible). */ let browserHeaded = false; let nextTab = 0; const tabs = new Map(); let handoffStorage; function registerTab(p) { if ([...tabs.values()].includes(p)) return; tabs.set(String(++nextTab), p); p.on("close", () => { for (const [id,t] of tabs) if(t===p) tabs.delete(id); if(page===p) page=[...tabs.values()][0] || null; }); } async function target(req) { await ensurePage(); if (req.tab_id) { const tab=tabs.get(String(req.tab_id)); if(!tab || tab.isClosed()) throw new Error("unknown tab_id; list tabs again"); page=tab; } if(req.frame) { const el=await page.locator(String(req.frame)).elementHandle(); const frame=await el?.contentFrame(); if(!frame) throw new Error("frame not found; obtain a new snapshot"); return frame; } return page; } function envHeadedDefault() { const v = (process.env.GROKBOY_BROWSER_HEADED || "").trim().toLowerCase(); if (v === "1" || v === "true" || v === "yes") return true; if (v === "0" || v === "false" || v === "no") return false; return false; } function wantHeaded(forceHeaded) { if (forceHeaded === true) return true; return envHeadedDefault(); } function ok(id, extra = {}) { return { id: id ?? null, ok: true, ...extra }; } function fail(id, error, extra = {}) { return { id: id ?? null, ok: false, blocked: true, error: String(error), ...extra, }; } async function loadPlaywright() { try { return await import("playwright"); } catch (e) { const err = new Error(`${INSTALL_HINT} (import failed: ${e.message})`); err.code = "PLAYWRIGHT_MISSING"; throw err; } } async function launchBrowser(headed) { const { chromium } = await loadPlaywright(); try { if (process.env.GROKBOY_BROWSER_PROFILE) { await mkdir(process.env.GROKBOY_BROWSER_PROFILE,{recursive:true,mode:0o700}); context=await chromium.launchPersistentContext(process.env.GROKBOY_BROWSER_PROFILE, { headless:!headed, acceptDownloads:true }); // Chromium does not restore session cookies on a clean restart; retain them per agent session. const stored=await readFile(resolve(process.env.GROKBOY_BROWSER_PROFILE,"grokboy-state.json"),"utf8").then(JSON.parse).catch(()=>null); if(stored?.cookies?.length) await context.addCookies(stored.cookies); browser=context.browser(); } else { browser=await chromium.launch({headless:!headed}); context=await browser.newContext({acceptDownloads:true, ...(handoffStorage ? {storageState:handoffStorage} : {})}); } browserHeaded = headed; } catch (e) { const err = new Error( `${INSTALL_HINT} (chromium launch failed: ${e.message})` ); err.code = "CHROMIUM_MISSING"; throw err; } tabs.clear(); context.on("page",registerTab); for (const p of context.pages()) registerTab(p); page = context.pages()[0] || await context.newPage(); registerTab(page); return page; } async function ensurePage(opts = {}) { const headed = wantHeaded(opts.headed === true); if (page) { // Already open; if caller needs headed and we are headless, relaunch below via handoff_prepare. return page; } return launchBrowser(headed); } /** Close current browser and reopen headed, restoring lastUrl when possible. */ async function ensureHeadedForHandoff() { const restore = (page && !page.isClosed() ? page.url() : null) || lastUrl; if (browser && browserHeaded && page) { try { await page.bringToFront(); } catch { /* ignore */ } return { prepared: true, headed: true, relaunched: false, url: page.url(), restored: false, }; } const previousTabs = context ? await Promise.all(context.pages().filter(p=>!p.isClosed()).map(async p=>({ url:p.url(), active:p===page, storage:await p.evaluate(()=>({origin:location.origin,values:Object.fromEntries(Object.entries(sessionStorage))})).catch(()=>null), }))) : []; if (context) { handoffStorage=await context.storageState().catch(()=>undefined); await persistContext(); await context.close().catch(() => {}); if (browser) await browser.close().catch(() => {}); browser = null; context = null; page = null; browserHeaded = false; } await launchBrowser(true); let restored = false; if (previousTabs.length) { const failures=[]; let activePage=page; for (const [index,tab] of previousTabs.entries()) { const reopened=index===0 ? page : await context.newPage(); const cdp=tab.storage ? await context.newCDPSession(reopened) : null; let restoreScript; try { if(cdp) await cdp.send("Page.enable"); if(cdp) restoreScript=await cdp.send("Page.addScriptToEvaluateOnNewDocument",{source: `(({origin,values})=>{if(location.origin===origin)for(const [key,value] of Object.entries(values))sessionStorage.setItem(key,value);})(${JSON.stringify(tab.storage)})`}); if(tab.url && tab.url!=="about:blank") { await reopened.goto(tab.url,{waitUntil:"domcontentloaded",timeout:30000}).catch(e=>failures.push(String(e.message))); } } finally { if(restoreScript) await cdp.send("Page.removeScriptToEvaluateOnNewDocument",{identifier:restoreScript.identifier}); if(cdp) await cdp.detach(); } if(tab.active) activePage=reopened; } page=activePage; lastUrl=page.url(); await page.bringToFront().catch(()=>{}); return {prepared:true,headed:true,relaunched:true,url:lastUrl,restored:failures.length===0, tabs_restored:previousTabs.length,...(failures.length ? {restore_errors:failures} : {})}; } if (restore && restore !== "about:blank") { try { await page.goto(String(restore), { waitUntil: "domcontentloaded", timeout: 30000, }); lastUrl = page.url(); restored = true; } catch (e) { // Keep headed page even if restore fails; caller still sees visible window. lastUrl = page.url(); return { prepared: true, headed: true, relaunched: true, url: lastUrl, restored: false, restore_error: String(e.message || e), }; } } try { await page.bringToFront(); } catch { /* ignore */ } return { prepared: true, headed: true, relaunched: true, url: page ? page.url() : null, restored, }; } /** Accessibility-ish text snapshot: roles, names, and useful selectors — not pixels. */ async function buildSnapshot(p) { const nodes = await p.evaluate(() => { const out = []; const max = 400; const interesting = new Set([ "a", "button", "input", "textarea", "select", "option", "h1", "h2", "h3", "h4", "h5", "h6", "label", "li", "summary", "nav", "main", "form", "img", "table", "th", "td", ]); function cssPath(el) { if (el.id) return `#${CSS.escape(el.id)}`; const parts = []; let cur = el; while (cur && cur.nodeType === 1 && parts.length < 5) { let part = cur.tagName.toLowerCase(); if (cur.id) { parts.unshift(`#${CSS.escape(cur.id)}`); break; } const parent = cur.parentElement; if (parent) { const siblings = [...parent.children].filter( (c) => c.tagName === cur.tagName ); if (siblings.length > 1) { const idx = siblings.indexOf(cur) + 1; part += `:nth-of-type(${idx})`; } } parts.unshift(part); cur = parent; } return parts.join(" > "); } const walker = document.createTreeWalker( document.body || document.documentElement, NodeFilter.SHOW_ELEMENT ); let node = walker.currentNode; while (node && out.length < max) { const tag = node.tagName ? node.tagName.toLowerCase() : ""; const role = node.getAttribute("role") || (tag === "a" ? "link" : tag === "button" ? "button" : tag === "input" ? node.getAttribute("type") === "submit" ? "button" : "textbox" : tag === "textarea" ? "textbox" : tag === "img" ? "img" : tag.startsWith("h") && tag.length === 2 ? "heading" : null); const name = node.getAttribute("aria-label") || node.getAttribute("alt") || node.getAttribute("placeholder") || node.getAttribute("title") || (node.innerText || node.textContent || "").trim().slice(0, 120); const href = node.getAttribute && node.getAttribute("href"); const type = node.getAttribute && node.getAttribute("type"); const value = type !== "password" && "value" in node && typeof node.value === "string" ? String(node.value).slice(0, 80) : null; if (interesting.has(tag) || node.getAttribute("role") || node.id) { const entry = { tag, role: role || tag, name: name || "", selector: cssPath(node), }; if (href) entry.href = href; if (type) entry.type = type; if (value) entry.value = value; if (node.id) entry.id = node.id; out.push(entry); } node = walker.nextNode(); } return out; }); const title = await p.evaluate(() => document.title); const url = p.url(); const lines = nodes.map((n, i) => { const bits = [`[${i}]`, n.role || n.tag]; if (n.name) bits.push(`name=${JSON.stringify(n.name)}`); if (n.selector) bits.push(`sel=${n.selector}`); if (n.href) bits.push(`href=${n.href}`); if (n.type) bits.push(`type=${n.type}`); if (n.value) bits.push(`value=${JSON.stringify(n.value)}`); return bits.join(" "); }); return { url, title, count: nodes.length, text: lines.join("\n"), nodes, frames: await p.locator("iframe").evaluateAll(els=>els.map((el,i)=>({selector:el.id ? `#${CSS.escape(el.id)}` : `iframe:nth-of-type(${i+1})`,src:el.src,name:el.name}))), }; } async function resolveLocator(p, args) { if (args.selector) { return p.locator(String(args.selector)).first(); } if (args.role) { const opts = {}; if (args.name) opts.name = String(args.name); return p.getByRole(String(args.role), opts).first(); } if (args.text) { return p.getByText(String(args.text), { exact: !!args.exact }).first(); } if (args.label) { return p.getByLabel(String(args.label)).first(); } if (args.placeholder) { return p.getByPlaceholder(String(args.placeholder)).first(); } throw new Error( "click/type requires selector, role(+name), text, label, or placeholder" ); } async function handle(req) { const id = req.id ?? null; const op = req.op || req.command; if (!op) return fail(id, "missing op"); try { switch (op) { case "ping": return ok(id, { pong: true, protocol: 1, ops: [ "ping", "navigate", "snapshot", "click", "type", "eval", "close", "status", "handoff_prepare", "read_page", "tabs", "press", "select", "scroll", "wait", "upload", "download", ], }); case "status": { return ok(id, { browser_open: !!browser, headed: browserHeaded, last_url: lastUrl, page_url: page ? page.url() : null, env_headed: envHeadedDefault(), }); } case "handoff_prepare": { // Bring a visible Chromium window forward for human login/OTP/captcha. const info = await ensureHeadedForHandoff(); return ok(id, { ...info, message: "Browser is headed (visible). Complete login/OTP/captcha in the window, then resume in the terminal.", }); } case "navigate": { const url = req.url; if (!url) return fail(id, "navigate: missing url"); const p = await target(req); const resp = await p.goto(String(url), { waitUntil: "domcontentloaded", timeout: req.timeout_ms ?? 30000, }); lastUrl = p.url(); return ok(id, { url: lastUrl, title: await p.title(), status: resp ? resp.status() : null, }); } case "snapshot": case "dom": { const p = await target(req); const snap = await buildSnapshot(p); lastUrl = snap.url; return ok(id, { url: snap.url, title: snap.title, count: snap.count, snapshot: snap.text, nodes: snap.nodes.slice(0, 200), frames: snap.frames, }); } case "click": { if ( !req.selector && !req.role && !req.text && !req.label && !req.placeholder ) { return fail( id, "click requires selector, role(+name), text, label, or placeholder" ); } const p = await target(req); const loc = await resolveLocator(p, req); const popupEvent=page.waitForEvent("popup",{timeout:750}).catch(()=>null); await loc.click({ timeout: req.timeout_ms ?? 10000 }); const popup=await popupEvent; if(popup) { registerTab(popup); await popup.waitForLoadState("domcontentloaded",{timeout:10000}).catch(()=>{}); } lastUrl = p.url(); return ok(id, { clicked: true, url: lastUrl, popup_tab_id:popup?[...tabs].find(([,p])=>p===popup)?.[0]:null }); } case "type": { const text = req.text ?? req.value; if (text == null) return fail(id, "type: missing text"); const p = await target(req); const loc = await resolveLocator(p, req); if (req.clear !== false) { await loc.fill(String(text), { timeout: req.timeout_ms ?? 10000 }); } else { await loc.type(String(text), { timeout: req.timeout_ms ?? 10000 }); } lastUrl = p.url(); return ok(id, { typed: true, url: lastUrl }); } case "read_page": { const p=await target(req); const offset=Math.max(0,Number(req.offset)||0), limit=Math.min(32000,Math.max(1,Number(req.limit)||12000)); const text=await p.locator("body").innerText(); const links=await p.locator("a[href]").evaluateAll(els=>els.slice(0,200).map(e=>({text:(e.innerText||"").slice(0,200),url:e.href}))); const chars=Array.from(text); return ok(id,{url:p.url(),title:await p.evaluate(()=>document.title),text:chars.slice(offset,offset+limit).join(""),offset,next_offset:Math.min(chars.length,offset+limit),truncated:chars.length>offset+limit,links}); } case "tabs": { await ensurePage(); const action=req.action||"list"; if(action==="new") { page=await context.newPage(); registerTab(page); } else if(action==="switch") { await target(req); await page.bringToFront(); } else if(action==="close") { await target(req); await page.close(); } else if(action!=="list") throw new Error("invalid tabs action"); const items=[]; for(const [tab_id,p] of tabs) { items.push({tab_id,url:p.url(),title:await p.title(),active:p===page}); } return ok(id,{tabs:items}); } case "press": { if(!req.key) throw new Error("press requires key"); const p=await target(req); if(req.selector||req.role||req.label||req.placeholder) await (await resolveLocator(p,req)).press(String(req.key)); else await page.keyboard.press(String(req.key)); return ok(id,{pressed:req.key,url:p.url()}); } case "select": { const p=await target(req); if(!Array.isArray(req.values)) throw new Error("select requires values array"); const selected=await (await resolveLocator(p,req)).selectOption(req.values.map(String)); return ok(id,{selected,url:p.url()}); } case "scroll": { const p=await target(req); const delta=Number(req.delta_y)||600; if(req.selector) await (await resolveLocator(p,req)).evaluate((el,y)=>el.scrollBy(0,y),delta); else await p.evaluate(y=>window.scrollBy(0,y),delta); return ok(id,{scrolled:delta,url:p.url()}); } case "wait": { const p=await target(req); const timeout=Math.min(30000,Math.max(1,Number(req.timeout_ms)||10000)); if(req.url) await p.waitForURL(String(req.url),{timeout}); else await (await resolveLocator(p,req)).waitFor({state:req.state||"visible",timeout}); return ok(id,{ready:true,url:p.url()}); } case "upload": { const p=await target(req); const path=await workspacePath(req.path,true); await (await resolveLocator(p,req)).setInputFiles(path); return ok(id,{uploaded:true,path,url:p.url()}); } case "download": { const p=await target(req); const path=await workspacePath(req.path,false); // Attach rejection handling immediately if clicking fails before the event arrives. const downloadPromise=page.waitForEvent("download",{timeout:30000}); downloadPromise.catch(()=>{}); await (await resolveLocator(p,req)).click(); const download=await downloadPromise; await download.saveAs(path); const failure=await download.failure(); if(failure) throw new Error(failure); return ok(id,{downloaded:true,path,bytes:(await stat(path)).size,suggested_filename:download.suggestedFilename(),url:p.url()}); } case "eval": { const p = await target(req); const expression = req.expression ?? req.js ?? req.code; if (!expression) return fail(id, "eval: missing expression"); // Evaluate as expression body; fail closed on throw. const result = await p.evaluate((expr) => { // eslint-disable-next-line no-eval return eval(expr); }, String(expression)); let serialized; try { serialized = JSON.parse(JSON.stringify(result)); } catch { serialized = String(result); } return ok(id, { result: serialized, url: p.url() }); } case "close": { if (context) { await persistContext(); await context.close().catch(() => {}); if (browser) await browser.close().catch(() => {}); } browser = null; context = null; page = null; browserHeaded = false; return ok(id, { closed: true, last_url: lastUrl }); } default: return fail(id, `unknown op: ${op}`); } } catch (e) { const extra = {}; if (e && e.code) extra.code = e.code; return fail(id, e.message || e, extra); } } async function persistContext() { if(context && process.env.GROKBOY_BROWSER_PROFILE) { const path=resolve(process.env.GROKBOY_BROWSER_PROFILE,"grokboy-state.json"); await context.storageState({path}); await chmod(path,0o600); } } async function workspacePath(value,existing) { if(!value) throw new Error("missing workspace file path"); const root=await realpath(process.cwd()); const path=resolve(root,String(value)); const rel=relative(root,path); if(rel.startsWith("..")||isAbsolute(rel)) throw new Error("path outside workspace"); let ancestor=existing?path:dirname(path); while(true) { try { const real=await realpath(ancestor); const r=relative(root,real); if(r.startsWith("..")||isAbsolute(r)) throw new Error("symlink outside workspace"); break; } catch(e) { if(e.code!=="ENOENT") throw e; const parent=dirname(ancestor); if(parent===ancestor) throw e; ancestor=parent; } } if(!existing) { try { await lstat(path); throw new Error("download target exists"); } catch(e) { if(e.code!=="ENOENT") throw e; } await mkdir(dirname(path),{recursive:true}); } return path; } function parseArgs(argv) { const out = { cmd: null, selfTest: false, daemon: true }; for (let i = 2; i < argv.length; i++) { const a = argv[i]; if (a === "--self-test") out.selfTest = true; else if (a === "--cmd" || a === "-c") { out.cmd = argv[++i]; out.daemon = false; } else if (a === "--daemon") out.daemon = true; else if (a === "--help" || a === "-h") out.help = true; } return out; } async function runSelfTest() { const cases = []; // Protocol: ping without playwright let r = await handle({ id: "t1", op: "ping" }); cases.push(["ping", r.ok === true && r.pong === true]); cases.push([ "ping_lists_handoff_prepare", Array.isArray(r.ops) && r.ops.includes("handoff_prepare"), ]); // Missing fields fail closed r = await handle({ id: "t2", op: "navigate" }); cases.push(["navigate_missing_url", r.ok === false && r.blocked === true]); r = await handle({ id: "t3", op: "type", selector: "#x" }); // ensurePage may fail if no playwright — either blocked missing text first // type checks text before ensurePage... actually type checks text then ensurePage then resolveLocator // missing text → fail before playwright cases.push(["type_missing_text", r.ok === false]); r = await handle({ id: "t4", op: "nope" }); cases.push(["unknown_op", r.ok === false]); await handle({ id: "t4b", op: "close" }); r = await handle({ id: "t5", op: "status" }); cases.push(["status", r.ok === true && r.browser_open === false]); const failed = cases.filter(([, okv]) => !okv); const summary = { ok: failed.length === 0, cases: Object.fromEntries(cases), failed: failed.map(([n]) => n), }; console.log(JSON.stringify(summary)); process.exit(failed.length === 0 ? 0 : 1); } async function main() { const args = parseArgs(process.argv); if (args.help) { console.log( JSON.stringify({ ok: true, usage: "browser_helper.mjs [--self-test] | [--cmd JSON] | (JSONL on stdin)", install: INSTALL_HINT, }) ); return; } if (args.selfTest) { await runSelfTest(); return; } if (args.cmd) { let req; try { req = JSON.parse(args.cmd); } catch (e) { console.log(JSON.stringify(fail(null, `invalid --cmd JSON: ${e.message}`))); process.exit(1); return; } const res = await handle(req); await persistContext(); await handle({id:"shutdown",op:"close"}); console.log(JSON.stringify(res)); process.exit(res.ok ? 0 : 1); return; } // JSONL daemon const rl = createInterface({ input: process.stdin, crlfDelay: Infinity }); for await (const line of rl) { const trimmed = line.trim(); if (!trimmed) continue; let req; try { req = JSON.parse(trimmed); } catch (e) { console.log( JSON.stringify(fail(null, `invalid JSON line: ${e.message}`)) ); continue; } const res = await handle(req); await persistContext(); console.log(JSON.stringify(res)); if (req.op === "close" || req.command === "close") { // keep process alive unless parent closes stdin } } if (context) { await persistContext(); await context.close().catch(() => {}); if (browser) await browser.close().catch(() => {}); } } let shuttingDown = false; async function shutdownSignal() { if (shuttingDown) return; shuttingDown = true; const deadline = setTimeout(() => process.exit(1), 5000); try { await handle({id:"shutdown",op:"close"}); } finally { clearTimeout(deadline); process.exit(0); } } process.on("SIGTERM", shutdownSignal); process.on("SIGINT", shutdownSignal); main().catch(async (e) => { await handle({id:"shutdown",op:"close"}).catch(() => {}); console.error(JSON.stringify(fail(null, e.message || e))); process.exit(1); });