LazyBoy2/tools/playwright/browser_helper.mjs

737 lines
24 KiB
JavaScript
Executable File

#!/usr/bin/env node
/**
* GrokBoy Playwright DOM helper (optional).
*
* Modes:
* JSONL daemon (default when no --cmd): read JSON lines from stdin, write JSON lines to stdout.
* One-shot: node browser_helper.mjs --cmd '<json>'
* Self-test: node browser_helper.mjs --self-test (no Chromium required)
*
* Ops: ping, navigate, snapshot, click, type, eval, close, status, handoff_prepare
* Prefer CSS selector or role+name. Screenshots are NOT the primary control surface.
*/
import { createInterface } from "node:readline";
import { dirname, resolve, relative, isAbsolute } from "node:path";
import { mkdir, stat, realpath, lstat, readFile, chmod } from "node:fs/promises";
import { fileURLToPath } from "node:url";
const __dirname = dirname(fileURLToPath(import.meta.url));
const INSTALL_HINT =
"Playwright not available. From the GrokBoy repo run: " +
"cd tools/playwright && npm install && npx playwright install chromium";
let browser = null;
let context = null;
let page = null;
let lastUrl = null;
/** Whether the current browser was launched headed (visible). */
let browserHeaded = false;
let nextTab = 0;
const tabs = new Map();
let handoffStorage;
function registerTab(p) {
if ([...tabs.values()].includes(p)) return;
tabs.set(String(++nextTab), p);
p.on("close", () => { for (const [id,t] of tabs) if(t===p) tabs.delete(id); if(page===p) page=[...tabs.values()][0] || null; });
}
async function target(req) {
await ensurePage();
if (req.tab_id) { const tab=tabs.get(String(req.tab_id)); if(!tab || tab.isClosed()) throw new Error("unknown tab_id; list tabs again"); page=tab; }
if(req.frame) { const el=await page.locator(String(req.frame)).elementHandle(); const frame=await el?.contentFrame(); if(!frame) throw new Error("frame not found; obtain a new snapshot"); return frame; }
return page;
}
function envHeadedDefault() {
const v = (process.env.GROKBOY_BROWSER_HEADED || "").trim().toLowerCase();
if (v === "1" || v === "true" || v === "yes") return true;
if (v === "0" || v === "false" || v === "no") return false;
return false;
}
function wantHeaded(forceHeaded) {
if (forceHeaded === true) return true;
return envHeadedDefault();
}
function ok(id, extra = {}) {
return { id: id ?? null, ok: true, ...extra };
}
function fail(id, error, extra = {}) {
return {
id: id ?? null,
ok: false,
blocked: true,
error: String(error),
...extra,
};
}
async function loadPlaywright() {
try {
return await import("playwright");
} catch (e) {
const err = new Error(`${INSTALL_HINT} (import failed: ${e.message})`);
err.code = "PLAYWRIGHT_MISSING";
throw err;
}
}
async function launchBrowser(headed) {
const { chromium } = await loadPlaywright();
try {
if (process.env.GROKBOY_BROWSER_PROFILE) {
await mkdir(process.env.GROKBOY_BROWSER_PROFILE,{recursive:true,mode:0o700});
context=await chromium.launchPersistentContext(process.env.GROKBOY_BROWSER_PROFILE, { headless:!headed, acceptDownloads:true });
// Chromium does not restore session cookies on a clean restart; retain them per agent session.
const stored=await readFile(resolve(process.env.GROKBOY_BROWSER_PROFILE,"grokboy-state.json"),"utf8").then(JSON.parse).catch(()=>null);
if(stored?.cookies?.length) await context.addCookies(stored.cookies);
browser=context.browser();
} else {
browser=await chromium.launch({headless:!headed});
context=await browser.newContext({acceptDownloads:true, ...(handoffStorage ? {storageState:handoffStorage} : {})});
}
browserHeaded = headed;
} catch (e) {
const err = new Error(
`${INSTALL_HINT} (chromium launch failed: ${e.message})`
);
err.code = "CHROMIUM_MISSING";
throw err;
}
tabs.clear();
context.on("page",registerTab);
for (const p of context.pages()) registerTab(p);
page = context.pages()[0] || await context.newPage();
registerTab(page);
return page;
}
async function ensurePage(opts = {}) {
const headed = wantHeaded(opts.headed === true);
if (page) {
// Already open; if caller needs headed and we are headless, relaunch below via handoff_prepare.
return page;
}
return launchBrowser(headed);
}
/** Close current browser and reopen headed, restoring lastUrl when possible. */
async function ensureHeadedForHandoff() {
const restore = (page && !page.isClosed() ? page.url() : null) || lastUrl;
if (browser && browserHeaded && page) {
try {
await page.bringToFront();
} catch {
/* ignore */
}
return {
prepared: true,
headed: true,
relaunched: false,
url: page.url(),
restored: false,
};
}
const previousTabs = context ? await Promise.all(context.pages().filter(p=>!p.isClosed()).map(async p=>({
url:p.url(), active:p===page,
storage:await p.evaluate(()=>({origin:location.origin,values:Object.fromEntries(Object.entries(sessionStorage))})).catch(()=>null),
}))) : [];
if (context) {
handoffStorage=await context.storageState().catch(()=>undefined);
await persistContext();
await context.close().catch(() => {});
if (browser) await browser.close().catch(() => {});
browser = null;
context = null;
page = null;
browserHeaded = false;
}
await launchBrowser(true);
let restored = false;
if (previousTabs.length) {
const failures=[];
let activePage=page;
for (const [index,tab] of previousTabs.entries()) {
const reopened=index===0 ? page : await context.newPage();
const cdp=tab.storage ? await context.newCDPSession(reopened) : null;
let restoreScript;
try {
if(cdp) await cdp.send("Page.enable");
if(cdp) restoreScript=await cdp.send("Page.addScriptToEvaluateOnNewDocument",{source:
`(({origin,values})=>{if(location.origin===origin)for(const [key,value] of Object.entries(values))sessionStorage.setItem(key,value);})(${JSON.stringify(tab.storage)})`});
if(tab.url && tab.url!=="about:blank") {
await reopened.goto(tab.url,{waitUntil:"domcontentloaded",timeout:30000}).catch(e=>failures.push(String(e.message)));
}
} finally {
if(restoreScript) await cdp.send("Page.removeScriptToEvaluateOnNewDocument",{identifier:restoreScript.identifier});
if(cdp) await cdp.detach();
}
if(tab.active) activePage=reopened;
}
page=activePage;
lastUrl=page.url();
await page.bringToFront().catch(()=>{});
return {prepared:true,headed:true,relaunched:true,url:lastUrl,restored:failures.length===0,
tabs_restored:previousTabs.length,...(failures.length ? {restore_errors:failures} : {})};
}
if (restore && restore !== "about:blank") {
try {
await page.goto(String(restore), {
waitUntil: "domcontentloaded",
timeout: 30000,
});
lastUrl = page.url();
restored = true;
} catch (e) {
// Keep headed page even if restore fails; caller still sees visible window.
lastUrl = page.url();
return {
prepared: true,
headed: true,
relaunched: true,
url: lastUrl,
restored: false,
restore_error: String(e.message || e),
};
}
}
try {
await page.bringToFront();
} catch {
/* ignore */
}
return {
prepared: true,
headed: true,
relaunched: true,
url: page ? page.url() : null,
restored,
};
}
/** Accessibility-ish text snapshot: roles, names, and useful selectors — not pixels. */
async function buildSnapshot(p) {
const nodes = await p.evaluate(() => {
const out = [];
const max = 400;
const interesting = new Set([
"a",
"button",
"input",
"textarea",
"select",
"option",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"label",
"li",
"summary",
"nav",
"main",
"form",
"img",
"table",
"th",
"td",
]);
function cssPath(el) {
if (el.id) return `#${CSS.escape(el.id)}`;
const parts = [];
let cur = el;
while (cur && cur.nodeType === 1 && parts.length < 5) {
let part = cur.tagName.toLowerCase();
if (cur.id) {
parts.unshift(`#${CSS.escape(cur.id)}`);
break;
}
const parent = cur.parentElement;
if (parent) {
const siblings = [...parent.children].filter(
(c) => c.tagName === cur.tagName
);
if (siblings.length > 1) {
const idx = siblings.indexOf(cur) + 1;
part += `:nth-of-type(${idx})`;
}
}
parts.unshift(part);
cur = parent;
}
return parts.join(" > ");
}
const walker = document.createTreeWalker(
document.body || document.documentElement,
NodeFilter.SHOW_ELEMENT
);
let node = walker.currentNode;
while (node && out.length < max) {
const tag = node.tagName ? node.tagName.toLowerCase() : "";
const role =
node.getAttribute("role") ||
(tag === "a"
? "link"
: tag === "button"
? "button"
: tag === "input"
? node.getAttribute("type") === "submit"
? "button"
: "textbox"
: tag === "textarea"
? "textbox"
: tag === "img"
? "img"
: tag.startsWith("h") && tag.length === 2
? "heading"
: null);
const name =
node.getAttribute("aria-label") ||
node.getAttribute("alt") ||
node.getAttribute("placeholder") ||
node.getAttribute("title") ||
(node.innerText || node.textContent || "").trim().slice(0, 120);
const href = node.getAttribute && node.getAttribute("href");
const type = node.getAttribute && node.getAttribute("type");
const value =
type !== "password" && "value" in node && typeof node.value === "string"
? String(node.value).slice(0, 80)
: null;
if (interesting.has(tag) || node.getAttribute("role") || node.id) {
const entry = {
tag,
role: role || tag,
name: name || "",
selector: cssPath(node),
};
if (href) entry.href = href;
if (type) entry.type = type;
if (value) entry.value = value;
if (node.id) entry.id = node.id;
out.push(entry);
}
node = walker.nextNode();
}
return out;
});
const title = await p.evaluate(() => document.title);
const url = p.url();
const lines = nodes.map((n, i) => {
const bits = [`[${i}]`, n.role || n.tag];
if (n.name) bits.push(`name=${JSON.stringify(n.name)}`);
if (n.selector) bits.push(`sel=${n.selector}`);
if (n.href) bits.push(`href=${n.href}`);
if (n.type) bits.push(`type=${n.type}`);
if (n.value) bits.push(`value=${JSON.stringify(n.value)}`);
return bits.join(" ");
});
return {
url,
title,
count: nodes.length,
text: lines.join("\n"),
nodes,
frames: await p.locator("iframe").evaluateAll(els=>els.map((el,i)=>({selector:el.id ? `#${CSS.escape(el.id)}` : `iframe:nth-of-type(${i+1})`,src:el.src,name:el.name}))),
};
}
async function resolveLocator(p, args) {
if (args.selector) {
return p.locator(String(args.selector)).first();
}
if (args.role) {
const opts = {};
if (args.name) opts.name = String(args.name);
return p.getByRole(String(args.role), opts).first();
}
if (args.text) {
return p.getByText(String(args.text), { exact: !!args.exact }).first();
}
if (args.label) {
return p.getByLabel(String(args.label)).first();
}
if (args.placeholder) {
return p.getByPlaceholder(String(args.placeholder)).first();
}
throw new Error(
"click/type requires selector, role(+name), text, label, or placeholder"
);
}
async function handle(req) {
const id = req.id ?? null;
const op = req.op || req.command;
if (!op) return fail(id, "missing op");
try {
switch (op) {
case "ping":
return ok(id, {
pong: true,
protocol: 1,
ops: [
"ping",
"navigate",
"snapshot",
"click",
"type",
"eval",
"close",
"status",
"handoff_prepare", "read_page", "tabs", "press", "select", "scroll", "wait", "upload", "download",
],
});
case "status": {
return ok(id, {
browser_open: !!browser,
headed: browserHeaded,
last_url: lastUrl,
page_url: page ? page.url() : null,
env_headed: envHeadedDefault(),
});
}
case "handoff_prepare": {
// Bring a visible Chromium window forward for human login/OTP/captcha.
const info = await ensureHeadedForHandoff();
return ok(id, {
...info,
message:
"Browser is headed (visible). Complete login/OTP/captcha in the window, then resume in the terminal.",
});
}
case "navigate": {
const url = req.url;
if (!url) return fail(id, "navigate: missing url");
const p = await target(req);
const resp = await p.goto(String(url), {
waitUntil: "domcontentloaded",
timeout: req.timeout_ms ?? 30000,
});
lastUrl = p.url();
return ok(id, {
url: lastUrl,
title: await p.title(),
status: resp ? resp.status() : null,
});
}
case "snapshot":
case "dom": {
const p = await target(req);
const snap = await buildSnapshot(p);
lastUrl = snap.url;
return ok(id, {
url: snap.url,
title: snap.title,
count: snap.count,
snapshot: snap.text,
nodes: snap.nodes.slice(0, 200),
frames: snap.frames,
});
}
case "click": {
if (
!req.selector &&
!req.role &&
!req.text &&
!req.label &&
!req.placeholder
) {
return fail(
id,
"click requires selector, role(+name), text, label, or placeholder"
);
}
const p = await target(req);
const loc = await resolveLocator(p, req);
const popupEvent=page.waitForEvent("popup",{timeout:750}).catch(()=>null);
await loc.click({ timeout: req.timeout_ms ?? 10000 });
const popup=await popupEvent;
if(popup) { registerTab(popup); await popup.waitForLoadState("domcontentloaded",{timeout:10000}).catch(()=>{}); }
lastUrl = p.url();
return ok(id, { clicked: true, url: lastUrl, popup_tab_id:popup?[...tabs].find(([,p])=>p===popup)?.[0]:null });
}
case "type": {
const text = req.text ?? req.value;
if (text == null) return fail(id, "type: missing text");
const p = await target(req);
const loc = await resolveLocator(p, req);
if (req.clear !== false) {
await loc.fill(String(text), { timeout: req.timeout_ms ?? 10000 });
} else {
await loc.type(String(text), { timeout: req.timeout_ms ?? 10000 });
}
lastUrl = p.url();
return ok(id, { typed: true, url: lastUrl });
}
case "read_page": {
const p=await target(req);
const offset=Math.max(0,Number(req.offset)||0), limit=Math.min(32000,Math.max(1,Number(req.limit)||12000));
const text=await p.locator("body").innerText();
const links=await p.locator("a[href]").evaluateAll(els=>els.slice(0,200).map(e=>({text:(e.innerText||"").slice(0,200),url:e.href})));
const chars=Array.from(text);
return ok(id,{url:p.url(),title:await p.evaluate(()=>document.title),text:chars.slice(offset,offset+limit).join(""),offset,next_offset:Math.min(chars.length,offset+limit),truncated:chars.length>offset+limit,links});
}
case "tabs": {
await ensurePage();
const action=req.action||"list";
if(action==="new") { page=await context.newPage(); registerTab(page); }
else if(action==="switch") { await target(req); await page.bringToFront(); }
else if(action==="close") { await target(req); await page.close(); }
else if(action!=="list") throw new Error("invalid tabs action");
const items=[];
for(const [tab_id,p] of tabs) { items.push({tab_id,url:p.url(),title:await p.title(),active:p===page}); }
return ok(id,{tabs:items});
}
case "press": {
if(!req.key) throw new Error("press requires key");
const p=await target(req);
if(req.selector||req.role||req.label||req.placeholder) await (await resolveLocator(p,req)).press(String(req.key));
else await page.keyboard.press(String(req.key));
return ok(id,{pressed:req.key,url:p.url()});
}
case "select": {
const p=await target(req);
if(!Array.isArray(req.values)) throw new Error("select requires values array");
const selected=await (await resolveLocator(p,req)).selectOption(req.values.map(String));
return ok(id,{selected,url:p.url()});
}
case "scroll": {
const p=await target(req); const delta=Number(req.delta_y)||600;
if(req.selector) await (await resolveLocator(p,req)).evaluate((el,y)=>el.scrollBy(0,y),delta);
else await p.evaluate(y=>window.scrollBy(0,y),delta);
return ok(id,{scrolled:delta,url:p.url()});
}
case "wait": {
const p=await target(req); const timeout=Math.min(30000,Math.max(1,Number(req.timeout_ms)||10000));
if(req.url) await p.waitForURL(String(req.url),{timeout});
else await (await resolveLocator(p,req)).waitFor({state:req.state||"visible",timeout});
return ok(id,{ready:true,url:p.url()});
}
case "upload": {
const p=await target(req); const path=await workspacePath(req.path,true);
await (await resolveLocator(p,req)).setInputFiles(path);
return ok(id,{uploaded:true,path,url:p.url()});
}
case "download": {
const p=await target(req); const path=await workspacePath(req.path,false);
// Attach rejection handling immediately if clicking fails before the event arrives.
const downloadPromise=page.waitForEvent("download",{timeout:30000});
downloadPromise.catch(()=>{});
await (await resolveLocator(p,req)).click();
const download=await downloadPromise;
await download.saveAs(path);
const failure=await download.failure(); if(failure) throw new Error(failure);
return ok(id,{downloaded:true,path,bytes:(await stat(path)).size,suggested_filename:download.suggestedFilename(),url:p.url()});
}
case "eval": {
const p = await target(req);
const expression = req.expression ?? req.js ?? req.code;
if (!expression) return fail(id, "eval: missing expression");
// Evaluate as expression body; fail closed on throw.
const result = await p.evaluate((expr) => {
// eslint-disable-next-line no-eval
return eval(expr);
}, String(expression));
let serialized;
try {
serialized = JSON.parse(JSON.stringify(result));
} catch {
serialized = String(result);
}
return ok(id, { result: serialized, url: p.url() });
}
case "close": {
if (context) {
await persistContext();
await context.close().catch(() => {});
if (browser) await browser.close().catch(() => {});
}
browser = null;
context = null;
page = null;
browserHeaded = false;
return ok(id, { closed: true, last_url: lastUrl });
}
default:
return fail(id, `unknown op: ${op}`);
}
} catch (e) {
const extra = {};
if (e && e.code) extra.code = e.code;
return fail(id, e.message || e, extra);
}
}
async function persistContext() {
if(context && process.env.GROKBOY_BROWSER_PROFILE) {
const path=resolve(process.env.GROKBOY_BROWSER_PROFILE,"grokboy-state.json");
await context.storageState({path});
await chmod(path,0o600);
}
}
async function workspacePath(value,existing) {
if(!value) throw new Error("missing workspace file path");
const root=await realpath(process.cwd());
const path=resolve(root,String(value));
const rel=relative(root,path);
if(rel.startsWith("..")||isAbsolute(rel)) throw new Error("path outside workspace");
let ancestor=existing?path:dirname(path);
while(true) { try { const real=await realpath(ancestor); const r=relative(root,real); if(r.startsWith("..")||isAbsolute(r)) throw new Error("symlink outside workspace"); break; } catch(e) { if(e.code!=="ENOENT") throw e; const parent=dirname(ancestor); if(parent===ancestor) throw e; ancestor=parent; } }
if(!existing) { try { await lstat(path); throw new Error("download target exists"); } catch(e) { if(e.code!=="ENOENT") throw e; } await mkdir(dirname(path),{recursive:true}); }
return path;
}
function parseArgs(argv) {
const out = { cmd: null, selfTest: false, daemon: true };
for (let i = 2; i < argv.length; i++) {
const a = argv[i];
if (a === "--self-test") out.selfTest = true;
else if (a === "--cmd" || a === "-c") {
out.cmd = argv[++i];
out.daemon = false;
} else if (a === "--daemon") out.daemon = true;
else if (a === "--help" || a === "-h") out.help = true;
}
return out;
}
async function runSelfTest() {
const cases = [];
// Protocol: ping without playwright
let r = await handle({ id: "t1", op: "ping" });
cases.push(["ping", r.ok === true && r.pong === true]);
cases.push([
"ping_lists_handoff_prepare",
Array.isArray(r.ops) && r.ops.includes("handoff_prepare"),
]);
// Missing fields fail closed
r = await handle({ id: "t2", op: "navigate" });
cases.push(["navigate_missing_url", r.ok === false && r.blocked === true]);
r = await handle({ id: "t3", op: "type", selector: "#x" });
// ensurePage may fail if no playwright — either blocked missing text first
// type checks text before ensurePage... actually type checks text then ensurePage then resolveLocator
// missing text → fail before playwright
cases.push(["type_missing_text", r.ok === false]);
r = await handle({ id: "t4", op: "nope" });
cases.push(["unknown_op", r.ok === false]);
await handle({ id: "t4b", op: "close" });
r = await handle({ id: "t5", op: "status" });
cases.push(["status", r.ok === true && r.browser_open === false]);
const failed = cases.filter(([, okv]) => !okv);
const summary = {
ok: failed.length === 0,
cases: Object.fromEntries(cases),
failed: failed.map(([n]) => n),
};
console.log(JSON.stringify(summary));
process.exit(failed.length === 0 ? 0 : 1);
}
async function main() {
const args = parseArgs(process.argv);
if (args.help) {
console.log(
JSON.stringify({
ok: true,
usage:
"browser_helper.mjs [--self-test] | [--cmd JSON] | (JSONL on stdin)",
install: INSTALL_HINT,
})
);
return;
}
if (args.selfTest) {
await runSelfTest();
return;
}
if (args.cmd) {
let req;
try {
req = JSON.parse(args.cmd);
} catch (e) {
console.log(JSON.stringify(fail(null, `invalid --cmd JSON: ${e.message}`)));
process.exit(1);
return;
}
const res = await handle(req);
await persistContext();
await handle({id:"shutdown",op:"close"});
console.log(JSON.stringify(res));
process.exit(res.ok ? 0 : 1);
return;
}
// JSONL daemon
const rl = createInterface({ input: process.stdin, crlfDelay: Infinity });
for await (const line of rl) {
const trimmed = line.trim();
if (!trimmed) continue;
let req;
try {
req = JSON.parse(trimmed);
} catch (e) {
console.log(
JSON.stringify(fail(null, `invalid JSON line: ${e.message}`))
);
continue;
}
const res = await handle(req);
await persistContext();
console.log(JSON.stringify(res));
if (req.op === "close" || req.command === "close") {
// keep process alive unless parent closes stdin
}
}
if (context) {
await persistContext();
await context.close().catch(() => {});
if (browser) await browser.close().catch(() => {});
}
}
let shuttingDown = false;
async function shutdownSignal() {
if (shuttingDown) return;
shuttingDown = true;
const deadline = setTimeout(() => process.exit(1), 5000);
try { await handle({id:"shutdown",op:"close"}); }
finally { clearTimeout(deadline); process.exit(0); }
}
process.on("SIGTERM", shutdownSignal);
process.on("SIGINT", shutdownSignal);
main().catch(async (e) => {
await handle({id:"shutdown",op:"close"}).catch(() => {});
console.error(JSON.stringify(fail(null, e.message || e)));
process.exit(1);
});