// AnyBot's browser, the full toolset. botbrowser.rs starts Chrome and keeps profiles; this file // drives a tab through the Chrome DevTools Protocol like a person would (a Firefox tab through // WebDriver BiDi: bidi.rs does each DevTools command the BiDi way, so the tools are the same): // - real mouse and keyboard (Input.dispatchMouseEvent / insertText / dispatchKeyEvent), so // React date pickers, dropdowns and menus that ignore scripted events still respond; // - a numbered list of everything clickable (links with their addresses, buttons, fields, // dropdowns, tabs, calendar cells — anything the page gives a pointer cursor); // - after each action only what changed (new address, new text), not the whole page again; // - tab screenshots (no screen-recording permission needed), tables as Markdown, find on page, // waiting for text, tabs, back/forward/reload, file uploads, downloads, PDF, page dialogs and // running a script on the page. use serde_json::{json, Value}; use std::io::ErrorKind; use std::net::{SocketAddr, TcpStream}; use std::time::{Duration, Instant}; use crate::botbrowser::{self, Kind}; const CALL_TIMEOUT_SECS: u64 = 30; /// Longest element list in one answer. const MAX_ELEMENTS: usize = 250; /// One connection to a tab: DevTools (Chrome-family), or a tab of a Firefox driven over BiDi. pub struct Tab { link: Link, next: u64, } enum Link { Cdp(Box>), Bidi { port: u16, context: String }, } /// The ordinary page tabs of a /json/list answer, most recently used first. pub(crate) fn pages(list: &Value) -> Vec { list.as_array().cloned().unwrap_or_default().into_iter().filter(|t| t["type"] == "page" && !t["url"].as_str().unwrap_or("").starts_with("devtools://")).collect() } /// The browser's port for a profile (starting it if needed), and which protocol it speaks. pub fn port_of(profile: Option<&str>) -> Result<(String, u16, Kind), String> { let prof = botbrowser::resolve(profile)?; let (_, port, _) = botbrowser::ensure(&prof, None)?; Ok((prof.clone(), port, botbrowser::kind_of(Some(&prof)))) } impl Tab { /// The tab this step works on: its job's latest own tab, else the latest one no other job /// has (the list has the most recently used first). pub fn current(profile: Option<&str>) -> Result { let (_, port, kind) = port_of(profile)?; let list = botbrowser::pages_of(kind, port)?; // This job's own latest tab (another bot's job may have one in front). let t = botbrowser::pick_tab(&list).ok_or("AnyBot 瀏覽器沒有開著的分頁,請先用 browser_open 打開網址。")?; botbrowser::claim(t["id"].as_str().unwrap_or("")); if kind == Kind::Firefox { let context = t["id"].as_str().unwrap_or("").to_string(); crate::bidi::touch(port, &context); return Ok(Tab { link: Link::Bidi { port, context }, next: 0 }); } Tab::connect(port, t) } fn connect(port: u16, t: &Value) -> Result { let ws_url = t["webSocketDebuggerUrl"].as_str().ok_or("這個分頁不能控制(可能已被別的工具連著)")?; let addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap(); let stream = TcpStream::connect_timeout(&addr, Duration::from_secs(5)).map_err(|e| e.to_string())?; stream.set_read_timeout(Some(Duration::from_secs(CALL_TIMEOUT_SECS))).ok(); let (ws, _) = tungstenite::client::client(ws_url, stream).map_err(|e| format!("連不上 AnyBot 瀏覽器:{e}"))?; Ok(Tab { link: Link::Cdp(Box::new(ws)), next: 0 }) } pub fn is_firefox(&self) -> bool { matches!(self.link, Link::Bidi { .. }) } /// One DevTools command; events that arrive meanwhile are skipped. (On Firefox, done the /// WebDriver BiDi way, with an answer shaped the same.) pub fn call(&mut self, method: &str, params: Value) -> Result { let ws = match &mut self.link { Link::Bidi { port, context } => return crate::bidi::cdp(*port, context, method, ¶ms), Link::Cdp(ws) => ws, }; self.next += 1; let id = self.next; ws.send(tungstenite::Message::Text(json!({"id": id, "method": method, "params": params}).to_string().into())).map_err(|e| e.to_string())?; let start = Instant::now(); loop { if start.elapsed() > Duration::from_secs(CALL_TIMEOUT_SECS) { return Err("瀏覽器太久沒有回應(頁面可能跳出了對話框:可以用 browser_dialog 處理)".into()); } match ws.read() { Ok(tungstenite::Message::Text(t)) => { let v: Value = serde_json::from_str(t.as_str()).unwrap_or(Value::Null); if v["id"] == id { if let Some(e) = v.get("error") { return Err(e["message"].as_str().unwrap_or("瀏覽器拒絕了這個動作").to_string()); } return Ok(v["result"].clone()); } } Ok(_) => {} Err(tungstenite::Error::Io(e)) if e.kind() == ErrorKind::WouldBlock || e.kind() == ErrorKind::TimedOut => return Err("瀏覽器太久沒有回應(頁面可能跳出了對話框:可以用 browser_dialog 處理)".into()), Err(e) => return Err(e.to_string()), } } } /// Run JavaScript and get its value back. pub fn eval(&mut self, expr: &str) -> Result { let r = self.call("Runtime.evaluate", json!({"expression": expr, "returnByValue": true, "awaitPromise": true, "userGesture": true}))?; if let Some(ex) = r.get("exceptionDetails") { return Err(format!("頁面上的程式出錯:{}", ex["exception"]["description"].as_str().or(ex["text"].as_str()).unwrap_or(""))); } Ok(r["result"]["value"].clone()) } fn close(self) { if let Link::Cdp(mut ws) = self.link { let _ = ws.close(None); } } } // ---------- what the page offers ---------- /// Every element matching `sel`, the page's own and those inside open shadow roots (web /// components: Chromium's settings pages, YouTube, many app frameworks), in the order they are /// laid out: a host, then what its shadow root shows, then its own children. Plain /// querySelectorAll stops at every shadow root, so those buttons had no number at all. const DEEP_JS: &str = "const deepAll = (sel, root) => { const out = []; const walk = r => { for (const el of r.querySelectorAll('*')) { if (sel === '*' || el.matches(sel)) out.push(el); if (el.shadowRoot) walk(el.shadowRoot); } }; walk(root || document); return out; };"; /// Numbers everything clickable or fillable (data-anybot-ref) and lists it. const ELEMENTS_JS: &str = r#"(() => { %DEEP% const vis = el => { const r = el.getBoundingClientRect(); if (r.width < 2 || r.height < 2) return false; const s = getComputedStyle(el); return s.visibility !== 'hidden' && s.display !== 'none' && +s.opacity > 0.05; }; const clean = s => (s || '').replace(/\s+/g, ' ').trim(); const name = el => clean(el.getAttribute('aria-label') || el.innerText || el.value || el.getAttribute('placeholder') || el.title || el.getAttribute('alt') || (el.querySelector('img[alt]') || {}).alt || '').slice(0, 80); const label = el => { const a = el.getAttribute('aria-label'); if (a) return clean(a).slice(0, 60); if (el.labels && el.labels[0]) return clean(el.labels[0].innerText).slice(0, 60); const id = el.getAttribute('aria-labelledby'); if (id) { const home = el.getRootNode(); const l = (home.getElementById ? home : document).getElementById(id); if (l) return clean(l.innerText).slice(0, 60); } return clean(el.getAttribute('placeholder') || el.name || el.title || ''); }; const base = 'a[href],button,input:not([type=hidden]),select,textarea,summary,[contenteditable=""],[contenteditable=true],[role=button],[role=link],[role=tab],[role=menuitem],[role=menuitemcheckbox],[role=option],[role=checkbox],[role=radio],[role=combobox],[role=switch],[role=treeitem],[role=gridcell],[onclick],[tabindex]:not([tabindex="-1"])'; const all = deepAll('*'); const order = new Map(all.map((el, i) => [el, i])); const set = new Set(all.filter(el => el.matches(base))); // Anything the page shows a pointer cursor for (calendar days, cards, custom buttons…). for (const el of all) { if (set.size > 4000) break; if (!el.matches('div,span,li,td,th,p,img,svg,i,label')) continue; if (getComputedStyle(el).cursor === 'pointer' && !(el.parentElement && getComputedStyle(el.parentElement).cursor === 'pointer')) set.add(el); } all.forEach(e => e.hasAttribute('data-anybot-ref') && e.removeAttribute('data-anybot-ref')); const inView = el => { const r = el.getBoundingClientRect(); return r.bottom > 0 && r.top < innerHeight && r.right > 0 && r.left < innerWidth; }; const ordered = [...set].filter(el => vis(el) && !el.closest('[aria-hidden=true]')).sort((a, b) => order.get(a) - order.get(b)); const out = []; const seen = new Set(); let n = 0; for (const el of ordered) { const tag = el.tagName.toLowerCase(); const role = el.getAttribute('role') || ''; let kind, text = name(el), extra = ''; if (tag === 'a') { kind = '連結'; extra = el.href ? '→ ' + el.href : ''; } else if (tag === 'select') { kind = '下拉選單'; text = label(el); extra = [...el.options].map(o => (o.selected ? '✓' : '') + clean(o.text)).slice(0, 20).join(' / '); } else if (tag === 'input' || tag === 'textarea') { const t = (el.type || 'text').toLowerCase(); if (t === 'checkbox' || t === 'radio') { kind = t === 'checkbox' ? '勾選框' : '選項'; text = label(el) || text; extra = el.checked ? '已勾' : '未勾'; } else if (t === 'submit' || t === 'button' || t === 'reset') kind = '按鈕'; else if (t === 'file') { kind = '上傳檔案'; text = label(el) || '選擇檔案'; } else { kind = '輸入框'; text = label(el); extra = t === 'password' ? (el.value ? '(已填)' : '(密碼)') : (el.value ? '目前:' + clean(el.value).slice(0, 40) : ''); if (el.readOnly) extra += '(唯讀,點它會打開選擇器)'; } } else if (el.isContentEditable) { kind = '輸入框'; text = label(el) || clean(el.innerText).slice(0, 40); } else if (tag === 'button' || role === 'button' || tag === 'summary') kind = '按鈕'; else if (role === 'tab') { kind = '分頁標籤'; if (el.getAttribute('aria-selected') === 'true') extra = '(目前)'; } else if (role === 'menuitem' || role === 'menuitemcheckbox' || role === 'option' || role === 'treeitem') { kind = '選單項目'; if (el.getAttribute('aria-selected') === 'true' || el.getAttribute('aria-checked') === 'true') extra = '(已選)'; } else if (role === 'combobox') { kind = '下拉選單'; extra = clean(el.innerText).slice(0, 40); } else if (role === 'link') kind = '連結'; else if (role === 'checkbox' || role === 'switch' || role === 'radio') { kind = '勾選框'; extra = el.getAttribute('aria-checked') === 'true' ? '已勾' : '未勾'; } else if (role === 'gridcell' || tag === 'td') { kind = '格子'; const full = el.getAttribute('title') || el.getAttribute('aria-label') || el.getAttribute('data-date') || (el.closest('[title]') || {}).title || ''; if (full && full !== text) text = text ? text + '(' + full + ')' : full; } else { kind = '可點的項目'; const cell = el.closest('td[title],[role=gridcell][aria-label],[data-date]'); if (cell) { kind = '格子'; const full = cell.getAttribute('title') || cell.getAttribute('aria-label') || cell.getAttribute('data-date'); if (full && full !== text) text = text ? text + '(' + full + ')' : full; } } if (!text && !extra) continue; if (el.disabled || el.getAttribute('aria-disabled') === 'true') extra += '(停用)'; const key = kind + '|' + text + '|' + extra; if (seen.has(key)) continue; seen.add(key); n += 1; el.setAttribute('data-anybot-ref', String(n)); out.push('[' + n + '] ' + kind + '「' + text + '」' + (extra ? ' ' + extra : '') + (inView(el) ? '' : ' (要捲動)')); if (n >= %MAX%) break; } return JSON.stringify({t: document.title, u: location.href, list: out, more: ordered.length > n}); })()"#; /// The numbered list of what can be clicked or filled. pub fn elements(tab: &mut Tab) -> Result<(String, String, Vec, bool), String> { let v = tab.eval(&ELEMENTS_JS.replace("%DEEP%", DEEP_JS).replace("%MAX%", &MAX_ELEMENTS.to_string()))?; let p: Value = v.as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or(Value::Null); let list = p["list"].as_array().map(|a| a.iter().filter_map(|x| x.as_str().map(String::from)).collect()).unwrap_or_default(); Ok((p["t"].as_str().unwrap_or("").into(), p["u"].as_str().unwrap_or("").into(), list, p["more"].as_bool().unwrap_or(false))) } /// Draws a numbered box over every listed element that is on screen (removed after the shot). const MARKS_JS: &str = r#"(() => { %DEEP% const old = document.getElementById('__anybot_marks'); if (old) old.remove(); const box = document.createElement('div'); box.id = '__anybot_marks'; box.style.cssText = 'position:fixed;inset:0;pointer-events:none;z-index:2147483647'; const colors = ['#e11d48', '#2563eb', '#16a34a', '#d97706', '#9333ea']; for (const el of deepAll('[data-anybot-ref]')) { const r = el.getBoundingClientRect(); if (r.bottom < 0 || r.top > innerHeight || r.right < 0 || r.left > innerWidth || r.width < 2) continue; const n = el.getAttribute('data-anybot-ref'); const c = colors[n % colors.length]; const f = document.createElement('div'); f.style.cssText = `position:fixed;left:${r.left}px;top:${r.top}px;width:${r.width}px;height:${r.height}px;border:1.5px solid ${c};border-radius:3px;box-sizing:border-box`; const tag = document.createElement('div'); tag.textContent = n; tag.style.cssText = `position:fixed;left:${Math.max(0, r.left - 1)}px;top:${Math.max(0, r.top - 15)}px;background:${c};color:#fff;font:bold 11px/14px -apple-system,Arial;padding:0 3px;border-radius:3px`; box.appendChild(f); box.appendChild(tag); } document.documentElement.appendChild(box); return 1; })()"#; const SNAPSHOT_JS: &str = "JSON.stringify({t: document.title, u: location.href, x: document.body ? document.body.innerText : ''})"; fn snapshot(tab: &mut Tab) -> (String, String, Vec) { let v = tab.eval(SNAPSHOT_JS).unwrap_or(Value::Null); let p: Value = v.as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or(Value::Null); let lines = p["x"].as_str().unwrap_or("").lines().map(str::trim).filter(|l| !l.is_empty()).map(String::from).collect(); (p["t"].as_str().unwrap_or("").into(), p["u"].as_str().unwrap_or("").into(), lines) } /// What an action changed: the address, and the lines of text that are new. fn changes(before: &(String, String, Vec), after: &(String, String, Vec)) -> String { let mut out = vec![]; if before.1 != after.1 { out.push(format!("網址變成:{}", after.1)); } if before.0 != after.0 { out.push(format!("標題:{}", after.0)); } let old: std::collections::HashSet<&String> = before.2.iter().collect(); let new: Vec<&String> = after.2.iter().filter(|l| !old.contains(l)).collect(); let gone = before.2.iter().filter(|l| !after.2.contains(l)).count(); if new.is_empty() && gone == 0 && before.1 == after.1 { out.push("頁面看起來沒有變化(可能要等一下、換一個元素,或用 browser_screenshot 看看)。".into()); } else if !new.is_empty() { let shown: Vec = new.iter().take(40).map(|l| l.chars().take(160).collect()).collect(); out.push(format!("新出現的文字:\n{}{}", shown.join("\n"), if new.len() > 40 { format!("\n…還有 {} 行(用 browser_read 看全部)", new.len() - 40) } else { String::new() })); } out.join("\n") } // ---------- finding an element ---------- /// The element a bot means: its number, else visible text / label / placeholder. fn find_js(r#ref: &str, text: &str) -> String { let rf = serde_json::to_string(r#ref.trim().trim_matches(['[', ']'])).unwrap_or_default(); let tx = serde_json::to_string(text.trim()).unwrap_or_default(); format!( r#"(() => {{ {deep} const rf = {rf}, tx = {tx}; if (rf) {{ const e = deepAll('[data-anybot-ref="' + rf + '"]')[0]; if (e) return e; }} if (!tx) return null; const t = s => (s || '').replace(/\s+/g, ' ').trim(); const vis = e => {{ const r = e.getBoundingClientRect(); return r.width > 1 && r.height > 1; }}; const all = deepAll('a,button,[role],input,select,textarea,label,summary,li,td,th,span,div,p,img').filter(vis); const hit = e => [t(e.innerText), t(e.getAttribute('aria-label')), t(e.getAttribute('placeholder')), t(e.title), t(e.value), e.labels && e.labels[0] ? t(e.labels[0].innerText) : '']; // The control the words name, not the box around it: 「Customer name:」 is a

holding a // label and its input, 「Submit order」 a

holding the button (its middle is empty page). const ctl = 'a,button,input,select,textarea,summary,[role=button],[role=link],[role=checkbox],[role=radio],[role=switch],[role=tab],[role=menuitem],[role=option],[role=textbox],[role=combobox],[contenteditable=""],[contenteditable=true]'; const isCtl = e => e.matches(ctl); const inner = e => {{ if (isCtl(e)) return e; if (e.tagName === 'LABEL' && e.control && vis(e.control)) return e.control; const c = deepAll(ctl, e).filter(vis); return c.length === 1 ? c[0] : e; }}; const shortest = l => l.slice().sort((a, b) => t(a.innerText).length - t(b.innerText).length)[0]; const pick = l => l.find(isCtl) || l.map(inner).find(isCtl) || shortest(l) || null; const exact = all.filter(e => hit(e).includes(tx)); return exact.length ? pick(exact) : pick(all.filter(e => hit(e).some(h => h.includes(tx)))); }})()"#, deep = DEEP_JS ) } /// Bring the element into view and get its centre (CSS pixels in the viewport). fn locate(tab: &mut Tab, r#ref: &str, text: &str) -> Result<(f64, f64, String), String> { let js = format!( "(() => {{ const el = {}; if (!el) return null; el.scrollIntoView({{block: 'center', inline: 'center'}}); const r = el.getBoundingClientRect(); return JSON.stringify({{x: r.left + r.width / 2, y: r.top + r.height / 2, tag: el.tagName, type: el.type || '', ro: !!el.readOnly}}); }})()", find_js(r#ref, text) ); let v = tab.eval(&js)?; let what = if !r#ref.trim().is_empty() { format!("[{}]", r#ref.trim().trim_matches(['[', ']'])) } else { format!("「{}」", text.trim()) }; let p: Value = v.as_str().and_then(|s| serde_json::from_str(s).ok()).ok_or_else(|| format!("這一頁找不到 {what}。用 browser_read 或 browser_elements 看最新的清單(編號每次讀取會重排)。"))?; Ok((p["x"].as_f64().unwrap_or(0.0), p["y"].as_f64().unwrap_or(0.0), format!("{what}{}", if p["tag"] == "INPUT" && p["ro"] == true { "(唯讀欄位)" } else { "" }))) } fn mouse(tab: &mut Tab, kind: &str, x: f64, y: f64, button: &str, count: i64) -> Result<(), String> { tab.call("Input.dispatchMouseEvent", json!({"type": kind, "x": x, "y": y, "button": button, "buttons": if kind == "mousePressed" { 1 } else { 0 }, "clickCount": count})).map(|_| ()) } /// A real click (the page can't tell it from a person's). fn click_at(tab: &mut Tab, x: f64, y: f64, button: &str, count: i64) -> Result<(), String> { mouse(tab, "mouseMoved", x, y, "none", 0)?; for n in 1..=count { mouse(tab, "mousePressed", x, y, button, n)?; mouse(tab, "mouseReleased", x, y, button, n)?; } Ok(()) } /// Key names the bot may use → (key, code, keyCode, text). fn key_info(k: &str) -> Option<(&'static str, &'static str, i64, &'static str)> { Some(match k.to_lowercase().as_str() { "enter" | "return" => ("Enter", "Enter", 13, "\r"), "tab" => ("Tab", "Tab", 9, ""), "escape" | "esc" => ("Escape", "Escape", 27, ""), "backspace" => ("Backspace", "Backspace", 8, ""), "delete" => ("Delete", "Delete", 46, ""), "space" => (" ", "Space", 32, " "), "arrowdown" | "down" => ("ArrowDown", "ArrowDown", 40, ""), "arrowup" | "up" => ("ArrowUp", "ArrowUp", 38, ""), "arrowleft" | "left" => ("ArrowLeft", "ArrowLeft", 37, ""), "arrowright" | "right" => ("ArrowRight", "ArrowRight", 39, ""), "pagedown" => ("PageDown", "PageDown", 34, ""), "pageup" => ("PageUp", "PageUp", 33, ""), "home" => ("Home", "Home", 36, ""), "end" => ("End", "End", 35, ""), "a" => ("a", "KeyA", 65, "a"), "c" => ("c", "KeyC", 67, "c"), "v" => ("v", "KeyV", 86, "v"), "x" => ("x", "KeyX", 88, "x"), "f" => ("f", "KeyF", 70, "f"), _ => return None, }) } /// Press a key or combination ("Enter", "ctrl+a", "shift+tab"). fn press(tab: &mut Tab, combo: &str) -> Result<(), String> { let parts: Vec<&str> = combo.split('+').map(str::trim).filter(|s| !s.is_empty()).collect(); let (key, mods) = parts.split_last().ok_or("請給按鍵,例如 Enter")?; let mut m = 0; for x in mods { m |= match x.to_lowercase().as_str() { "alt" | "option" => 1, "ctrl" | "control" => 2, "cmd" | "meta" | "command" => 4, "shift" => 8, other => return Err(format!("不認得的按鍵「{other}」")), }; } let (k, code, kc, text) = key_info(key).ok_or_else(|| format!("不認得的按鍵「{key}」,可以用 Enter、Tab、Escape、Backspace、方向鍵、PageDown、Home、End,或 ctrl+a 這類組合"))?; let text = if m & (1 | 2 | 4) != 0 { "" } else { text }; tab.call("Input.dispatchKeyEvent", json!({"type": if text.is_empty() { "rawKeyDown" } else { "keyDown" }, "key": k, "code": code, "windowsVirtualKeyCode": kc, "modifiers": m, "text": text}))?; tab.call("Input.dispatchKeyEvent", json!({"type": "keyUp", "key": k, "code": code, "windowsVirtualKeyCode": kc, "modifiers": m}))?; Ok(()) } fn settle(ms: u64) { std::thread::sleep(Duration::from_millis(ms)); } /// What the page shows, cheaply: load state, address and how much text. const SIG_JS: &str = "document.readyState+'|'+location.href+'|'+(document.body?document.body.innerText.length:0)"; fn page_sig(tab: &mut Tab) -> String { tab.eval(SIG_JS).ok().and_then(|v| v.as_str().map(String::from)).unwrap_or_default() } /// Wait for an action's effect, not a fixed time: back as soon as the page has changed and holds /// still (loaded, same address and text twice in a row), or when nothing has started to change /// within `quiet_ms`; never longer than `max_ms`. (Fixed sleeps read slow pages too early and /// kept the bot waiting on fast ones.) pub fn settle_after(tab: &mut Tab, before: &str, quiet_ms: u64, max_ms: u64) { let start = Instant::now(); let mut last = String::new(); let mut changed = false; loop { settle(150); let now = page_sig(tab); changed |= now != before; let elapsed = start.elapsed().as_millis() as u64; let still = now.starts_with("complete|") && now == last; if (still && (changed || elapsed >= quiet_ms)) || elapsed >= max_ms { return; } last = now; } } /// Wait until the tab in front has loaded a real page (not the blank tab it starts as) and holds /// still, at most `max_ms`. Says whether it did. Connects afresh each look: a navigation can swap /// the page behind the tab. pub fn wait_loaded(profile: Option<&str>, max_ms: u64) -> bool { let start = Instant::now(); let mut last = String::new(); while (start.elapsed().as_millis() as u64) < max_ms { let now = Tab::current(profile).map(|mut t| { let sig = page_sig(&mut t); t.close(); sig }); let now = now.unwrap_or_default(); if now.starts_with("complete|") && !now.contains("|about:blank|") && now == last { return true; } last = now; settle(200); } false } // ---------- the tools ---------- fn s(args: &Value, k: &str) -> String { args.get(k).map(|v| v.as_str().map(String::from).unwrap_or_else(|| if v.is_null() { String::new() } else { v.to_string() })).unwrap_or_default().trim().to_string() } pub const NAMES: [&str; 18] = [ "browser_elements", "browser_click", "browser_hover", "browser_type", "browser_press", "browser_select", "browser_scroll", "browser_wait", "browser_find", "browser_table", "browser_screenshot", "browser_tabs", "browser_back", "browser_upload", "browser_download", "browser_pdf", "browser_dialog", "browser_eval", ]; pub fn is_tool(name: &str) -> bool { NAMES.contains(&name) } pub fn label(name: &str) -> &'static str { match name { "browser_elements" => "列出網頁上的按鈕", "browser_click" => "在瀏覽器點擊", "browser_hover" => "在瀏覽器停在上面", "browser_type" => "在瀏覽器輸入", "browser_press" => "在瀏覽器按鍵", "browser_select" => "在瀏覽器選擇", "browser_scroll" => "捲動瀏覽器", "browser_wait" => "等網頁載入", "browser_find" => "在網頁上找字", "browser_table" => "讀網頁表格", "browser_screenshot" => "瀏覽器截圖", "browser_tabs" => "瀏覽器分頁", "browser_back" => "瀏覽器上一頁", "browser_upload" => "上傳檔案到網頁", "browser_download" => "從網頁下載", "browser_pdf" => "把網頁存成 PDF", "browser_dialog" => "處理網頁對話框", "browser_eval" => "在網頁執行程式", _ => "瀏覽器", } } pub fn describe(name: &str, args: &str) -> String { let v: Value = serde_json::from_str(args).unwrap_or(Value::Null); let target = if !s(&v, "ref").is_empty() { format!("[{}]", s(&v, "ref")) } else { s(&v, "text") }; match name { "browser_type" => format!("{target} ← {}", s(&v, "value").chars().take(40).collect::()), "browser_select" => format!("{target} → {}", s(&v, "option")), "browser_press" => s(&v, "key"), "browser_scroll" => if target.is_empty() { s(&v, "direction") } else { target }, "browser_wait" => if s(&v, "text").is_empty() { format!("{} 秒", s(&v, "seconds")) } else { format!("等「{}」", s(&v, "text")) }, "browser_find" => format!("「{}」", s(&v, "text")), "browser_tabs" => format!("{} {}", s(&v, "action"), s(&v, "index")), "browser_back" => s(&v, "direction"), "browser_upload" => s(&v, "path"), "browser_eval" => s(&v, "script").chars().take(60).collect(), _ => target, } } /// The one browser tool's name (H2: every browser action rides on it, like `computer`). pub const TOOL: &str = "browser"; /// action -> the internal per-action tool it runs as. The internal names stay everywhere /// downstream (approvals, oplog, the work log, remote limbs): the model only ever sees `browser`. pub fn action_tool(action: &str) -> Option<&'static str> { Some(match action { "open" => "browser_open", "read" => "browser_read", "profiles" => "browser_profiles", "elements" => "browser_elements", "find" => "browser_find", "table" => "browser_table", "screenshot" => "browser_screenshot", "click" => "browser_click", "hover" => "browser_hover", "type" => "browser_type", "press" => "browser_press", "select" => "browser_select", "scroll" => "browser_scroll", "wait" => "browser_wait", "tabs" => "browser_tabs", "back" => "browser_back", "upload" => "browser_upload", "download" => "browser_download", "pdf" => "browser_pdf", "dialog" => "browser_dialog", "eval" => "browser_eval", "fill" => "browser_fill_login", // the kind picks login/card/contact; see expand_call _ => return None, }) } /// The one browser tool. One schema, trained-prior shapes (open/read/click/type…); the old /// eighteen schemas' guidance lives in the action list below. pub fn tool(profile: &Value) -> Value { json!({ "name": TOOL, "description": "Drive AnyBot's browser: one tool, picked by action. For anything on a web page use this, never screen coordinates. Start with open (it answers like read) or read, then act by the numbers (ref) or visible text the last result showed. Ask the person before anything that buys, pays, sends, posts or deletes. actions: - open {url}: open a link in a new tab and report the page (title, address, text, numbered controls) - read {max_chars}: read the current tab the same way (numbers for click/type/select) - elements: just the numbered clickable/fillable controls (cheaper than read) - find {text}: like Cmd-F: every place the text appears with words around it - table {index, all, max_rows}: read the page's tables as Markdown (reports, product lists) - screenshot {full, marks}: the page as a picture with numbered boxes on every control; needs no screen permission - click {ref|text, double, button}: real mouse click (menus, date pickers, dropdowns, calendar days) - hover {ref|text}: move over a control (opens hover menus) and report what appeared - type {ref|text, value, append, submit}: real keystrokes into a field (clears first unless append). Never type passwords here: use fill - press {key}: Enter, Tab, Escape, ctrl+a, … (optionally on ref/text first) - select {ref|text, option}: choose a dropdown option (custom dropdowns included) - scroll {direction: down|up|top|bottom} (on ref/text brings it into view; loads endless lists) - wait {text | seconds}: until text appears (≤20 s) or a few seconds - tabs {action: list|switch|new|close, index, url}: the tab strip - back {direction: back|forward|reload} - upload {ref|text, path}: put a file from this computer into a file chooser - download {ref|text}: click a download/export button, save to Downloads/AnyBot, return the path - pdf: save the page as a PDF in Downloads/AnyBot - dialog {accept, text}: answer a pop-up dialog the page opened - eval {script}: short read-only JavaScript for data the other actions can't reach; it can't send requests, navigate or submit (site APIs get accounts flagged) - fill {kind: login|card|contact, id, site, submit}: sign in / pay card / address from the person's 保險庫 — you never see the secret; the person approves every use (login_id/card_id/contact_id come from vault_list; default login: the current site's)", "parameters": {"type": "object", "properties": { "action": {"type": "string", "enum": ["open","read","elements","find","table","screenshot","click","hover","type","press","select","scroll","wait","tabs","back","upload","download","pdf","dialog","eval","profiles","fill"]}, "url": {"type": "string"}, "max_chars": {"type": "integer", "description": "read: longest text to return (default 20000)"}, "text": {"type": "string", "description": "find/wait: the text; click/type/…: the control's visible text"}, "ref": {"type": "string", "description": "The element's number from the latest read/elements/screenshot result"}, "index": {"type": "integer", "description": "table: which table; tabs: which tab"}, "all": {"type": "boolean", "description": "table: every table"}, "max_rows": {"type": "integer"}, "full": {"type": "boolean", "description": "screenshot: the whole page"}, "marks": {"type": "boolean", "description": "screenshot: false for no number boxes"}, "double": {"type": "boolean", "description": "click: double-click"}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "value": {"type": "string", "description": "type: the text"}, "append": {"type": "boolean", "description": "type: keep what's in the field"}, "submit": {"type": "boolean", "description": "type/fill: press Enter / sign in afterwards"}, "key": {"type": "string", "description": "press: Enter, Tab, Escape, ctrl+a…"}, "option": {"type": "string", "description": "select: the option's visible text"}, "direction": {"type": "string", "enum": ["down", "up", "top", "bottom", "back", "forward", "reload"], "description": "scroll: down/up/top/bottom; back: back/forward/reload"}, "seconds": {"type": "number"}, "path": {"type": "string", "description": "upload: a file on this computer"}, "accept": {"type": "boolean", "description": "dialog"}, "script": {"type": "string", "description": "eval: read-only JavaScript"}, "kind": {"type": "string", "enum": ["login", "card", "contact"], "description": "fill: what to fill"}, "id": {"type": "string", "description": "fill: the vault item's id from vault_list"}, "site": {"type": "string", "description": "fill login: domain (default: the current tab's)"}, "name": {"type": "string", "description": "profiles: name for add/use"}, "profile": profile }, "required": ["action"]} }) } /// Expand one `browser` call into the internal per-action tool at the model-output boundary, /// so every guard downstream (approvals incl. the vault's, oplog, replay, the remote link, the /// work log) keeps its stable per-action name. pub fn expand_call(c: &mut crate::model::ToolCall) { if c.name != TOOL { return; } let mut v = crate::tools::args_of(&c.args); let Some(action) = v["action"].as_str().and_then(action_tool) else { return }; let mut name = action.to_string(); if let Some(o) = v.as_object_mut() { o.remove("action"); if name == "browser_fill_login" { // fill {kind, id} names the exact vault tool and its id field. let kind = o.remove("kind").and_then(|k| k.as_str().map(|s| s.to_string())).unwrap_or_else(|| "login".into()); let id = o.remove("id"); name = match kind.as_str() { "card" => { if let Some(id) = id { o.insert("card_id".to_string(), id); } "browser_fill_card" } "contact" => { if let Some(id) = id { o.insert("contact_id".to_string(), id); } "browser_fill_contact" } _ => { if let Some(id) = id { o.insert("login_id".to_string(), id); } "browser_fill_login" } } .to_string(); } } c.name = name; c.args = v.to_string(); } fn downloads_dir() -> std::path::PathBuf { let d = dirs::download_dir().unwrap_or_else(std::env::temp_dir).join("AnyBot"); let _ = std::fs::create_dir_all(&d); d } /// Run one browser tool. Returns (text for the bot, a JPEG as base64 when it took a picture). pub fn run(name: &str, args: &Value) -> Result<(String, Option), String> { let profile = Some(s(args, "profile")).filter(|p| !p.is_empty()); let profile = profile.as_deref(); let (rf, text) = (s(args, "ref"), s(args, "text")); match name { "browser_tabs" => return tabs(args, profile).map(|t| (t, None)), "browser_wait" if s(args, "text").is_empty() => { let secs = args["seconds"].as_f64().unwrap_or(2.0).clamp(0.2, 30.0); settle((secs * 1000.0) as u64); return Ok((format!("已等 {secs} 秒。"), None)); } _ => {} } let mut tab = Tab::current(profile)?; let r = (|| -> Result<(String, Option), String> { match name { "browser_elements" => { let (title, url, list, more) = elements(&mut tab)?; Ok((format!("標題:{title}\n網址:{url}\n\n{}{}", list.join("\n"), if more { "\n…還有更多(捲動後再列一次)" } else { "" }), None)) } "browser_click" | "browser_hover" => { let sig = page_sig(&mut tab); let before = snapshot(&mut tab); let (x, y, what) = locate(&mut tab, &rf, &text)?; settle(150); if name == "browser_hover" { // Travel onto it (a jump straight there can skip the hover handlers). mouse(&mut tab, "mouseMoved", x - 40.0, y - 20.0, "none", 0)?; settle(60); mouse(&mut tab, "mouseMoved", x - 10.0, y - 4.0, "none", 0)?; settle(60); mouse(&mut tab, "mouseMoved", x, y, "none", 0)?; // Chrome may not update hover for a window that isn't in front: fire the // hover events on the element too. tab.eval(&format!("(() => {{ const el = {}; if (!el) return 0; const r = el.getBoundingClientRect(); const o = {{bubbles: true, cancelable: true, view: window, clientX: r.left + r.width / 2, clientY: r.top + r.height / 2}}; for (const t of ['pointerover', 'pointerenter', 'mouseover', 'mouseenter', 'pointermove', 'mousemove']) el.dispatchEvent(new (t.startsWith('pointer') ? PointerEvent : MouseEvent)(t, o)); return 1; }})()", find_js(&rf, &text)))?; } else { let button = match s(args, "button").as_str() { "right" => "right", "middle" => "middle", _ => "left", }; click_at(&mut tab, x, y, button, if args["double"].as_bool().unwrap_or(false) { 2 } else { 1 })?; } settle_after(&mut tab, &sig, 900, 4000); let after = snapshot(&mut tab); Ok((format!("已{} {what}。\n{}", if name == "browser_hover" { "停在" } else { "點" }, changes(&before, &after)), None)) } "browser_type" => { let sig = page_sig(&mut tab); let before = snapshot(&mut tab); let (x, y, what) = locate(&mut tab, &rf, &text)?; click_at(&mut tab, x, y, "left", 1)?; settle(150); if !args["append"].as_bool().unwrap_or(false) { // Select what's there so typing replaces it (works in inputs and editors). tab.eval("(() => { const e = document.activeElement; if (e && 'select' in e && typeof e.select === 'function') e.select(); else document.execCommand('selectAll'); return 1; })()")?; press(&mut tab, "Backspace")?; } tab.call("Input.insertText", json!({"text": s(args, "value")}))?; if args["submit"].as_bool().unwrap_or(false) { settle(150); press(&mut tab, "Enter")?; } settle_after(&mut tab, &sig, 700, 4000); let after = snapshot(&mut tab); let now: String = tab.eval("(() => { const e = document.activeElement; return e && 'value' in e ? (e.type === 'password' ? '' : String(e.value)) : ''; })()")?.as_str().unwrap_or("").chars().take(80).collect(); Ok((format!("已在 {what} 輸入。{}\n{}", if now.is_empty() { String::new() } else { format!("欄位現在是「{now}」。") }, changes(&before, &after)), None)) } "browser_press" => { let sig = page_sig(&mut tab); let before = snapshot(&mut tab); if !rf.is_empty() || !text.is_empty() { let (x, y, _) = locate(&mut tab, &rf, &text)?; click_at(&mut tab, x, y, "left", 1)?; settle(150); } press(&mut tab, &s(args, "key"))?; settle_after(&mut tab, &sig, 700, 4000); let after = snapshot(&mut tab); Ok((format!("已按 {}。\n{}", s(args, "key"), changes(&before, &after)), None)) } "browser_select" => { let option = s(args, "option"); let sig = page_sig(&mut tab); let before = snapshot(&mut tab); let js = format!( r#"(() => {{ const el = {}; if (!el) return 'missing'; if (el.tagName !== 'SELECT') return 'custom'; const want = {}.toLowerCase(); const o = [...el.options].find(o => o.text.trim().toLowerCase() === want || o.value.toLowerCase() === want) || [...el.options].find(o => o.text.trim().toLowerCase().includes(want)); if (!o) return 'no-option:' + [...el.options].map(o => o.text.trim()).join(' / '); Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'value').set.call(el, o.value); el.dispatchEvent(new Event('input', {{bubbles: true}})); el.dispatchEvent(new Event('change', {{bubbles: true}})); return 'ok'; }})()"#, find_js(&rf, &text), serde_json::to_string(&option).unwrap_or_default() ); let r = tab.eval(&js)?.as_str().unwrap_or("").to_string(); if r == "missing" { return Err("這一頁找不到那個下拉選單。用 browser_elements 看最新的清單。".into()); } if let Some(opts) = r.strip_prefix("no-option:") { return Err(format!("沒有「{option}」這個選項。可選:{opts}")); } if r == "custom" { // A custom dropdown: open it with a real click, then click the option. let (x, y, _) = locate(&mut tab, &rf, &text)?; click_at(&mut tab, x, y, "left", 1)?; settle(700); let (ox, oy, _) = locate(&mut tab, "", &option).map_err(|_| format!("已打開選單,但找不到「{option}」這個選項。用 browser_elements 看選單裡有什麼。"))?; click_at(&mut tab, ox, oy, "left", 1)?; } settle_after(&mut tab, &sig, 700, 4000); let after = snapshot(&mut tab); Ok((format!("已選「{option}」。\n{}", changes(&before, &after)), None)) } "browser_scroll" => { let sig = page_sig(&mut tab); let before = snapshot(&mut tab); if !rf.is_empty() || !text.is_empty() { let (_, _, what) = locate(&mut tab, &rf, &text)?; settle(500); let after = snapshot(&mut tab); return Ok((format!("已捲到 {what}。\n{}", changes(&before, &after)), None)); } let js = match s(args, "direction").as_str() { "up" => "window.scrollBy(0, -innerHeight * 0.85)", "top" => "window.scrollTo(0, 0)", "bottom" => "window.scrollTo(0, document.documentElement.scrollHeight)", _ => "window.scrollBy(0, innerHeight * 0.85)", }; // Pages that scroll an inner box: scroll the biggest scrollable element too. tab.eval(&format!("(() => {{ {js}; const boxes = [...document.querySelectorAll('*')].filter(e => e.scrollHeight > e.clientHeight + 40 && /(auto|scroll)/.test(getComputedStyle(e).overflowY)).sort((a, b) => b.clientHeight - a.clientHeight); if (boxes[0]) {{ const b = boxes[0]; const d = '{}'; if (d === 'top') b.scrollTop = 0; else if (d === 'bottom') b.scrollTop = b.scrollHeight; else b.scrollTop += (d === 'up' ? -1 : 1) * b.clientHeight * 0.85; }} return 1; }})()", s(args, "direction")))?; settle_after(&mut tab, &sig, 500, 2500); let after = snapshot(&mut tab); let pos = tab.eval("JSON.stringify([Math.round(scrollY), document.documentElement.scrollHeight, innerHeight])")?; Ok((format!("已捲動(位置 {})。\n{}", pos.as_str().unwrap_or(""), changes(&before, &after)), None)) } "browser_wait" => { let want = s(args, "text"); let limit = args["seconds"].as_f64().unwrap_or(20.0).clamp(1.0, 60.0); let start = Instant::now(); while start.elapsed().as_secs_f64() < limit { let (_, _, lines) = snapshot(&mut tab); if lines.iter().any(|l| l.contains(&want)) { return Ok((format!("「{want}」出現了({:.1} 秒)。", start.elapsed().as_secs_f64()), None)); } settle(500); } Err(format!("等了 {limit} 秒,「{want}」沒有出現。")) } "browser_find" => { let want = s(args, "text"); let (_, url, lines) = snapshot(&mut tab); let hits: Vec = lines.iter().enumerate().filter(|(_, l)| l.to_lowercase().contains(&want.to_lowercase())).take(30).map(|(i, _)| { let ctx: Vec<&str> = lines[i.saturating_sub(2)..(i + 3).min(lines.len())].iter().map(String::as_str).collect(); format!("• {}", ctx.join(" | ").chars().take(300).collect::()) }).collect(); if hits.is_empty() { return Ok((format!("這一頁({url})找不到「{want}」。"), None)); } Ok((format!("「{want}」出現 {} 處:\n{}", hits.len(), hits.join("\n")), None)) } "browser_table" => table(&mut tab, args).map(|t| (t, None)), "browser_screenshot" => { let mut params = json!({"format": "jpeg", "quality": 75}); // Numbered boxes on every clickable thing in view (the default for a plain // screenshot): look once, then click by number. let marks = args["marks"].as_bool().unwrap_or(rf.is_empty() && text.is_empty()); let mut listed = vec![]; if marks { let (_, _, list, _) = elements(&mut tab)?; tab.eval(&MARKS_JS.replace("%DEEP%", DEEP_JS))?; let in_view: std::collections::HashSet = tab.eval(&format!("(() => {{ {DEEP_JS} return JSON.stringify(deepAll('[data-anybot-ref]').filter(e => {{ const r = e.getBoundingClientRect(); return r.bottom > 0 && r.top < innerHeight && r.right > 0 && r.left < innerWidth && r.width > 1; }}).map(e => e.getAttribute('data-anybot-ref'))); }})()"))?.as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or_default(); listed = list.into_iter().filter(|l| l.strip_prefix('[').and_then(|r| r.split(']').next()).map(|n| in_view.contains(n)).unwrap_or(false)).collect(); settle(120); } if !rf.is_empty() || !text.is_empty() { let v = tab.eval(&format!("(() => {{ const el = {}; if (!el) return null; el.scrollIntoView({{block: 'center'}}); const r = el.getBoundingClientRect(); return JSON.stringify([r.left + scrollX, r.top + scrollY, r.width, r.height]); }})()", find_js(&rf, &text)))?; let r: Vec = v.as_str().and_then(|s| serde_json::from_str(s).ok()).ok_or("找不到那個元素")?; settle(300); params["clip"] = json!({"x": r[0], "y": r[1], "width": r[2].max(1.0), "height": r[3].max(1.0), "scale": 1}); params["captureBeyondViewport"] = json!(true); } else if args["full"].as_bool().unwrap_or(false) { let m = tab.call("Page.getLayoutMetrics", json!({}))?; let (w, h) = (m["cssContentSize"]["width"].as_f64().unwrap_or(1280.0), m["cssContentSize"]["height"].as_f64().unwrap_or(2000.0).min(12000.0)); params["clip"] = json!({"x": 0, "y": 0, "width": w, "height": h, "scale": 1}); params["captureBeyondViewport"] = json!(true); } let r = tab.call("Page.captureScreenshot", params); if marks { let _ = tab.eval("(() => { const o = document.getElementById('__anybot_marks'); if (o) o.remove(); return 1; })()"); } let r = r?; let b64 = r["data"].as_str().ok_or("截圖失敗")?.to_string(); // Keep pictures a sensible size for the model. let b64 = shrink(&b64).unwrap_or(b64); let (title, url, _) = snapshot(&mut tab); let mut text_out = format!("AnyBot 瀏覽器分頁的截圖:{title}\n網址:{url}"); if marks && !listed.is_empty() { text_out.push_str(&format!("\n圖上紅框裡的數字就是元素編號,用 browser_click 的 ref 點它:\n{}", listed.join("\n"))); } Ok((text_out, Some(b64))) } "browser_back" => { let sig = page_sig(&mut tab); let before = snapshot(&mut tab); match s(args, "direction").as_str() { "forward" => tab.eval("history.forward()").map(|_| ())?, "reload" => tab.call("Page.reload", json!({})).map(|_| ())?, _ => tab.eval("history.back()").map(|_| ())?, } settle_after(&mut tab, &sig, 900, 5000); let after = snapshot(&mut tab); Ok((format!("好了。\n{}", changes(&before, &after)), None)) } "browser_upload" => { let path = s(args, "path"); // The browser opens the file from its own disk: Bot 的電腦's when it runs there. let in_box = botbrowser::using_box(); let full = if in_box { crate::cloudbox::box_path(&path) } else { crate::tools::expand_home(&path).display().to_string() }; let there = if in_box { crate::cloudbox::box_has_file(&full) } else { std::path::Path::new(&full).is_file() }; if !there { return Err(format!("這台電腦上沒有「{path}」這個檔案")); } let js = format!("(() => {{ let el = {}; if (el && el.tagName !== 'INPUT') el = el.querySelector('input[type=file]') || el.closest('label')?.querySelector('input[type=file]'); return el || document.querySelector('input[type=file]'); }})()", find_js(&rf, &text)); let r = tab.call("Runtime.evaluate", json!({"expression": js, "returnByValue": false}))?; let obj = r["result"]["objectId"].as_str().ok_or("這一頁找不到上傳檔案的欄位")?.to_string(); let before = snapshot(&mut tab); tab.call("DOM.setFileInputFiles", json!({"files": [full], "objectId": obj}))?; tab.eval(&format!("(() => {{ const el = {js}; if (el) el.dispatchEvent(new Event('change', {{bubbles: true}})); return 1; }})()"))?; settle(1000); // What the field holds now (a page that shows nothing about it left the bot // checking with five more steps), and what the page did with it. let held = tab.eval(&format!("(() => {{ const el = {js}; return el && el.files ? Array.from(el.files).map(f => f.name + '(' + f.size + ' 位元組)').join('、') : ''; }})()")).ok().and_then(|v| v.as_str().map(String::from)).unwrap_or_default(); if held.is_empty() { return Err(format!("放了 {full},但上傳欄位還是空的(這一頁可能要先按「上傳」之類的按鈕,或欄位不是這一個)。")); } let after = snapshot(&mut tab); let seen = changes(&before, &after); let seen = if seen.contains("沒有變化") { String::new() } else { format!("\n{seen}") }; Ok((format!("已把 {full} 放進上傳欄位,欄位裡現在是:{held}。{seen}"), None)) } "browser_download" => { // The file lands on the browser's own disk: Bot 的電腦's when it runs there. let in_box = botbrowser::using_box(); let dir = if in_box { crate::cloudbox::BOX_DOWNLOADS.to_string() } else { downloads_dir().display().to_string() }; if in_box { crate::cloudbox::box_mkdir(&dir)?; } let clicked_at = std::time::SystemTime::now() - Duration::from_secs(1); let set = tab.call("Browser.setDownloadBehavior", json!({"behavior": "allow", "downloadPath": dir, "eventsEnabled": false})).or_else(|_| tab.call("Page.setDownloadBehavior", json!({"behavior": "allow", "downloadPath": dir}))); // A Firefox that can't be told where saves to its own Downloads folder: watch that. let dir = match set { Ok(_) => dir, Err(_) if tab.is_firefox() && !in_box => dirs::download_dir().unwrap_or_else(std::env::temp_dir).display().to_string(), Err(e) => return Err(e), }; let (x, y, what) = locate(&mut tab, &rf, &text)?; click_at(&mut tab, x, y, "left", 1)?; let start = Instant::now(); while start.elapsed().as_secs() < 60 { settle(700); // A new file, or one written again since the click (same name replaced). let now: Vec = if in_box { crate::cloudbox::box_new_files(&dir, clicked_at.duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0)) } else { std::fs::read_dir(&dir).map(|r| r.flatten().filter(|e| e.metadata().and_then(|m| m.modified()).map(|t| t >= clicked_at).unwrap_or(false)).map(|e| e.path().display().to_string()).collect()).unwrap_or_default() }; if let Some(f) = now.iter().find(|p| !(p.ends_with(".crdownload") || p.ends_with(".tmp") || p.ends_with(".part"))) { return Ok((format!("已點 {what},下載好了:{f}(可以用 read_file 讀它)"), None)); } } Err(format!("已點 {what},但 60 秒內沒有下載到檔案(有些網站要先選格式或日期,或在新分頁產生報表)。")) } "browser_pdf" => { let r = tab.call("Page.printToPDF", json!({"printBackground": true}))?; use base64::Engine; let bytes = base64::engine::general_purpose::STANDARD.decode(r["data"].as_str().unwrap_or("")).map_err(|e| e.to_string())?; let (title, _, _) = snapshot(&mut tab); let safe: String = title.chars().filter(|c| !"/\\:*?\"<>|".contains(*c)).take(60).collect(); let name = format!("{}-{}.pdf", if safe.trim().is_empty() { "網頁".into() } else { safe }, chrono::Utc::now().with_timezone(&crate::routines::person_tz()).format("%Y%m%d-%H%M%S")); // Saved where the bot can open it: on Bot 的電腦 when that's the browser's computer. let path = if botbrowser::using_box() { let p = format!("{}/{name}", crate::cloudbox::BOX_DOWNLOADS); crate::cloudbox::box_write(&p, &bytes)?; p } else { let p = downloads_dir().join(name); std::fs::write(&p, bytes).map_err(|e| e.to_string())?; p.display().to_string() }; Ok((format!("已存成 PDF:{path}"), None)) } "browser_dialog" => { let mut p = json!({"accept": args["accept"].as_bool().unwrap_or(true)}); if !s(args, "text").is_empty() { p["promptText"] = json!(s(args, "text")); } tab.call("Page.handleJavaScriptDialog", p).map_err(|_| "現在沒有對話框".to_string())?; Ok(("已處理對話框。".into(), None)) } "browser_eval" => { let script = s(args, "script"); if let Some(bad) = eval_refusal(&script) { return Err(bad); } let v = tab.eval(&script)?; let out = if v.is_string() { v.as_str().unwrap_or("").to_string() } else { serde_json::to_string_pretty(&v).unwrap_or_default() }; Ok((out.chars().take(20000).collect(), None)) } other => Err(format!("沒有「{other}」這個工具")), } })(); tab.close(); r } /// browser_eval only reads the page: no requests to the site's servers, no navigation, no /// submitting. Calling a site's private APIs gets accounts flagged (Shopee answered with a captcha). pub fn eval_refusal(script: &str) -> Option { let l = script.to_lowercase().replace(' ', ""); let banned = ["fetch(", "xmlhttprequest", "sendbeacon", "websocket(", "eventsource(", ".submit(", "location=", "location.href=", "location.assign", "location.replace", "window.open(", "document.cookie=", "localstorage.setitem", "import(", "navigator.serviceworker", "window.fetch="]; banned.iter().find(|b| l.contains(*b)).map(|b| { format!("browser_eval 只能讀取頁面上已經有的東西,不能發送請求或改變網站(不允許 {b})。請像人一樣用 browser_click / browser_type / browser_select 操作頁面;直接呼叫網站的內部 API 會被當成機器人(例如跳出拼圖驗證)。") }) } /// Re-encode a screenshot no wider than 1280 px. fn shrink(b64: &str) -> Option { use base64::Engine; let bytes = base64::engine::general_purpose::STANDARD.decode(b64).ok()?; let img = image::load_from_memory(&bytes).ok()?; let img = if img.width() > 1280 { img.resize(1280, u32::MAX, image::imageops::FilterType::Triangle) } else { img }; let img = if img.height() > 4000 { img.crop_imm(0, 0, img.width(), 4000) } else { img }; let mut out = std::io::Cursor::new(vec![]); image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, 72).encode_image(&img.to_rgb8()).ok()?; Some(base64::engine::general_purpose::STANDARD.encode(out.into_inner())) } /// A job is done with this browser: keep the page it ended on, close the rest (the list is /// latest-used first). A browser that isn't open stays closed. Not offered to the model; the /// loop calls it (browser_tabs action "tidy") when a job ends. fn tidy(profile: Option<&str>) -> Result { let job = botbrowser::job(); // (Its tabs are nobody's from here on, whatever happens below.) let mine = if job.is_empty() { vec![] } else { botbrowser::release(&job) }; let Some(port) = botbrowser::running_port(profile) else { return Ok("瀏覽器沒有開著。".into()) }; let kind = botbrowser::kind_of(profile); let list = botbrowser::pages_of(kind, port)?; let close: Vec<&Value> = if job.is_empty() { // (An older brain that doesn't say which job: the old way, all but the front page.) list.iter().skip(1).collect() } else { // The job's own tabs but its latest; other jobs' and the person's are left alone. list.iter().filter(|t| mine.iter().any(|m| t["id"].as_str() == Some(m.as_str()))).skip(1).collect() }; let closed = close.iter().filter(|t| botbrowser::close_tab(kind, port, t["id"].as_str().unwrap_or("")).is_ok()).count(); Ok(format!("關了 {closed} 個用完的分頁。")) } fn tabs(args: &Value, profile: Option<&str>) -> Result { if s(args, "action") == "tidy" { return tidy(profile); } let (_, port, kind) = port_of(profile)?; // Another job's tabs aren't listed, switched to or closed. let list: Vec = botbrowser::pages_of(kind, port)?.into_iter().filter(|t| !botbrowser::others_tab(t["id"].as_str().unwrap_or(""))).collect(); let current = botbrowser::pick_tab(&list).and_then(|t| t["id"].as_str()).unwrap_or("").to_string(); let idx = args["index"].as_u64().map(|i| i as usize); let pick = |i: Option| -> Result<&Value, String> { i.and_then(|i| list.get(i.wrapping_sub(1))).ok_or_else(|| "請給分頁編號(先用 action=list 看)".to_string()) }; match s(args, "action").as_str() { "switch" => { let t = pick(idx)?; botbrowser::activate_tab(kind, port, t["id"].as_str().unwrap_or(""))?; botbrowser::claim(t["id"].as_str().unwrap_or("")); Ok(format!("已切到分頁 {}:{}({})", idx.unwrap_or(0), t["title"].as_str().unwrap_or(""), t["url"].as_str().unwrap_or(""))) } "close" => { let t = pick(idx)?; botbrowser::close_tab(kind, port, t["id"].as_str().unwrap_or(""))?; Ok(format!("已關閉分頁:{}", t["title"].as_str().unwrap_or(""))) } "new" => { let url = crate::web::normalize_url(&s(args, "url")).unwrap_or_else(|_| "about:blank".into()); let made = botbrowser::new_tab(kind, port, &url)?; botbrowser::claim(&made); let loaded = wait_loaded(profile, 10_000); Ok(format!("已開新分頁:{url}{}", if loaded { "" } else { "(還在載入,可以用 browser_wait 等它)" })) } _ => Ok(if list.is_empty() { "沒有開著的分頁。".into() } else { list.iter().enumerate().map(|(i, t)| format!("{}. {}{}\n {}", i + 1, t["title"].as_str().unwrap_or(""), if t["id"].as_str() == Some(current.as_str()) { "(目前)" } else { "" }, t["url"].as_str().unwrap_or(""))).collect::>().join("\n") }), } } /// HTML tables and ARIA grids as Markdown. fn table(tab: &mut Tab, args: &Value) -> Result { let js = r#"(() => { const clean = s => (s || '').replace(/\s+/g, ' ').trim().replace(/\|/g, '/'); const vis = el => { const r = el.getBoundingClientRect(); return r.width > 1 && r.height > 1; }; const tables = []; for (const t of document.querySelectorAll('table')) { if (!vis(t)) continue; const rows = [...t.querySelectorAll('tr')].map(r => [...r.children].filter(c => c.tagName === 'TD' || c.tagName === 'TH').map(c => clean(c.innerText))).filter(r => r.some(x => x)); if (rows.length) tables.push({caption: clean((t.caption || {}).innerText || t.getAttribute('aria-label') || ''), rows}); } for (const g of document.querySelectorAll('[role=grid],[role=table],[role=treegrid]')) { if (!vis(g) || g.tagName === 'TABLE') continue; const rows = [...g.querySelectorAll('[role=row]')].map(r => [...r.querySelectorAll('[role=columnheader],[role=cell],[role=gridcell],[role=rowheader]')].map(c => clean(c.innerText))).filter(r => r.some(x => x)); if (rows.length) tables.push({caption: clean(g.getAttribute('aria-label') || ''), rows}); } // Fixed-header tables split into a header table and a body table (common in admin pages): // glue a lone header row onto the next table with the same width. const out = []; for (let i = 0; i < tables.length; i++) { const t = tables[i], n = tables[i + 1]; if (t.rows.length === 1 && n && n.rows[0] && n.rows[0].length === t.rows[0].length) { out.push({caption: t.caption || n.caption, rows: t.rows.concat(n.rows)}); i++; } else out.push(t); } return JSON.stringify(out); })()"#; let v = tab.eval(js)?; let tables: Vec = v.as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or_default(); if tables.is_empty() { return Ok("這一頁沒有表格。數字如果在圖表裡,用 browser_screenshot 看,或用 browser_read 讀文字。".into()); } let max_rows = args["max_rows"].as_u64().unwrap_or(200) as usize; let pick: Vec<(usize, &Value)> = match (args["index"].as_u64(), args["all"].as_bool().unwrap_or(false)) { (Some(i), _) => tables.iter().enumerate().filter(|(j, _)| *j + 1 == i as usize).collect(), (None, true) => tables.iter().enumerate().collect(), (None, false) => tables.iter().enumerate().max_by_key(|(_, t)| t["rows"].as_array().map(|r| r.len()).unwrap_or(0)).into_iter().collect(), }; let mut out = format!("這一頁有 {} 個表格。\n", tables.len()); for (i, t) in pick { let rows: Vec> = t["rows"].as_array().map(|rs| rs.iter().map(|r| r.as_array().map(|c| c.iter().map(|x| x.as_str().unwrap_or("").to_string()).collect()).unwrap_or_default()).collect()).unwrap_or_default(); let width = rows.iter().map(|r| r.len()).max().unwrap_or(0); out.push_str(&format!("\n### 表格 {}{}({} 列)\n", i + 1, if t["caption"].as_str().unwrap_or("").is_empty() { String::new() } else { format!(":{}", t["caption"].as_str().unwrap_or("")) }, rows.len().saturating_sub(1))); for (k, r) in rows.iter().take(max_rows + 1).enumerate() { let mut cells = r.clone(); cells.resize(width, String::new()); out.push_str(&format!("| {} |\n", cells.join(" | "))); if k == 0 { out.push_str(&format!("|{}\n", "---|".repeat(width))); } } if rows.len() > max_rows + 1 { out.push_str(&format!("…還有 {} 列(可以捲動或換頁再讀)\n", rows.len() - max_rows - 1)); } } Ok(out) } #[cfg(test)] mod tests { #[test] fn the_one_tool_maps_every_action() { let mut c = |args: &str| { let mut call = crate::model::ToolCall { id: "x".into(), name: TOOL.into(), args: args.into(), status: crate::model::ToolStatus::Done }; expand_call(&mut call); (call.name, call.args) }; let (n, a) = c(r#"{"action":"click","ref":"12","double":true}"#); assert_eq!(n, "browser_click"); assert!(a.contains(r#""ref":"12""#) && a.contains("double") && !a.contains("action"), "{a}"); let (n, a) = c(r#"{"action":"fill","kind":"card","id":"7"}"#); assert_eq!(n, "browser_fill_card"); assert!(a.contains(r#""card_id":"7""#) && !a.contains("kind"), "{a}"); let (n, _) = c(r#"{"action":"fill"}"#); assert_eq!(n, "browser_fill_login"); let (n, _) = c(r#"{"action":"nope"}"#); assert_eq!(n, TOOL, "an unknown action stays put (the runner explains it)"); // Every internal browser tool is reachable from the one schema. let mut mapped: Vec<&str> = tool(&json!({"type": "string"}))["parameters"]["properties"]["action"]["enum"] .as_array() .unwrap() .iter() .map(|a| action_tool(a.as_str().unwrap()).unwrap()) .collect(); mapped.extend(["browser_fill_card", "browser_fill_contact"]); for n in NAMES { assert!(mapped.contains(&n), "internal tool {n} is unreachable from browser"); } } use super::*; #[test] fn keys_and_changes() { assert_eq!(key_info("Enter").unwrap().0, "Enter"); assert!(key_info("F13").is_none()); let a = ("t".to_string(), "u1".to_string(), vec!["a".to_string(), "b".to_string()]); let b = ("t".to_string(), "u2".to_string(), vec!["a".to_string(), "c".to_string()]); let c = changes(&a, &b); assert!(c.contains("網址變成:u2") && c.contains("新出現的文字:\nc")); assert!(changes(&a, &a).contains("沒有變化")); } #[test] fn the_front_page_tab_is_used() { let list = json!([ {"type": "service_worker", "url": "https://x", "webSocketDebuggerUrl": "ws://a"}, {"type": "page", "url": "devtools://devtools", "webSocketDebuggerUrl": "ws://b"}, {"type": "page", "url": "https://www.threads.net/", "title": "Threads", "webSocketDebuggerUrl": "ws://c"}, {"type": "page", "url": "https://example.com/", "webSocketDebuggerUrl": "ws://d"} ]); let p = pages(&list); assert_eq!(p.len(), 2); assert_eq!(p[0]["webSocketDebuggerUrl"], "ws://c"); } /// Controls inside web components (open shadow roots, two levels deep) get numbers and can be /// clicked and typed into, like Chromium's settings pages and many app frameworks. Headless: /// `ANYBOT_DATA_DIR= ANYBOT_BROWSER_ENGINE=firefox ANYBOT_FIREFOX_HEADLESS=1 cargo test /// numbers_controls_inside_web_components -- --ignored --nocapture` (or Chrome, with a window). #[test] #[ignore] fn numbers_controls_inside_web_components() { use std::io::{Read, Write}; let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let port = l.local_addr().unwrap().port(); std::thread::spawn(move || { for mut c in l.incoming().flatten() { let mut buf = [0u8; 4096]; let _ = c.read(&mut buf); let body = r#"元件

設定

還沒按

"#; let _ = write!(c, "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); let _ = c.write_all(body.as_bytes()); } }); crate::botbrowser::add("shadow-test").unwrap(); let p = |v: Value| { let mut v = v; v["profile"] = json!("shadow-test"); v }; let go = || -> Result<(), String> { crate::botbrowser::open(&format!("http://127.0.0.1:{port}/"), Some("shadow-test"))?; let (els, _) = run("browser_elements", &p(json!({})))?; println!("{els}"); for want in ["「名稱」", "「開始」", "「自動更新」"] { assert!(els.contains(want), "{want} missing: {els}"); } let n = els.lines().find(|l| l.contains("「開始」")).and_then(|l| l.trim_start().strip_prefix('[')).and_then(|r| r.split(']').next()).unwrap().to_string(); run("browser_type", &p(json!({"text": "名稱", "value": "小明"})))?; let (r, _) = run("browser_click", &p(json!({"ref": n})))?; assert!(r.contains("按了開始:小明"), "{r}"); let (r, _) = run("browser_click", &p(json!({"text": "自動更新"})))?; assert!(r.contains("自動更新:開"), "{r}"); Ok(()) }; let r = go(); let _ = crate::botbrowser::remove("shadow-test"); r.unwrap(); } /// A dashboard like Shopee's affiliate pages: a React-style date picker that only reacts to /// real clicks, a custom dropdown, a table, a hover menu, an export. Run with ANYBOT_DATA_DIR /// set to a scratch folder; opens a browser window for a moment. #[test] #[ignore] fn drives_a_dashboard() { use std::io::{Read, Write}; let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let port = l.local_addr().unwrap().port(); std::thread::spawn(move || { for mut c in l.incoming().flatten() { let mut buf = [0u8; 4096]; let n = c.read(&mut buf).unwrap_or(0); let req = String::from_utf8_lossy(&buf[..n]).to_string(); let (ctype, body): (&str, Vec) = if req.starts_with("GET /export") { ("text/csv\r\nContent-Disposition: attachment; filename=\"report.csv\"", "商品,佣金\nA,10\n".as_bytes().to_vec()) } else { ("text/html; charset=utf-8", r#"成效
2026-11-03
預設
商品分潤率
手機殼8%
乳液12%

點擊 0

匯出報表 "#.as_bytes().to_vec()) }; let _ = write!(c, "HTTP/1.1 200 OK\r\nContent-Type: {ctype}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); let _ = c.write_all(&body); } }); crate::botbrowser::add("drive-test").unwrap(); let p = |v: Value| { let mut v = v; v["profile"] = json!("drive-test"); v }; let go = || -> Result<(), String> { crate::botbrowser::open(&format!("http://127.0.0.1:{port}/"), Some("drive-test"))?; let (els, _) = run("browser_elements", &p(json!({})))?; assert!(els.contains("「數據周期」"), "{els}"); run("browser_hover", &p(json!({"text": "商品分析"})))?; let (shown, _) = run("browser_eval", &p(json!({"script": "getComputedStyle(document.getElementById('sub')).display"})))?; assert_eq!(shown.trim(), "block"); // Real click opens the picker; a calendar day (only a pointer cursor) is listed and clickable. let (r, _) = run("browser_click", &p(json!({"text": "數據周期"})))?; assert!(r.contains("近30天"), "{r}"); let (r, _) = run("browser_click", &p(json!({"text": "近30天"})))?; assert!(r.contains("點擊 60,訂單 3"), "{r}"); let (r, _) = run("browser_select", &p(json!({"text": "排序", "option": "佣金由高到低"})))?; assert!(r.contains("佣金由高到低"), "{r}"); let (r, _) = run("browser_type", &p(json!({"text": "搜尋商品", "value": "手機殼", "submit": true})))?; assert!(r.contains("搜尋:手機殼"), "{r}"); let (t, _) = run("browser_table", &p(json!({})))?; assert!(t.contains("| 商品 | 分潤率 |") && t.contains("| 乳液 | 12% |"), "{t}"); let (f, _) = run("browser_find", &p(json!({"text": "手機殼"})))?; assert!(f.contains("搜尋:手機殼") && f.contains("8%"), "{f}"); let (_, img) = run("browser_screenshot", &p(json!({})))?; assert!(img.map(|b| b.len() > 1000).unwrap_or(false)); let (tabs, _) = run("browser_tabs", &p(json!({"action": "list"})))?; assert!(tabs.contains("成效"), "{tabs}"); let (d, _) = run("browser_download", &p(json!({"text": "匯出報表"})))?; assert!(d.contains("report") && d.contains(".csv"), "{d}"); let (v, _) = run("browser_eval", &p(json!({"script": "document.querySelectorAll('[role=row]').length"})))?; assert_eq!(v.trim(), "3"); assert!(run("browser_eval", &p(json!({"script": "fetch('/api/x')"}))).is_err()); let (t, img) = run("browser_screenshot", &p(json!({})))?; assert!(img.is_some() && t.contains("紅框") && t.contains("[1]"), "{t}"); let (gone, _) = run("browser_eval", &p(json!({"script": "!document.getElementById('__anybot_marks')"})))?; assert_eq!(gone.trim(), "true"); Ok(()) }; let r = go(); let _ = crate::botbrowser::remove("drive-test"); r.unwrap(); } }