Merge h2/browser-tools: one browser tool (H2)
This commit is contained in:
daniel wang
commit
4f3f096e37
|
|
@ -1054,6 +1054,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc<
|
|||
// approvals, replay, the remote link, the work log) keeps its stable per-action name.
|
||||
for c in out.tool_calls.iter_mut() {
|
||||
computer::expand_call(c);
|
||||
crate::browser::expand_call(c);
|
||||
}
|
||||
let quick = if plan.quick_is_off() { None } else { tried_quick };
|
||||
let via = if quick.is_none() && scripted.is_none() { job.via.clone() } else { via };
|
||||
|
|
|
|||
210
src/browser.rs
210
src/browser.rs
|
|
@ -479,53 +479,140 @@ pub fn describe(name: &str, args: &str) -> String {
|
|||
}
|
||||
}
|
||||
|
||||
/// The tools' definitions (the model reads these).
|
||||
pub fn specs(profile: &Value) -> Vec<Value> {
|
||||
let target = |extra: Value| {
|
||||
let mut p = json!({"ref": {"type": "string", "description": "The element's number from the latest browser_read / browser_elements / action result"}, "text": {"type": "string", "description": "Or its visible text, label or placeholder"}, "profile": profile});
|
||||
if let (Some(o), Some(e)) = (p.as_object_mut(), extra.as_object()) {
|
||||
o.extend(e.clone());
|
||||
/// The one browser tool's name (H2: every browser action rides on it, like `computer`).
|
||||
pub const TOOL: &str = "browser";
|
||||
|
||||
/// action -> the internal per-action tool it runs as. The internal names stay everywhere
|
||||
/// downstream (approvals, oplog, the work log, remote limbs): the model only ever sees `browser`.
|
||||
pub fn action_tool(action: &str) -> Option<&'static str> {
|
||||
Some(match action {
|
||||
"open" => "browser_open",
|
||||
"read" => "browser_read",
|
||||
"profiles" => "browser_profiles",
|
||||
"elements" => "browser_elements",
|
||||
"find" => "browser_find",
|
||||
"table" => "browser_table",
|
||||
"screenshot" => "browser_screenshot",
|
||||
"click" => "browser_click",
|
||||
"hover" => "browser_hover",
|
||||
"type" => "browser_type",
|
||||
"press" => "browser_press",
|
||||
"select" => "browser_select",
|
||||
"scroll" => "browser_scroll",
|
||||
"wait" => "browser_wait",
|
||||
"tabs" => "browser_tabs",
|
||||
"back" => "browser_back",
|
||||
"upload" => "browser_upload",
|
||||
"download" => "browser_download",
|
||||
"pdf" => "browser_pdf",
|
||||
"dialog" => "browser_dialog",
|
||||
"eval" => "browser_eval",
|
||||
"fill" => "browser_fill_login", // the kind picks login/card/contact; see expand_call
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
p
|
||||
};
|
||||
vec![
|
||||
json!({"name": "browser_elements", "description": "List everything that can be clicked or filled on the current tab of AnyBot's browser, numbered: links (with addresses), buttons, fields, dropdowns, tabs, menu items, calendar days and other clickable items. Items marked （要捲動） are off screen. Cheaper than browser_read when you only need the controls.",
|
||||
"parameters": {"type": "object", "properties": {"profile": profile}}}),
|
||||
json!({"name": "browser_click", "description": "Click an element in AnyBot's browser with a real mouse click (works on menus, date pickers, custom dropdowns, tabs, calendar days). By number or visible text. Returns what changed on the page. Use double for double-click, button=right for a context menu. Ask the person before anything that buys, pays, sends, posts or deletes.",
|
||||
"parameters": {"type": "object", "properties": target(json!({"double": {"type": "boolean"}, "button": {"type": "string", "enum": ["left", "right", "middle"]}}))}}),
|
||||
json!({"name": "browser_hover", "description": "Move the mouse over an element (opens hover menus and tooltips) and report what appeared.",
|
||||
"parameters": {"type": "object", "properties": target(json!({}))}}),
|
||||
json!({"name": "browser_type", "description": "Type into a field with real keystrokes (clears it first unless append is true). Works for search boxes, forms and date fields (type the date the way the field shows it, then press Enter). Never type passwords: use browser_fill_login.",
|
||||
"parameters": {"type": "object", "properties": target(json!({"value": {"type": "string"}, "append": {"type": "boolean"}, "submit": {"type": "boolean", "description": "Press Enter afterwards"}})), "required": ["value"]}}),
|
||||
json!({"name": "browser_press", "description": "Press a key or combination in AnyBot's browser: Enter, Tab, Escape, Backspace, ArrowDown, PageDown, Home, End, ctrl+a, shift+tab… Optionally on an element first (it gets focus).",
|
||||
"parameters": {"type": "object", "properties": target(json!({"key": {"type": "string"}})), "required": ["key"]}}),
|
||||
json!({"name": "browser_select", "description": "Choose an option in a dropdown. Native dropdowns are set directly; custom ones are opened and the option is clicked for you.",
|
||||
"parameters": {"type": "object", "properties": target(json!({"option": {"type": "string", "description": "The option's visible text, e.g. 近 30 天"}})), "required": ["option"]}}),
|
||||
json!({"name": "browser_scroll", "description": "Scroll the page (down/up/top/bottom), or bring an element into view by number or text. Use it to load more items on long or endless lists.",
|
||||
"parameters": {"type": "object", "properties": target(json!({"direction": {"type": "string", "enum": ["down", "up", "top", "bottom"]}}))}}),
|
||||
json!({"name": "browser_wait", "description": "Wait until some text appears on the page (up to 20 s), or just wait some seconds, e.g. after starting a report or a slow search.",
|
||||
"parameters": {"type": "object", "properties": {"text": {"type": "string"}, "seconds": {"type": "number"}, "profile": profile}}}),
|
||||
json!({"name": "browser_find", "description": "Find text on the current page and return each place with the words around it (like ⌘F). Good for checking a number or a product name without reading the whole page.",
|
||||
"parameters": {"type": "object", "properties": {"text": {"type": "string"}, "profile": profile}, "required": ["text"]}}),
|
||||
json!({"name": "browser_table", "description": "Read the tables (and table-like grids) on the current page as Markdown, with headers. Use it for reports, product lists, orders and rankings instead of reading the raw text. index picks one table; all=true reads every table.",
|
||||
"parameters": {"type": "object", "properties": {"index": {"type": "integer"}, "all": {"type": "boolean"}, "max_rows": {"type": "integer"}, "profile": profile}}}),
|
||||
json!({"name": "browser_screenshot", "description": "See the current tab of AnyBot's browser as a picture with a numbered box on every clickable thing (the numbers are browser_click refs) — the quickest way to find where to click: look once, then click by number. Also for charts, layouts and captchas. full=true takes the whole page; ref/text takes just one element; marks=false for a clean picture. Needs no screen-recording permission.",
|
||||
"parameters": {"type": "object", "properties": target(json!({"full": {"type": "boolean"}, "marks": {"type": "boolean"}}))}}),
|
||||
json!({"name": "browser_tabs", "description": "The tabs of AnyBot's browser: list them, switch to one (index), open a new one (url), or close one (index).",
|
||||
"parameters": {"type": "object", "properties": {"action": {"type": "string", "enum": ["list", "switch", "new", "close"]}, "index": {"type": "integer"}, "url": {"type": "string"}, "profile": profile}, "required": ["action"]}}),
|
||||
json!({"name": "browser_back", "description": "Go back, forward or reload the current tab.",
|
||||
"parameters": {"type": "object", "properties": {"direction": {"type": "string", "enum": ["back", "forward", "reload"]}, "profile": profile}}}),
|
||||
json!({"name": "browser_upload", "description": "Put a file from this computer into a page's file chooser (the field listed as 上傳檔案). path is a full path on this computer.",
|
||||
"parameters": {"type": "object", "properties": target(json!({"path": {"type": "string"}})), "required": ["path"]}}),
|
||||
json!({"name": "browser_download", "description": "Click a download/export button (by number or text) and wait for the file; returns where it was saved (Downloads/AnyBot). Use for 匯出報表 / Export CSV, then read the file.",
|
||||
"parameters": {"type": "object", "properties": target(json!({}))}}),
|
||||
json!({"name": "browser_pdf", "description": "Save the current page as a PDF in Downloads/AnyBot and return its path.",
|
||||
"parameters": {"type": "object", "properties": {"profile": profile}}}),
|
||||
json!({"name": "browser_dialog", "description": "Answer a pop-up dialog the page opened (alert / confirm / prompt) when the browser stops responding: accept or dismiss, with text for prompts.",
|
||||
"parameters": {"type": "object", "properties": {"accept": {"type": "boolean"}, "text": {"type": "string"}, "profile": profile}, "required": ["accept"]}}),
|
||||
json!({"name": "browser_eval", "description": "Run a short read-only JavaScript expression on the current page and return its value (JSON), for data the other tools can't reach (e.g. values inside a chart). It can't send requests, navigate or submit: never call a site's own APIs or download its code — that gets the account flagged as a bot.",
|
||||
"parameters": {"type": "object", "properties": {"script": {"type": "string"}, "profile": profile}, "required": ["script"]}}),
|
||||
]
|
||||
|
||||
/// The one browser tool. One schema, trained-prior shapes (open/read/click/type…); the old
|
||||
/// eighteen schemas' guidance lives in the action list below.
|
||||
pub fn tool(profile: &Value) -> Value {
|
||||
json!({
|
||||
"name": TOOL,
|
||||
"description": "Drive AnyBot's browser: one tool, picked by action. For anything on a web page use this, never screen coordinates. Start with open (it answers like read) or read, then act by the numbers (ref) or visible text the last result showed. Ask the person before anything that buys, pays, sends, posts or deletes.
|
||||
actions:
|
||||
- open {url}: open a link in a new tab and report the page (title, address, text, numbered controls)
|
||||
- read {max_chars}: read the current tab the same way (numbers for click/type/select)
|
||||
- elements: just the numbered clickable/fillable controls (cheaper than read)
|
||||
- find {text}: like Cmd-F: every place the text appears with words around it
|
||||
- table {index, all, max_rows}: read the page's tables as Markdown (reports, product lists)
|
||||
- screenshot {full, marks}: the page as a picture with numbered boxes on every control; needs no screen permission
|
||||
- click {ref|text, double, button}: real mouse click (menus, date pickers, dropdowns, calendar days)
|
||||
- hover {ref|text}: move over a control (opens hover menus) and report what appeared
|
||||
- type {ref|text, value, append, submit}: real keystrokes into a field (clears first unless append). Never type passwords here: use fill
|
||||
- press {key}: Enter, Tab, Escape, ctrl+a, … (optionally on ref/text first)
|
||||
- select {ref|text, option}: choose a dropdown option (custom dropdowns included)
|
||||
- scroll {direction: down|up|top|bottom} (on ref/text brings it into view; loads endless lists)
|
||||
- wait {text | seconds}: until text appears (≤20 s) or a few seconds
|
||||
- tabs {action: list|switch|new|close, index, url}: the tab strip
|
||||
- back {direction: back|forward|reload}
|
||||
- upload {ref|text, path}: put a file from this computer into a file chooser
|
||||
- download {ref|text}: click a download/export button, save to Downloads/AnyBot, return the path
|
||||
- pdf: save the page as a PDF in Downloads/AnyBot
|
||||
- dialog {accept, text}: answer a pop-up dialog the page opened
|
||||
- eval {script}: short read-only JavaScript for data the other actions can't reach; it can't send requests, navigate or submit (site APIs get accounts flagged)
|
||||
- fill {kind: login|card|contact, id, site, submit}: sign in / pay card / address from the person's 保險庫 — you never see the secret; the person approves every use (login_id/card_id/contact_id come from vault_list; default login: the current site's)",
|
||||
"parameters": {"type": "object", "properties": {
|
||||
"action": {"type": "string", "enum": ["open","read","elements","find","table","screenshot","click","hover","type","press","select","scroll","wait","tabs","back","upload","download","pdf","dialog","eval","profiles","fill"]},
|
||||
"url": {"type": "string"},
|
||||
"max_chars": {"type": "integer", "description": "read: longest text to return (default 20000)"},
|
||||
"text": {"type": "string", "description": "find/wait: the text; click/type/…: the control's visible text"},
|
||||
"ref": {"type": "string", "description": "The element's number from the latest read/elements/screenshot result"},
|
||||
"index": {"type": "integer", "description": "table: which table; tabs: which tab"},
|
||||
"all": {"type": "boolean", "description": "table: every table"},
|
||||
"max_rows": {"type": "integer"},
|
||||
"full": {"type": "boolean", "description": "screenshot: the whole page"},
|
||||
"marks": {"type": "boolean", "description": "screenshot: false for no number boxes"},
|
||||
"double": {"type": "boolean", "description": "click: double-click"},
|
||||
"button": {"type": "string", "enum": ["left", "right", "middle"]},
|
||||
"value": {"type": "string", "description": "type: the text"},
|
||||
"append": {"type": "boolean", "description": "type: keep what's in the field"},
|
||||
"submit": {"type": "boolean", "description": "type/fill: press Enter / sign in afterwards"},
|
||||
"key": {"type": "string", "description": "press: Enter, Tab, Escape, ctrl+a…"},
|
||||
"option": {"type": "string", "description": "select: the option's visible text"},
|
||||
"direction": {"type": "string", "enum": ["down", "up", "top", "bottom", "back", "forward", "reload"], "description": "scroll: down/up/top/bottom; back: back/forward/reload"},
|
||||
"seconds": {"type": "number"},
|
||||
"path": {"type": "string", "description": "upload: a file on this computer"},
|
||||
"accept": {"type": "boolean", "description": "dialog"},
|
||||
"script": {"type": "string", "description": "eval: read-only JavaScript"},
|
||||
"kind": {"type": "string", "enum": ["login", "card", "contact"], "description": "fill: what to fill"},
|
||||
"id": {"type": "string", "description": "fill: the vault item's id from vault_list"},
|
||||
"site": {"type": "string", "description": "fill login: domain (default: the current tab's)"},
|
||||
"name": {"type": "string", "description": "profiles: name for add/use"},
|
||||
"profile": profile
|
||||
}, "required": ["action"]}
|
||||
})
|
||||
}
|
||||
|
||||
/// Expand one `browser` call into the internal per-action tool at the model-output boundary,
|
||||
/// so every guard downstream (approvals incl. the vault's, oplog, replay, the remote link, the
|
||||
/// work log) keeps its stable per-action name.
|
||||
pub fn expand_call(c: &mut crate::model::ToolCall) {
|
||||
if c.name != TOOL {
|
||||
return;
|
||||
}
|
||||
let mut v = crate::tools::args_of(&c.args);
|
||||
let Some(action) = v["action"].as_str().and_then(action_tool) else { return };
|
||||
let mut name = action.to_string();
|
||||
if let Some(o) = v.as_object_mut() {
|
||||
o.remove("action");
|
||||
if name == "browser_fill_login" {
|
||||
// fill {kind, id} names the exact vault tool and its id field.
|
||||
let kind = o.remove("kind").and_then(|k| k.as_str().map(|s| s.to_string())).unwrap_or_else(|| "login".into());
|
||||
let id = o.remove("id");
|
||||
name = match kind.as_str() {
|
||||
"card" => {
|
||||
if let Some(id) = id {
|
||||
o.insert("card_id".to_string(), id);
|
||||
}
|
||||
"browser_fill_card"
|
||||
}
|
||||
"contact" => {
|
||||
if let Some(id) = id {
|
||||
o.insert("contact_id".to_string(), id);
|
||||
}
|
||||
"browser_fill_contact"
|
||||
}
|
||||
_ => {
|
||||
if let Some(id) = id {
|
||||
o.insert("login_id".to_string(), id);
|
||||
}
|
||||
"browser_fill_login"
|
||||
}
|
||||
}
|
||||
.to_string();
|
||||
}
|
||||
}
|
||||
c.name = name;
|
||||
c.args = v.to_string();
|
||||
}
|
||||
|
||||
fn downloads_dir() -> std::path::PathBuf {
|
||||
|
|
@ -989,6 +1076,36 @@ fn table(tab: &mut Tab, args: &Value) -> Result<String, String> {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
#[test]
|
||||
fn the_one_tool_maps_every_action() {
|
||||
let mut c = |args: &str| {
|
||||
let mut call = crate::model::ToolCall { id: "x".into(), name: TOOL.into(), args: args.into(), status: crate::model::ToolStatus::Done };
|
||||
expand_call(&mut call);
|
||||
(call.name, call.args)
|
||||
};
|
||||
let (n, a) = c(r#"{"action":"click","ref":"12","double":true}"#);
|
||||
assert_eq!(n, "browser_click");
|
||||
assert!(a.contains(r#""ref":"12""#) && a.contains("double") && !a.contains("action"), "{a}");
|
||||
let (n, a) = c(r#"{"action":"fill","kind":"card","id":"7"}"#);
|
||||
assert_eq!(n, "browser_fill_card");
|
||||
assert!(a.contains(r#""card_id":"7""#) && !a.contains("kind"), "{a}");
|
||||
let (n, _) = c(r#"{"action":"fill"}"#);
|
||||
assert_eq!(n, "browser_fill_login");
|
||||
let (n, _) = c(r#"{"action":"nope"}"#);
|
||||
assert_eq!(n, TOOL, "an unknown action stays put (the runner explains it)");
|
||||
// Every internal browser tool is reachable from the one schema.
|
||||
let mut mapped: Vec<&str> = tool(&json!({"type": "string"}))["parameters"]["properties"]["action"]["enum"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|a| action_tool(a.as_str().unwrap()).unwrap())
|
||||
.collect();
|
||||
mapped.extend(["browser_fill_card", "browser_fill_contact"]);
|
||||
for n in NAMES {
|
||||
assert!(mapped.contains(&n), "internal tool {n} is unreachable from browser");
|
||||
}
|
||||
}
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
|
|
@ -1000,7 +1117,6 @@ mod tests {
|
|||
let c = changes(&a, &b);
|
||||
assert!(c.contains("網址變成：u2") && c.contains("新出現的文字：\nc"));
|
||||
assert!(changes(&a, &a).contains("沒有變化"));
|
||||
assert_eq!(NAMES.len(), specs(&json!({"type": "string"})).len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
11
src/web.rs
11
src/web.rs
|
|
@ -69,7 +69,7 @@ pub fn specs_for(browsers: &[String], default: &str) -> Vec<Value> {
|
|||
let mut v = specs_inner(!browsers.is_empty());
|
||||
if !browsers.is_empty() {
|
||||
let desc = format!("Which computer's AnyBot browser to use, by the computer's name. Each computer's browser has its own logins (a site logged in on one isn't on another). Default: {default}. Use another only when the person names it.");
|
||||
for t in v.iter_mut().filter(|t| t["name"].as_str().map(|n| n.starts_with("browser_")).unwrap_or(false)) {
|
||||
for t in v.iter_mut().filter(|t| t["name"].as_str().map(|n| n == "browser" || n.starts_with("browser_")).unwrap_or(false)) {
|
||||
t["parameters"]["properties"]["computer"] = json!({"type": "string", "enum": browsers, "description": desc});
|
||||
}
|
||||
}
|
||||
|
|
@ -86,15 +86,10 @@ fn specs_inner(browser: bool) -> Vec<Value> {
|
|||
let mut v = vec![
|
||||
json!({"name": "web_fetch", "description": "Download a public web page and return its text (no login). For pages that need the person's login, or that only show text after scripts run, use browser_open.",
|
||||
"parameters": {"type": "object", "properties": {"url": {"type": "string"}}, "required": ["url"]}}),
|
||||
json!({"name": "browser_open", "description": "Open a link in AnyBot's own browser window (a separate browser profile the person logs in to once, for sites like Threads, Instagram, Facebook, Gmail). Waits until the page has loaded and returns what it shows, like browser_read (title, address, text and the numbered links, buttons and fields), so there is no need to call browser_read right after. Never ask the person to change browser settings.",
|
||||
"parameters": {"type": "object", "properties": {"url": {"type": "string"}, "profile": profile}, "required": ["url"]}}),
|
||||
json!({"name": "browser_read", "description": "Read the title, address and visible text of the current tab in AnyBot's browser (logged-in pages included), plus a numbered list of its links (with their addresses), buttons, fields and dropdowns. Use those numbers with browser_click / browser_type / browser_select instead of guessing URLs. Each computer has its own AnyBot browser with its own logins (on a bot's own computer it is the Chromium on its desktop). For anything on a web page, read it here and act by these numbers; never guess screen coordinates on a web page. If it shows a login page, hand the computer to the person (hand_over_computer) to log in once.",
|
||||
"parameters": {"type": "object", "properties": {"max_chars": {"type": "integer", "description": "Longest text to return (default 20000)."}, "profile": profile}}}),
|
||||
json!({"name": "browser_profiles", "description": "List, add or switch AnyBot browser profiles (設定檔/profile). Use when the person wants another profile or account: 'list' shows them, 'add' makes a new empty one (they log in once there) and switches to it, 'use' switches. Switching changes which profile browser_open/browser_read use.",
|
||||
"parameters": {"type": "object", "properties": {"action": {"type": "string", "enum": ["list", "add", "use"]}, "name": {"type": "string", "description": "Profile name for add/use."}}, "required": ["action"]}}),
|
||||
crate::browser::tool(&profile),
|
||||
];
|
||||
if browser {
|
||||
v.extend(crate::browser::specs(&profile));
|
||||
// (browser_open/read/profiles are already the one tool above; nothing more to add.)
|
||||
}
|
||||
v
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue