Merge h1/computer-tool: one computer tool (H1)
This commit is contained in:
daniel wang
commit
701d0967a2
62
src/agent.rs
62
src/agent.rs
|
|
@ -864,7 +864,7 @@ fn quick_tools(tools: &[serde_json::Value]) -> Vec<serde_json::Value> {
|
|||
.iter()
|
||||
.map(|t| {
|
||||
let mut t = t.clone();
|
||||
let screen = t["name"].as_str().is_some_and(|n| matches!(n, "computer_use_click" | "computer_use_typing" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_batch"));
|
||||
let screen = t["name"].as_str().is_some_and(|n| matches!(n, "computer" | "computer_use_click" | "computer_use_typing" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_batch"));
|
||||
if screen {
|
||||
// `get_mut`, not `t[..]`: indexing a Value mutably inserts every missing key, which
|
||||
// once gave click and typing an `actions` with `properties: null` — the provider then
|
||||
|
|
@ -994,7 +994,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc<
|
|||
};
|
||||
let tried_quick = quick.clone();
|
||||
let mut attempt = 0;
|
||||
let out = loop {
|
||||
let mut out = loop {
|
||||
// A replay step: no model.
|
||||
if let Some(c) = scripted.clone() {
|
||||
break providers::StepOutput { text: String::new(), thinking: String::new(), tool_calls: vec![c], blocks: vec![] };
|
||||
|
|
@ -1037,6 +1037,12 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc<
|
|||
}
|
||||
}
|
||||
};
|
||||
// The model drives the screen with one tool: expand `computer` + action into the
|
||||
// internal per-action tool right here, so every guard below (repeat detection,
|
||||
// approvals, replay, the remote link, the work log) keeps its stable per-action name.
|
||||
for c in out.tool_calls.iter_mut() {
|
||||
computer::expand_call(c);
|
||||
}
|
||||
let quick = if plan.quick_is_off() { None } else { tried_quick };
|
||||
let via = if quick.is_none() && scripted.is_none() { job.via.clone() } else { via };
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
|
|
@ -2020,30 +2026,46 @@ mod tests {
|
|||
serde_json::json!({"tool_calls": [{"index": 0, "id": id, "function": {"name": name, "arguments": args.to_string()}}]})
|
||||
}
|
||||
|
||||
/// The same click twice in one step: the second isn't done (it closed a menu the first had
|
||||
/// just opened). The clicks name a computer that isn't there, so nothing is clicked here.
|
||||
/// The quick model's `computer` tool names controls only: looking a number up in the list
|
||||
/// is where it went wrong. The bot's own model keeps `ref`; expand_call maps the one tool
|
||||
/// back to the internal per-action tools everything downstream runs on.
|
||||
#[test]
|
||||
fn the_quick_models_screen_tools_name_controls_only() {
|
||||
fn the_quick_models_screen_tool_names_controls_only() {
|
||||
let specs = crate::computer::specs_for(&["C".to_string()], "C", true);
|
||||
assert_eq!(specs.len(), 1);
|
||||
assert_eq!(specs[0]["name"], "computer");
|
||||
let quick = quick_tools(&specs);
|
||||
for t in &quick {
|
||||
let n = t["name"].as_str().unwrap();
|
||||
if matches!(n, "computer_use_click" | "computer_use_typing" | "computer_use_scroll" | "computer_use_mouse_move") {
|
||||
assert!(t["parameters"]["properties"].get("ref").is_none() && t["parameters"]["properties"].get("name").is_some(), "{n}");
|
||||
}
|
||||
if n == "computer_use_batch" {
|
||||
assert!(t["parameters"]["properties"]["actions"]["items"]["properties"].get("ref").is_none());
|
||||
}
|
||||
}
|
||||
assert!(quick[0]["parameters"]["properties"].get("ref").is_none());
|
||||
assert!(quick[0]["parameters"]["properties"].get("name").is_some());
|
||||
assert!(quick[0]["parameters"]["properties"]["actions"]["items"]["properties"].get("ref").is_none());
|
||||
// Nothing else changes shape: only `ref` is gone (a stray key here makes the provider refuse
|
||||
// the whole request).
|
||||
for (a, b) in specs.iter().zip(&quick) {
|
||||
let keys = |t: &serde_json::Value| t["parameters"]["properties"].as_object().map(|p| p.keys().filter(|k| *k != "ref").cloned().collect::<Vec<_>>());
|
||||
assert_eq!(keys(a), keys(b), "{}", a["name"]);
|
||||
}
|
||||
let keys = |t: &serde_json::Value| t["parameters"]["properties"].as_object().map(|p| p.keys().filter(|k| *k != "ref").cloned().collect::<Vec<_>>());
|
||||
assert_eq!(keys(&specs[0]), keys(&quick[0]));
|
||||
assert!(!serde_json::to_string(&quick).unwrap().contains(":null"), "a key with no value");
|
||||
// The bot's own model keeps both.
|
||||
assert!(specs.iter().any(|t| t["name"] == "computer_use_click" && t["parameters"]["properties"].get("ref").is_some()));
|
||||
// The bot's own model keeps ref.
|
||||
assert!(specs[0]["parameters"]["properties"].get("ref").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_one_screen_tool_expands_to_the_internal_ones() {
|
||||
let mut c = crate::model::ToolCall { id: "x".into(), name: "computer".into(), args: r#"{"action":"click","ref":"4","on":"My Mac"}"#.into(), status: crate::model::ToolStatus::Running };
|
||||
crate::computer::expand_call(&mut c);
|
||||
assert_eq!(c.name, "computer_use_click");
|
||||
let v: serde_json::Value = serde_json::from_str(&c.args).unwrap();
|
||||
assert_eq!(v["ref"], "4");
|
||||
assert_eq!(v["computer"], "My Mac");
|
||||
assert!(v.get("action").is_none() && v.get("on").is_none());
|
||||
// Screenshots keep their name; unknown actions and other tools pass through.
|
||||
let mut s = crate::model::ToolCall { id: "x".into(), name: "computer".into(), args: r#"{"action":"screenshot"}"#.into(), status: crate::model::ToolStatus::Running };
|
||||
crate::computer::expand_call(&mut s);
|
||||
assert_eq!(s.name, "computer_use_screenshot");
|
||||
let mut bad = crate::model::ToolCall { id: "x".into(), name: "computer".into(), args: r#"{"action":"fly"}"#.into(), status: crate::model::ToolStatus::Running };
|
||||
crate::computer::expand_call(&mut bad);
|
||||
assert_eq!(bad.name, "computer");
|
||||
// describe follows the expansion: the work log says 「點一下 #4」.
|
||||
assert_eq!(crate::computer::describe("computer", r##"{"action":"click","ref":"#4"}"##), "點 #4");
|
||||
assert_eq!(crate::computer::describe("computer", r#"{"action":"screenshot"}"#), "螢幕截圖");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -4216,9 +4216,9 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu
|
|||
- 絕對不要直接呼叫網站的內部 API、下載網站的程式碼來研究，或用程式大量抓資料：網站會把帳號當成機器人（例如跳出拼圖驗證、封鎖帳號）。也絕對不要用指令去讀瀏覽器的 Cookie、瀏覽紀錄、密碼或設定檔資料庫。
|
||||
- 需要使用者親手做的步驟（讀到登入頁、輸入密碼、驗證碼、拼圖、簡訊或 email 驗證、確認付款）：先在那台電腦上把那一頁打開、點到要輸入的地方，再呼叫 hand_over_computer，instruction 寫清楚要他做什麼。聊天裡會出現那台電腦的即時畫面，他點「接管」做完後按「我完成了」，你再確認、接著做（登入一次之後就會記住）。不要只用文字叫他自己去找視窗，絕對不要替他打密碼，也不要叫他改瀏覽器的設定或選單、或說要從 Chrome 複製登入或 Cookie（都不需要，也沒有這個功能）。\n\
|
||||
- AnyBot 瀏覽器有設定檔（profile），每個有自己的登入。使用者說要換 profile／設定檔／另一個帳號時，用 browser_profiles：list 看有哪些，use 切換，add 新增（新的要登入一次）。也可以在 browser_open / browser_read 帶 profile 參數。使用者想用自己平常的 Chrome（自己的 Chrome 設定檔、已經登入的帳號）時，用 profile「你的瀏覽器」：請使用者在 Chrome 切到想要的設定檔，再 browser_read。\n\
|
||||
- 你也能像人一樣操作螢幕（{screens}），用在檔案、指令和網頁工具做不到的事（例如操作其他 App）。照截圖上的編號操作（工具說明裡有怎麼用），按鈕名字清楚的也可以直接用 name 指定（例如計算機的 name「4」、name「=」），不用自己對編號；只有沒有編號的東西才用 x、y。結果說「畫面和上一步完全一樣」就是沒點到：換一個編號或換方法，絕對不要重複同一步。已經從畫面知道接下來幾步時（填一張表、一組捷徑），用 computer_use_batch 一次做完；要看上一步結果才知道下一步的，就一步一步來。每次看截圖用一句話說畫面顯示什麼、這步成功與否。\n\
|
||||
- 你也能像人一樣操作螢幕（{screens}），用在檔案、指令和網頁工具做不到的事（例如操作其他 App）。照截圖上的編號操作（工具說明裡有怎麼用），按鈕名字清楚的也可以直接用 name 指定（例如計算機的 name「4」、name「=」），不用自己對編號；只有沒有編號的東西才用 x、y。結果說「畫面和上一步完全一樣」就是沒點到：換一個編號或換方法，絕對不要重複同一步。已經從畫面知道接下來幾步時（填一張表、一組捷徑），用 computer（action=batch）一次做完；要看上一步結果才知道下一步的，就一步一步來。每次看截圖用一句話說畫面顯示什麼、這步成功與否。\n\
|
||||
- 每件事動手前先選好路：① 有專用工具就用專用工具（網頁用 browser_*，Mac 的提醒事項、行事曆、郵件等用它們的專門工具）。② 改設定、處理檔案、安裝、開關服務、App 有自己的指令列：用指令。③ 只有畫面上才做得到的（App 裡的按鈕或開關、選單列的圖示、沒有指令可改的設定）：操作畫面，先把那個 App 的視窗打開到最前面（Mac 用 open -a App名稱），再照編號點。\n\
|
||||
- 換路的規則：一條指令因為環境失敗（Operation not permitted、程式當掉、結果寫著是沙盒擋的），就不要換寫法再試同一條路，改走畫面；畫面上找不到要的東西，先想有沒有指令或專用工具做得到。絕對不要用指令截圖或分析圖片（screencapture、import、PIL）：看畫面一律用 computer_use_screenshot，看不清楚就用 computer_use_zoom。\n\
|
||||
- 換路的規則：一條指令因為環境失敗（Operation not permitted、程式當掉、結果寫著是沙盒擋的），就不要換寫法再試同一條路，改走畫面；畫面上找不到要的東西，先想有沒有指令或專用工具做得到。絕對不要用指令截圖或分析圖片（screencapture、import、PIL）：看畫面一律用 computer（action=screenshot），看不清楚就用 action=zoom。\n\
|
||||
- 像一個熟練的人那樣做事：動手前先想好要做哪幾步、做完怎樣才算成功（照「電腦現況」挑那台電腦上最直接的做法，不用再試探環境）；選定一條路就做到底，不要在指令和畫面之間來回換。出錯先讀錯誤訊息想原因再換方法，同一個做法失敗兩次就換。做完只用一種最直接的方法確認一次（讀設定檔、查狀態、看一眼畫面），確認了就回答，不要用好幾種方法反覆確認。\n\
|
||||
- 沒見過的 App 或畫面也一樣：先看清楚（截圖、需要就放大），找選單、設定、按鈕上的字，照編號點；網頁上的東西用 browser_* 的編號。\n\
|
||||
- 使用者交代的事就做完：不要做到一半停下來問「要我繼續嗎」「要我現在做嗎」，畫面不如預期就自己處理（關掉擋路的視窗、打開要用的 App）再接著做。只有需要使用者親手做、或要先確認會切斷連線、刪除、付款這類事情時才停下來問。\n\
|
||||
|
|
@ -4287,7 +4287,7 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu
|
|||
}
|
||||
s.push_str("\n\n使用者要你追蹤、整理、比較、研究一件之後還會回來看的事（例如「幫我追蹤台灣 FSD 狀況」），就用 write_page 做成一頁：先寫結論和現況，再寫細節和來源（附日期），適合時用表格、```stats、```steps、圖表。回答時簡短說重點，並用 [頁面標題](page:ID) 連到它。要定期追蹤時，再用 create_routine 設例行任務，每次執行：read_page、查新消息，有實質進展才用 write_page 更新（note 寫改了什麼）並在聊天裡說；沒有新消息就簡短回答「沒有新消息」，不要更新頁面。");
|
||||
s.push_str("\n\n幫使用者寫貼文、留言或文案，要「用我的語氣」時：先打開他的帳號，讀最近幾則他自己寫的貼文，照那樣的長度、用詞、標點和開頭結尾寫，不要只看自介。給草稿就好，沒有他說「發」不要自己發出去。");
|
||||
s.push_str("\n\n使用者要看某台電腦的畫面（截圖給我看、讓我看看）時，用 computer_use_show：畫面會直接出現在聊天裡。");
|
||||
s.push_str("\n\n使用者要看某台電腦的畫面（截圖給我看、讓我看看）時，用 computer（action=show）：畫面會直接出現在聊天裡。");
|
||||
s.push_str("\n\n做好的檔案（試算表、文件、Markdown、網頁、圖片）用 send_file 傳進聊天，使用者點一下就能預覽；不要只說檔案放在哪裡。適合的時候可以用 react 對使用者的訊息按個表情（例如謝謝你時按 ❤️），不用每次都按。");
|
||||
s.push_str("\n\n");
|
||||
s.push_str(&crate::i18n::reply_instruction(&st.language));
|
||||
|
|
|
|||
168
src/computer.rs
168
src/computer.rs
|
|
@ -224,7 +224,7 @@ pub fn is_pointer_step(name: &str) -> bool {
|
|||
pub const SHOWN_NOTE: &str = "已把這台電腦現在的畫面（使用者看到的樣子，沒有編號）放進聊天，使用者看得到這張圖；你也看得到，可以用一兩句話說明畫面上有什麼";
|
||||
|
||||
pub fn is_computer_tool(name: &str) -> bool {
|
||||
NAMES.contains(&name)
|
||||
NAMES.contains(&name) || name == TOOL
|
||||
}
|
||||
|
||||
pub fn label(name: &str) -> &'static str {
|
||||
|
|
@ -247,6 +247,21 @@ pub fn label(name: &str) -> &'static str {
|
|||
|
||||
|
||||
pub fn describe(name: &str, args: &str) -> String {
|
||||
if name == TOOL {
|
||||
// Same words as the internal tool it runs as.
|
||||
let v = crate::tools::args_of(args);
|
||||
let a = v["action"].as_str().unwrap_or("");
|
||||
return match top_action_tool(a) {
|
||||
Some(t) => {
|
||||
let mut v2 = v.clone();
|
||||
if let Some(o) = v2.as_object_mut() {
|
||||
o.remove("action");
|
||||
}
|
||||
describe(t, &v2.to_string())
|
||||
}
|
||||
None => format!("電腦操作 {a}"),
|
||||
};
|
||||
}
|
||||
let v = crate::tools::args_of(args);
|
||||
// By number from the app's list (e7), else where on the screen (it said "(0, 0)" for every
|
||||
// click by number).
|
||||
|
|
@ -282,51 +297,106 @@ pub fn describe(name: &str, args: &str) -> String {
|
|||
|
||||
/// The screen tools, each with a `computer` choice. One way to see (the numbered picture), one
|
||||
/// way to act (a real click or keystroke, best by number).
|
||||
/// One tool for the screen, like the models were trained on (Anthropic's `computer`, OpenAI's
|
||||
/// CUA): choosing among twelve near-identical names was its own error source, and twelve schemas
|
||||
/// are the biggest stable part of every request. Actions keep the old behaviour exactly:
|
||||
/// `expand_call` maps a call back to the internal per-action tool at the model-output boundary,
|
||||
/// so approvals, replay, skills, the remote link and the work log all keep their names.
|
||||
pub fn specs_for(computers: &[String], default: &str, chosen: bool) -> Vec<Value> {
|
||||
let mods = json!({"type": "array", "items": {"type": "string", "enum": ["cmd", "shift", "option", "control"]}});
|
||||
let rf = json!({"type": "string", "description": "The number of a control on the latest picture (e.g. 12)."});
|
||||
let named = json!({"type": "string", "description": "Or the control's name exactly as the latest list shows it (e.g. 4, Close, 自動更新): the numbered control with that name. Easiest for keys and buttons with plain names; use ref when two have the same name."});
|
||||
let mut v = vec![
|
||||
json!({"name": "computer_use_screenshot", "description": "See a computer's screen: a picture with every button, link, field and menu item boxed and numbered, and the same numbers in a list with their names. Act on a control by its number (ref). Every action below already returns the next picture, so call this only to start or to look again.",
|
||||
"parameters": {"type": "object", "properties": {}}}),
|
||||
json!({"name": "computer_use_zoom", "description": "Look closer at part of the latest picture, at the screen's full sharpness: small icons (the menu bar, a toolbar, a tray), small print, a tiny control. Give the area in the latest picture's pixels. You get a bigger, sharper picture of just that area; to act there, use the numbers on the normal picture, or turn a point in the zoomed picture back into the normal picture's pixels as the result explains. Use it instead of guessing, and never make your own zoom with commands.",
|
||||
"parameters": {"type": "object", "properties": {"x": {"type": "number", "description": "Left edge, in the latest picture's pixels"}, "y": {"type": "number", "description": "Top edge"}, "width": {"type": "number"}, "height": {"type": "number"}}, "required": ["x", "y", "width", "height"]}}),
|
||||
json!({"name": "computer_use_show", "description": "Show the person a computer's screen: a picture of it as they would see it (no numbers) goes into the chat for them, and you see it too. Use it when they ask to see a screen (截圖給我看、讓我看看畫面). One call, no file needed.",
|
||||
"parameters": {"type": "object", "properties": {}}}),
|
||||
json!({"name": "computer_use_click", "description": "Click a control: by its name as the latest list shows it (name, e.g. 4, Close, 自動更新 — easiest, no number to look up), or by its number (ref) when two share a name. Only for something with neither (a spot in a picture or game): x, y in the picture's pixels. Returns the next picture.",
|
||||
"parameters": {"type": "object", "properties": {"ref": rf, "name": named, "x": {"type": "number"}, "y": {"type": "number"}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "count": {"type": "integer", "description": "2 for a double click"}, "modifier_keys": mods}}}),
|
||||
json!({"name": "computer_use_typing", "description": "Type text (any language) into a field by its name or number (it is clicked first; a one-line field's old text is replaced, as if selected first — append=true adds to it instead), or where the cursor is. submit=true presses return after it (send, search, open the address). Returns the next picture.",
|
||||
"parameters": {"type": "object", "properties": {"value": {"type": "string"}, "ref": rf, "name": named, "submit": {"type": "boolean"}, "append": {"type": "boolean", "description": "Keep the field's text and add to it"}}, "required": ["value"]}}),
|
||||
json!({"name": "computer_use_press_key", "description": "Press a key or shortcut in the app in front: return, tab, escape, delete, up, slash, cmd+c, cmd+shift+t, cmd+space. Several in order: separate with spaces (\"cmd+a delete\"). On a Linux computer cmd means ctrl. Returns the next picture.",
|
||||
"parameters": {"type": "object", "properties": {"key": {"type": "string"}, "hold_duration_ms": {"type": "integer"}}, "required": ["key"]}}),
|
||||
json!({"name": "computer_use_scroll", "description": "Scroll a list or page: on a control by number (ref), or at x, y, or where the pointer is. Returns the next picture.",
|
||||
"parameters": {"type": "object", "properties": {"ref": rf, "name": named, "x": {"type": "number"}, "y": {"type": "number"}, "direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "amount": {"type": "integer", "description": "Scroll steps of 16 pixels, 1-20, default 3"}, "modifier_keys": mods}, "required": ["direction"]}}),
|
||||
json!({"name": "computer_use_open_app", "description": "Open an app, or bring it to the front when it is already open (its name as shown, e.g. Ghostty, Calculator, 計算機, Chromium, xterm). Returns the next picture.",
|
||||
"parameters": {"type": "object", "properties": {"app": {"type": "string"}}, "required": ["app"]}}),
|
||||
json!({"name": "computer_use_mouse_move", "description": "Move the mouse onto a control by number (ref) or to x, y (to open a hover menu). Returns the next picture.",
|
||||
"parameters": {"type": "object", "properties": {"ref": rf, "name": named, "x": {"type": "number"}, "y": {"type": "number"}}}}),
|
||||
json!({"name": "computer_use_drag", "description": "Drag along a path of points in the picture's pixels (first = press, last = release). Returns the next picture.",
|
||||
"parameters": {"type": "object", "properties": {"path": {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "button": {"type": "string", "enum": ["left", "right"]}, "modifier_keys": mods}, "required": ["path"]}}),
|
||||
json!({"name": "computer_use_check_permissions", "description": "Check whether the screen can be seen and the mouse and keyboard used on that computer.",
|
||||
"parameters": {"type": "object", "properties": {}}}),
|
||||
json!({"name": "computer_use_batch", "description": format!("Do a planned sequence of steps in one call (max {MAX_BATCH}): each item is one action with its own arguments, best naming the control, e.g. {{\"action\":\"click\",\"name\":\"7\"}}, {{\"action\":\"typing\",\"name\":\"Search\",\"value\":\"cat\"}}, {{\"action\":\"press_key\",\"key\":\"tab\"}}. Use it when the current picture already tells you every step (a row of keys to press, filling a form, a known shortcut chain): put the whole sequence in one call, they run in order, the first step that fails stops the rest, and you get one picture at the end. Several separate clicks in one step are slower and easier to get wrong than one batch. Do not use it to explore, or where step 2 depends on what step 1's screen shows — act one step at a time there."),
|
||||
"parameters": {"type": "object", "properties": {"actions": {"type": "array", "maxItems": MAX_BATCH, "items": {"type": "object", "properties": {"action": {"type": "string", "enum": ["click", "typing", "press_key", "scroll", "mouse_move", "drag"]}, "ref": rf.clone(), "name": named.clone(), "x": {"type": "number"}, "y": {"type": "number"}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "count": {"type": "integer"}, "value": {"type": "string"}, "submit": {"type": "boolean"}, "key": {"type": "string"}, "hold_duration_ms": {"type": "integer"}, "direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "amount": {"type": "integer"}, "path": {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "modifier_keys": mods.clone()}, "required": ["action"]}}}, "required": ["actions"]}}),
|
||||
];
|
||||
// Always named: each computer goes by its own name (the instructions say which is which),
|
||||
// so the bot says whose screen it means and the person can tell where it worked.
|
||||
if !computers.is_empty() {
|
||||
// The person's chosen computer may have no screen (a bot's computer without Docker): then
|
||||
// the default is only the first screen there is, and the bot is told so.
|
||||
let why = if chosen { "the one the person chose" } else { "the computer the person chose has no screen" };
|
||||
let desc = format!("Whose screen, by the computer's name. Default: {default} ({why}). Use another only when the person names it.");
|
||||
for t in v.iter_mut() {
|
||||
t["parameters"]["properties"]["computer"] = json!({"type": "string", "enum": computers, "description": desc});
|
||||
let _ = (computers, default, chosen);
|
||||
let list = "每一種電腦（Mac/Windows/Linux）都算，computer 參數選哪一台";
|
||||
vec![json!({
|
||||
"name": TOOL,
|
||||
"description": format!(
|
||||
"用滑鼠鍵盤操作一台電腦的螢幕（{list}）。action 選一種：screenshot（看畫面：回傳截圖＋編號控件清單，結果頭兩行是座標對應，x、y 要用那張圖的座標）; zoom（看小字：給 x,y,width,height 框一块）; show（把畫面放進聊天給人看）; click（ref=截圖編號、name=控件名字【優先：如 name=「4」、name=「=」】、或 x,y）; typing（value 必填；name=先點那個欄位、append=true 接在原文後面、submit=true 按 Enter）; press_key（key 如 Return/Escape/Tab/cmd+s，一次一個組合鍵；要按住給 hold_duration_ms）; scroll（direction=down/up、amount 約幾格、可給 x,y）; mouse_move（移滑鼠並顯示游標）; drag（path=至少兩點 [{{x,y}}…] 拖過的路徑）; open_app（app：Finder/系統設定/計算機/文字編輯/Safari）; batch（你已經從畫面看懂的連續 2-16 步一次做完，中途不能看畫面；每項 {{action:…}} 加各自參數；要跟上一步結果的步不要放）; check_permissions。macOS 一律優先 ref/name，x、y 只給沒有編號的東西。每次看畫面用一句話說畫面顯示什麼、這步成功與否；結果說「畫面和上一步完全一樣」就是沒點到，換編號換方法，不要重複同一步。",
|
||||
list = list
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": { "type": "string", "enum": ["screenshot", "zoom", "show", "click", "typing", "press_key", "scroll", "mouse_move", "drag", "open_app", "batch", "check_permissions"] },
|
||||
"computer": { "type": "string", "description": "哪一台電腦（列表裡的名字）。" },
|
||||
"ref": { "type": "string", "description": "截圖左邊紅框的編號。" },
|
||||
"name": { "type": "string", "description": "控件的名字（優先於座標）。" },
|
||||
"x": { "type": "number", "description": "click/zoom/scroll/mouse_move 的 x（用最近一張截圖的座標）; drag 不用。" }, "y": { "type": "number" },
|
||||
"width": { "type": "number", "description": "zoom 區域寬。" },
|
||||
"height": { "type": "number", "description": "zoom 區域高。" },
|
||||
"button": { "type": "string", "enum": ["left", "right", "middle"] },
|
||||
"count": { "type": "integer", "minimum": 1, "maximum": 3 },
|
||||
"modifier_keys": { "type": "array", "items": { "type": "string", "enum": ["cmd", "shift", "option", "ctrl", "alt"] } },
|
||||
"value": { "type": "string", "description": "typing 要輸入的文字。" },
|
||||
"submit": { "type": "boolean", "description": "typing 輸入後按 Enter。" },
|
||||
"append": { "type": "boolean", "description": "typing 接在原文後面。" },
|
||||
"key": { "type": "string", "description": "press_key 的按鍵，如 Return、Escape、cmd+s。" },
|
||||
"hold_duration_ms": { "type": "integer", "minimum": 100, "maximum": 10000, "description": "press_key/click 按住多久。" },
|
||||
"direction": { "type": "string", "enum": ["down", "up"] },
|
||||
"amount": { "type": "integer", "minimum": 1, "description": "scroll 約幾格。" },
|
||||
"path": { "type": "array", "items": { "type": "object", "properties": { "x": { "type": "number" }, "y": { "type": "number" } }, "required": ["x", "y"] }, "description": "drag 的點陣（至少兩點，用截圖座標）。" },
|
||||
"app": { "type": "string", "description": "open_app 的軟體名。" },
|
||||
"actions": { "type": "array", "items": { "type": "object",
|
||||
"properties": {
|
||||
"action": { "type": "string", "enum": ["click", "typing", "press_key", "scroll", "mouse_move", "drag", "open_app"] },
|
||||
"ref": { "type": "string" }, "name": { "type": "string" },
|
||||
"x": { "type": "number" }, "y": { "type": "number" },
|
||||
"button": { "type": "string", "enum": ["left", "right", "middle"] },
|
||||
"count": { "type": "integer", "minimum": 1, "maximum": 3 },
|
||||
"modifier_keys": { "type": "array", "items": { "type": "string", "enum": ["cmd", "shift", "option", "ctrl", "alt"] } },
|
||||
"value": { "type": "string" },
|
||||
"path": { "type": "array", "items": { "type": "object", "properties": { "x": { "type": "number" }, "y": { "type": "number" } } } },
|
||||
"submit": { "type": "boolean" }, "append": { "type": "boolean" },
|
||||
"key": { "type": "string" },
|
||||
"direction": { "type": "string", "enum": ["down", "up"] },
|
||||
"amount": { "type": "integer", "minimum": 1 }
|
||||
}, "required": ["action"] }, "minItems": 2, "maxItems": 16 }
|
||||
},
|
||||
"required": ["action"]
|
||||
}
|
||||
}
|
||||
v
|
||||
})]
|
||||
}
|
||||
|
||||
/// Longest edge of a zoomed picture (sharp enough to read a menu-bar icon, still one image).
|
||||
/// The one screen tool's name (see `specs_for`).
|
||||
pub const TOOL: &str = "computer";
|
||||
|
||||
/// The tool's `action` → internal per-action tool (the batch's own items use `action_tool`,
|
||||
/// which stays the stricter batch-only map).
|
||||
pub fn top_action_tool(action: &str) -> Option<&'static str> {
|
||||
Some(match action {
|
||||
"screenshot" => "computer_use_screenshot",
|
||||
"zoom" => "computer_use_zoom",
|
||||
"show" => "computer_use_show",
|
||||
"click" => "computer_use_click",
|
||||
"typing" => "computer_use_typing",
|
||||
"press_key" => "computer_use_press_key",
|
||||
"scroll" => "computer_use_scroll",
|
||||
"mouse_move" => "computer_use_mouse_move",
|
||||
"drag" => "computer_use_drag",
|
||||
"open_app" => "computer_use_open_app",
|
||||
"batch" => "computer_use_batch",
|
||||
"check_permissions" => "computer_use_check_permissions",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Rewrite a `computer` call into its internal tool in place; anything else passes through.
|
||||
/// `on` is accepted as the machine name (a parameter named like the tool reads badly); the
|
||||
/// internal tools keep answering to `computer`, which is what history, skills and limbs speak.
|
||||
pub fn expand_call(c: &mut crate::model::ToolCall) {
|
||||
if c.name != TOOL {
|
||||
return;
|
||||
}
|
||||
let mut v = crate::tools::args_of(&c.args);
|
||||
let Some(action) = v["action"].as_str().and_then(top_action_tool) else { return };
|
||||
if let Some(o) = v.as_object_mut() {
|
||||
o.remove("action");
|
||||
if let Some(on) = o.remove("on") {
|
||||
o.entry("computer".to_string()).or_insert(on);
|
||||
}
|
||||
}
|
||||
c.name = action.to_string();
|
||||
c.args = v.to_string();
|
||||
}
|
||||
|
||||
/// The longest edge of a zoom picture.
|
||||
pub const ZOOM_LONG_EDGE: u32 = 1024;
|
||||
/// A zoom never blows up the real pixels more than this (it would only blur).
|
||||
pub const ZOOM_MAX_UPSCALE: f64 = 4.0;
|
||||
|
|
@ -522,8 +592,17 @@ pub fn run(name: &str, args: &str) -> (String, bool, Option<String>) {
|
|||
run_by(name, args, "")
|
||||
}
|
||||
|
||||
/// Same, remembering which bot acted (for the 「正在操作你的電腦」 bar).
|
||||
/// Same, remembering which bot acted (for the 「正在操作你的電腦」 bar). A `computer` call is
|
||||
/// expanded here too, so remote limbs and every other caller accept the one-tool shape.
|
||||
pub fn run_by(name: &str, args: &str, who: &str) -> (String, bool, Option<String>) {
|
||||
let (name, args) = if name == TOOL {
|
||||
let mut c = crate::model::ToolCall { id: String::new(), name: name.to_string(), args: args.to_string(), status: crate::model::ToolStatus::Running };
|
||||
expand_call(&mut c);
|
||||
(c.name, c.args)
|
||||
} else {
|
||||
(name.to_string(), args.to_string())
|
||||
};
|
||||
let (name, args) = (name.as_str(), args.as_str());
|
||||
if moves_things(name) {
|
||||
*LAST_ACTION.lock().unwrap_or_else(|e| e.into_inner()) = Some((std::time::Instant::now(), who.to_string()));
|
||||
}
|
||||
|
|
@ -533,7 +612,8 @@ pub fn run_by(name: &str, args: &str, who: &str) -> (String, bool, Option<String
|
|||
}
|
||||
}
|
||||
|
||||
/// Steps that use the mouse or keyboard (and come back with the screen after them).
|
||||
/// Steps that use the mouse or keyboard (and come back with the screen after them). A
|
||||
/// `computer` call never reaches here: every caller expands it first (`run_by`, the agent loop).
|
||||
pub fn moves_things(name: &str) -> bool {
|
||||
matches!(name, "computer_use_click" | "computer_use_typing" | "computer_use_press_key" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_drag" | "computer_use_open_app" | "computer_use_batch")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
#![recursion_limit = "512"] // the one `computer` tool's schema is one deep json! literal
|
||||
|
||||
mod agent;
|
||||
mod app;
|
||||
mod appctl;
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ impl Spots {
|
|||
/// "12", "e12", "[12]" → the control's centre and what it is.
|
||||
pub fn get(&self, r: &str) -> Result<((f64, f64), String), String> {
|
||||
let n = number_of(r).ok_or_else(|| format!("「{}」不是編號：請用截圖上的編號（例如 12）", r.trim()))?;
|
||||
self.by_number.get(&n).cloned().ok_or_else(|| format!("最新的畫面裡沒有編號 {n}：先 computer_use_screenshot 再看一次。"))
|
||||
self.by_number.get(&n).cloned().ok_or_else(|| format!("最新的畫面裡沒有編號 {n}：先用 computer（action=screenshot）看一次畫面再點。"))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -727,8 +727,8 @@ pub fn sandbox_note(cmd: &str, out: &str) -> Option<&'static str> {
|
|||
return Some(match osascript_outside(cmd) {
|
||||
// Plain osascript that still got -10004 can't be the sandbox: macOS itself said no.
|
||||
Ok(()) => return None,
|
||||
Err(_) if cmd.contains("osascript") => "（這是 AnyBot 的指令沙盒擋下的，不是那個 App 壞了：AppleScript 只有寫成單獨一條 osascript -e '…' 指令（不接其他指令、不用 heredoc，多行就多個 -e），而且不跑 shell 指令、不碰終端機、不讀檔案時，才會在沙盒外跑、能指揮別的 App。改寫成那樣，或改用畫面操作（computer_use_*）。）",
|
||||
Err(_) => "（這是 AnyBot 的指令沙盒擋下的，不是那個 App 壞了：在沙盒裡不能指揮別的 App（-10004）。改用畫面操作（computer_use_*），或寫成單獨一條 osascript -e '…' 指令。）",
|
||||
Err(_) if cmd.contains("osascript") => "（這是 AnyBot 的指令沙盒擋下的，不是那個 App 壞了：AppleScript 只有寫成單獨一條 osascript -e '…' 指令（不接其他指令、不用 heredoc，多行就多個 -e），而且不跑 shell 指令、不碰終端機、不讀檔案時，才會在沙盒外跑、能指揮別的 App。改寫成那樣，或改用電腦操作（computer 工具的 action，如 click、typing）。）",
|
||||
Err(_) => "（這是 AnyBot 的指令沙盒擋下的，不是那個 App 壞了：在沙盒裡不能指揮別的 App（-10004）。改用電腦操作（computer 工具的 action），或寫成單獨一條 osascript -e '…' 指令。）",
|
||||
});
|
||||
}
|
||||
if (out.contains("Trace/BPT trap") || out.contains("指令因為收到訊號而終止")) && cmd.contains(".app/Contents/MacOS/") {
|
||||
|
|
|
|||
Loading…
Reference in New Issue