diff --git a/src/agent.rs b/src/agent.rs index 6490c0e..bba401a 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -855,6 +855,32 @@ impl StepPlan { } } +/// The quick model's screen tools name their controls only: looking a number up in the list +/// is where it went wrong (1, 2, 3, then 「1」's number again for 4). Without `ref` it says which +/// button the way a person would; two of the same name come back as trouble for the bot's own +/// model. +fn quick_tools(tools: &[serde_json::Value]) -> Vec { + tools + .iter() + .map(|t| { + let mut t = t.clone(); + let screen = t["name"].as_str().is_some_and(|n| matches!(n, "computer_use_click" | "computer_use_typing" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_batch")); + if screen { + if let Some(p) = t["parameters"]["properties"].as_object_mut() { + p.remove("ref"); + } + if let Some(p) = t["parameters"]["properties"]["actions"]["items"]["properties"].as_object_mut() { + p.remove("ref"); + } + if let Some(d) = t["description"].as_str() { + t["description"] = serde_json::Value::String(format!("{d} Name the control exactly as the latest list shows it (name); there are no numbers here.")); + } + } + t + }) + .collect() +} + /// Which way a tool goes: commands and files, the screen, or a web page. fn route_of(name: &str) -> &'static str { if computer::is_computer_tool(name) || crate::macapps::is_mac_tool(name) { @@ -968,7 +994,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< let main_req = ChatRequest { history: history.clone(), ..job.req.clone() }; let quick_now = if plan.quick_is_off() { None } else { quick.clone() }; let got = match quick_now { - Some(q) => match providers::run_step(&ChatRequest { model: q.clone(), ..main_req.clone() }, &cancel, &quiet) { + Some(q) => match providers::run_step(&ChatRequest { model: q.clone(), tools: quick_tools(&main_req.tools), ..main_req.clone() }, &cancel, &quiet) { Ok(o) => Ok(o), // Too long is the chat's problem, not the quick model's: folded below. Err(e) if providers::is_context_overflow(&e) => Err(e), @@ -1986,6 +2012,23 @@ mod tests { /// The same click twice in one step: the second isn't done (it closed a menu the first had /// just opened). The clicks name a computer that isn't there, so nothing is clicked here. + #[test] + fn the_quick_models_screen_tools_name_controls_only() { + let specs = crate::computer::specs_for(&["C".to_string()], "C", true); + let quick = quick_tools(&specs); + for t in &quick { + let n = t["name"].as_str().unwrap(); + if matches!(n, "computer_use_click" | "computer_use_typing" | "computer_use_scroll" | "computer_use_mouse_move") { + assert!(t["parameters"]["properties"].get("ref").is_none() && t["parameters"]["properties"].get("name").is_some(), "{n}"); + } + if n == "computer_use_batch" { + assert!(t["parameters"]["properties"]["actions"]["items"]["properties"].get("ref").is_none()); + } + } + // The bot's own model keeps both. + assert!(specs.iter().any(|t| t["name"] == "computer_use_click" && t["parameters"]["properties"].get("ref").is_some())); + } + #[test] fn the_same_click_twice_in_a_step_is_done_once() { let click = serde_json::json!({"name": "computer_use_click", "arguments": serde_json::json!({"computer": "沒有這台", "name": "Wi‑Fi"}).to_string()}); diff --git a/src/app.rs b/src/app.rs index 1f4b660..a5ddaff 100644 --- a/src/app.rs +++ b/src/app.rs @@ -4222,6 +4222,8 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu - 像一個熟練的人那樣做事:動手前先想好要做哪幾步、做完怎樣才算成功(照「電腦現況」挑那台電腦上最直接的做法,不用再試探環境);選定一條路就做到底,不要在指令和畫面之間來回換。出錯先讀錯誤訊息想原因再換方法,同一個做法失敗兩次就換。做完只用一種最直接的方法確認一次(讀設定檔、查狀態、看一眼畫面),確認了就回答,不要用好幾種方法反覆確認。\n\ - 沒見過的 App 或畫面也一樣:先看清楚(截圖、需要就放大),找選單、設定、按鈕上的字,照編號點;網頁上的東西用 browser_* 的編號。\n\ - 使用者交代的事就做完:不要做到一半停下來問「要我繼續嗎」「要我現在做嗎」,畫面不如預期就自己處理(關掉擋路的視窗、打開要用的 App)再接著做。只有需要使用者親手做、或要先確認會切斷連線、刪除、付款這類事情時才停下來問。\n\ + - 系統跳出的權限或安全詢問(「允許…存取」「要允許…嗎」、輸入密碼、Touch ID)不要替使用者按:告訴他是什麼、請他自己決定(需要時用 hand_over_computer)。\n\ + - 按快捷鍵(cmd+w、cmd+q、cmd+s…)前先確定最前面的是你要操作的 App;關掉一個視窗以後,最前面的可能已經換成別的 App。\n\ - 做不到就照實說哪裡卡住、需要什麼。絕對不要用變通的替代品假裝做到(例如只改環境變數讓時鐘看起來對、自己寫一個同名的視窗或程式),也不要做使用者沒要的額外修改。\n\ - 會切斷你和那台電腦連線的事(關網路、Wi-Fi、VPN、Tailscale、重開機、登出):先告訴使用者做了會斷線、問他要不要做,他說好再做,做之前先把之後要確認的事情想好。\n\ {}\ diff --git a/src/computer.rs b/src/computer.rs index a1b9518..40ab4bc 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -802,7 +802,29 @@ pub fn batch(v: &Value, scale: (f64, f64)) -> Result { /// Do one mouse or keyboard step at the screen's own coordinates (`scale`: screen points per /// picture pixel). The bot's tools and a person tapping on the phone's picture both come here: /// one way of acting on a screen. +/// The app in front at the latest look (its pid): keys go to whatever is in front, so they wait +/// until that is still the app the bot was looking at. +static LOOKED_AT: Mutex> = Mutex::new(None); + +/// Keys and typing into no field land in the app in front. When that is no longer the app of the +/// latest look (the bot closed its window, the person switched, a dialog took over), they are not +/// sent: a cmd+w meant for TextEdit reached the terminal the person was working in. +fn keys_go_where_looked(name: &str, v: &Value) -> Result<(), String> { + let keyboard = name == "computer_use_press_key" || (name == "computer_use_typing" && v["ref"].is_null()); + if !keyboard { + return Ok(()); + } + let looked = *LOOKED_AT.lock().unwrap_or_else(|e| e.into_inner()); + match (looked, mac::front_app()) { + (Some(was), Some((now_name, now))) if was != now => Err(format!( + "沒有按:現在最前面的是「{now_name}」,不是你上一次看畫面時在前面的 App。按鍵會送到最前面的那個,先用 computer_use_open_app 把要操作的 App 叫到前面,或先截圖看一下,再按。" + )), + _ => Ok(()), + } +} + pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result { + keys_go_where_looked(name, v)?; let button = v["button"].as_str().unwrap_or("left"); match name { "computer_use_click" => { @@ -938,6 +960,7 @@ fn look(after: Option, pointer: bool) -> Result<(String, Option) // A number is a promise that a click there lands in this app: a control another window // covers (or one on another desktop) gets none. let hidden = cover_check(window, &mut els); + *LOOKED_AT.lock().unwrap_or_else(|e| e.into_inner()) = mac::front_app().map(|(_, pid)| pid); // The menu bar's status icons (Tailscale, Wi-Fi, a VPN…) belong to no window in front, so // they had no numbers and a bot cropped and measured screenshots to find them. #[cfg(target_os = "macos")] @@ -1593,6 +1616,23 @@ mod mac { mod tests { use super::*; + /// Keys wait when the app in front isn't the one of the latest look (no real key is pressed + /// here: the check refuses before anything is sent, or there is no front app at all). + #[test] + fn keys_wait_when_another_app_came_to_the_front() { + *LOOKED_AT.lock().unwrap() = Some(-424242); + let refused = keys_go_where_looked("computer_use_press_key", &json!({"key": "cmd+w"})); + if mac::front_app().is_some() { + assert!(refused.unwrap_err().contains("現在最前面的是")); + } else { + assert!(refused.is_ok()); + } + assert!(keys_go_where_looked("computer_use_click", &json!({"ref": "3"})).is_ok(), "a click brings its own app forward"); + assert!(keys_go_where_looked("computer_use_typing", &json!({"ref": "3", "value": "x"})).is_ok(), "typing into a named field clicks it first"); + *LOOKED_AT.lock().unwrap() = None; + assert!(keys_go_where_looked("computer_use_press_key", &json!({"key": "escape"})).is_ok(), "no look yet: nothing to compare"); + } + #[test] fn menu_bar_icons_are_named_plainly() { assert_eq!(icon_name("MenuBarAgent", "Wi‑Fi,已連接,3格"), "Wi‑Fi,已連接,3格");