Keys wait until the app the bot looked at is still in front; the quick model names its controls; system prompts are the person's

- Keys and typing into no field go to whatever is in front: a cmd+w meant for TextEdit reached
  the person's terminal after the TextEdit window had closed (its "close this window?" came up
  and was cancelled). They are now refused, with why, when the app in front isn't the one of the
  latest look.
- The quick model's screen tools have no `ref`: looking numbers up is where it pressed the wrong
  keys; it names controls (two of a name come back as trouble for the bot's own model).
- Instructions: macOS permission and security prompts are for the person to answer (a bot
  pressed 「不允許」 on one), and check what's in front before a shortcut.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS

src/agent.rs

@@ -855,6 +855,32 @@ impl StepPlan {
     }
 }
 
+/// The quick model's screen tools name their controls only: looking a number up in the list
+/// is where it went wrong (1, 2, 3, then 「1」's number again for 4). Without `ref` it says which
+/// button the way a person would; two of the same name come back as trouble for the bot's own
+/// model.
+fn quick_tools(tools: &[serde_json::Value]) -> Vec<serde_json::Value> {
+    tools
+        .iter()
+        .map(|t| {
+            let mut t = t.clone();
+            let screen = t["name"].as_str().is_some_and(|n| matches!(n, "computer_use_click" | "computer_use_typing" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_batch"));
+            if screen {
+                if let Some(p) = t["parameters"]["properties"].as_object_mut() {
+                    p.remove("ref");
+                }
+                if let Some(p) = t["parameters"]["properties"]["actions"]["items"]["properties"].as_object_mut() {
+                    p.remove("ref");
+                }
+                if let Some(d) = t["description"].as_str() {
+                    t["description"] = serde_json::Value::String(format!("{d} Name the control exactly as the latest list shows it (name); there are no numbers here."));
+                }
+            }
+            t
+        })
+        .collect()
+}
+
 /// Which way a tool goes: commands and files, the screen, or a web page.
 fn route_of(name: &str) -> &'static str {
     if computer::is_computer_tool(name) || crate::macapps::is_mac_tool(name) {
@@ -968,7 +994,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc<
                 let main_req = ChatRequest { history: history.clone(), ..job.req.clone() };
                 let quick_now = if plan.quick_is_off() { None } else { quick.clone() };
                 let got = match quick_now {
-                    Some(q) => match providers::run_step(&ChatRequest { model: q.clone(), ..main_req.clone() }, &cancel, &quiet) {
+                    Some(q) => match providers::run_step(&ChatRequest { model: q.clone(), tools: quick_tools(&main_req.tools), ..main_req.clone() }, &cancel, &quiet) {
                         Ok(o) => Ok(o),
                         // Too long is the chat's problem, not the quick model's: folded below.
                         Err(e) if providers::is_context_overflow(&e) => Err(e),
@@ -1986,6 +2012,23 @@ mod tests {
 
     /// The same click twice in one step: the second isn't done (it closed a menu the first had
     /// just opened). The clicks name a computer that isn't there, so nothing is clicked here.
+    #[test]
+    fn the_quick_models_screen_tools_name_controls_only() {
+        let specs = crate::computer::specs_for(&["C".to_string()], "C", true);
+        let quick = quick_tools(&specs);
+        for t in &quick {
+            let n = t["name"].as_str().unwrap();
+            if matches!(n, "computer_use_click" | "computer_use_typing" | "computer_use_scroll" | "computer_use_mouse_move") {
+                assert!(t["parameters"]["properties"].get("ref").is_none() && t["parameters"]["properties"].get("name").is_some(), "{n}");
+            }
+            if n == "computer_use_batch" {
+                assert!(t["parameters"]["properties"]["actions"]["items"]["properties"].get("ref").is_none());
+            }
+        }
+        // The bot's own model keeps both.
+        assert!(specs.iter().any(|t| t["name"] == "computer_use_click" && t["parameters"]["properties"].get("ref").is_some()));
+    }
+
     #[test]
     fn the_same_click_twice_in_a_step_is_done_once() {
         let click = serde_json::json!({"name": "computer_use_click", "arguments": serde_json::json!({"computer": "沒有這台", "name": "Wi‑Fi"}).to_string()});

src/app.rs

@@ -4222,6 +4222,8 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu
          - 像一個熟練的人那樣做事：動手前先想好要做哪幾步、做完怎樣才算成功（照「電腦現況」挑那台電腦上最直接的做法，不用再試探環境）；選定一條路就做到底，不要在指令和畫面之間來回換。出錯先讀錯誤訊息想原因再換方法，同一個做法失敗兩次就換。做完只用一種最直接的方法確認一次（讀設定檔、查狀態、看一眼畫面），確認了就回答，不要用好幾種方法反覆確認。\n\
          - 沒見過的 App 或畫面也一樣：先看清楚（截圖、需要就放大），找選單、設定、按鈕上的字，照編號點；網頁上的東西用 browser_* 的編號。\n\
          - 使用者交代的事就做完：不要做到一半停下來問「要我繼續嗎」「要我現在做嗎」，畫面不如預期就自己處理（關掉擋路的視窗、打開要用的 App）再接著做。只有需要使用者親手做、或要先確認會切斷連線、刪除、付款這類事情時才停下來問。\n\
+         - 系統跳出的權限或安全詢問（「允許…存取」「要允許…嗎」、輸入密碼、Touch ID）不要替使用者按：告訴他是什麼、請他自己決定（需要時用 hand_over_computer）。\n\
+         - 按快捷鍵（cmd+w、cmd+q、cmd+s…）前先確定最前面的是你要操作的 App；關掉一個視窗以後，最前面的可能已經換成別的 App。\n\
          - 做不到就照實說哪裡卡住、需要什麼。絕對不要用變通的替代品假裝做到（例如只改環境變數讓時鐘看起來對、自己寫一個同名的視窗或程式），也不要做使用者沒要的額外修改。\n\
          - 會切斷你和那台電腦連線的事（關網路、Wi-Fi、VPN、Tailscale、重開機、登出）：先告訴使用者做了會斷線、問他要不要做，他說好再做，做之前先把之後要確認的事情想好。\n\
 {}\

src/computer.rs

@@ -802,7 +802,29 @@ pub fn batch(v: &Value, scale: (f64, f64)) -> Result<String, String> {
 /// Do one mouse or keyboard step at the screen's own coordinates (`scale`: screen points per
 /// picture pixel). The bot's tools and a person tapping on the phone's picture both come here:
 /// one way of acting on a screen.
+/// The app in front at the latest look (its pid): keys go to whatever is in front, so they wait
+/// until that is still the app the bot was looking at.
+static LOOKED_AT: Mutex<Option<i32>> = Mutex::new(None);
+
+/// Keys and typing into no field land in the app in front. When that is no longer the app of the
+/// latest look (the bot closed its window, the person switched, a dialog took over), they are not
+/// sent: a cmd+w meant for TextEdit reached the terminal the person was working in.
+fn keys_go_where_looked(name: &str, v: &Value) -> Result<(), String> {
+    let keyboard = name == "computer_use_press_key" || (name == "computer_use_typing" && v["ref"].is_null());
+    if !keyboard {
+        return Ok(());
+    }
+    let looked = *LOOKED_AT.lock().unwrap_or_else(|e| e.into_inner());
+    match (looked, mac::front_app()) {
+        (Some(was), Some((now_name, now))) if was != now => Err(format!(
+            "沒有按：現在最前面的是「{now_name}」，不是你上一次看畫面時在前面的 App。按鍵會送到最前面的那個，先用 computer_use_open_app 把要操作的 App 叫到前面，或先截圖看一下，再按。"
+        )),
+        _ => Ok(()),
+    }
+}
+
 pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result<String, String> {
+    keys_go_where_looked(name, v)?;
     let button = v["button"].as_str().unwrap_or("left");
     match name {
         "computer_use_click" => {
@@ -938,6 +960,7 @@ fn look(after: Option<String>, pointer: bool) -> Result<(String, Option<String>)
     // A number is a promise that a click there lands in this app: a control another window
     // covers (or one on another desktop) gets none.
     let hidden = cover_check(window, &mut els);
+    *LOOKED_AT.lock().unwrap_or_else(|e| e.into_inner()) = mac::front_app().map(|(_, pid)| pid);
     // The menu bar's status icons (Tailscale, Wi-Fi, a VPN…) belong to no window in front, so
     // they had no numbers and a bot cropped and measured screenshots to find them.
     #[cfg(target_os = "macos")]
@@ -1593,6 +1616,23 @@ mod mac {
 mod tests {
     use super::*;
 
+    /// Keys wait when the app in front isn't the one of the latest look (no real key is pressed
+    /// here: the check refuses before anything is sent, or there is no front app at all).
+    #[test]
+    fn keys_wait_when_another_app_came_to_the_front() {
+        *LOOKED_AT.lock().unwrap() = Some(-424242);
+        let refused = keys_go_where_looked("computer_use_press_key", &json!({"key": "cmd+w"}));
+        if mac::front_app().is_some() {
+            assert!(refused.unwrap_err().contains("現在最前面的是"));
+        } else {
+            assert!(refused.is_ok());
+        }
+        assert!(keys_go_where_looked("computer_use_click", &json!({"ref": "3"})).is_ok(), "a click brings its own app forward");
+        assert!(keys_go_where_looked("computer_use_typing", &json!({"ref": "3", "value": "x"})).is_ok(), "typing into a named field clicks it first");
+        *LOOKED_AT.lock().unwrap() = None;
+        assert!(keys_go_where_looked("computer_use_press_key", &json!({"key": "escape"})).is_ok(), "no look yet: nothing to compare");
+    }
+
     #[test]
     fn menu_bar_icons_are_named_plainly() {
         assert_eq!(icon_name("MenuBarAgent", "Wi‑Fi，已連接，3格"), "Wi‑Fi，已連接，3格");