A click by number reads as the element it is in the question (點 e7（按鈕「7」）), not (0, 0)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/agent.rs

@@ -234,6 +234,14 @@ fn keep_to_chosen(job: &Job, history: &[Message], args: &str) -> (String, Option
     }
 }
 
+/// What element `e7` is, from the latest app list in the chat (「按鈕「7」」).
+fn element_named(history: &[Message], r: &str) -> Option<String> {
+    let tag = format!("{r} ");
+    history.iter().rev().filter(|m| m.role == Role::Tool).take(12).find_map(|m| {
+        m.text.lines().find_map(|l| l.trim_start().strip_prefix(&tag).map(|rest| rest.trim().to_string())).filter(|e| !e.is_empty())
+    })
+}
+
 /// A step's result, with a word first when the step was moved back to the chosen computer.
 fn noted(moved: &std::collections::HashMap<String, String>, id: &str, text: &str) -> String {
     match moved.get(id) {
@@ -508,7 +516,9 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc<
                 let what = if is_mac {
                     format!("{} {}", crate::macapps::label(&call.name), crate::macapps::describe(&call.name, &call.args))
                 } else if is_screen {
-                    format!("{} {}", computer::label(&call.name), computer::describe(&call.name, &call.args))
+                    // A control by its number: name it from the app's list the bot last read.
+                    let named = tools::args_of(&call.args)["ref"].as_str().map(str::trim).filter(|r| !r.is_empty()).and_then(|r| element_named(&history, r)).map(|e| format!("（{e}）")).unwrap_or_default();
+                    format!("{} {}{named}", computer::label(&call.name), computer::describe(&call.name, &call.args))
                 } else if is_web {
                     web::describe(&call.name, &call.args)
                 } else {
@@ -1132,6 +1142,15 @@ mod tests {
         assert!(!mentions("在 macOS 的登入機制下，幫 Bot 的電腦截圖", &mac, &roles));
     }
 
+    #[test]
+    fn a_click_by_number_is_named_in_the_question() {
+        let list = Message::tool_result("c1", "computer_use_read_app", "[王小明的Mac mini] 「計算機」的畫面：\n          文字「0」\n        e4 按鈕「全部清除」\n        e7 按鈕「7」");
+        assert_eq!(element_named(&[list.clone()], "e7").as_deref(), Some("按鈕「7」"));
+        assert_eq!(element_named(&[list], "e9"), None);
+        assert_eq!(computer::describe("computer_use_click", r#"{"ref":"e7"}"#), "點 e7");
+        assert_eq!(computer::describe("computer_use_click", r#"{"x":10,"y":20}"#), "點 (10, 20)");
+    }
+
     #[test]
     fn every_computer_answers_to_its_own_name() {
         // One name per computer: the bot's own goes by the bot's name, the others by theirs, and

src/computer.rs

@@ -161,7 +161,12 @@ fn args_of(args: &str) -> Value {
 
 pub fn describe(name: &str, args: &str) -> String {
     let v = args_of(args);
-    let xy = || format!("({}, {})", v["x"].as_f64().unwrap_or(0.0), v["y"].as_f64().unwrap_or(0.0));
+    // By number from the app's list (e7), else where on the screen (it said "(0, 0)" for every
+    // click by number).
+    let xy = || match v["ref"].as_str().map(str::trim).filter(|r| !r.is_empty()) {
+        Some(r) => r.to_string(),
+        None => format!("({}, {})", v["x"].as_f64().unwrap_or(0.0), v["y"].as_f64().unwrap_or(0.0)),
+    };
     match name {
         "computer_use_screenshot" => "螢幕截圖".into(),
         "computer_use_click" => {
@@ -172,7 +177,8 @@ pub fn describe(name: &str, args: &str) -> String {
         "computer_use_typing" => {
             let t = v["value"].as_str().unwrap_or("");
             let short: String = t.chars().take(60).collect();
-            format!("「{short}{}」", if t.chars().count() > 60 { "…" } else { "" })
+            let at = v["ref"].as_str().filter(|r| !r.trim().is_empty()).map(|r| format!("在 {r} ")).unwrap_or_default();
+            format!("{at}「{short}{}」", if t.chars().count() > 60 { "…" } else { "" })
         }
         "computer_use_press_key" => v["key"].as_str().unwrap_or("").to_string(),
         "computer_use_scroll" => format!("{} {} 格，在 {}", v["direction"].as_str().unwrap_or("down"), v["amount"].as_i64().unwrap_or(3), xy()),