The same click twice in a step is done once, a front app that vanishes for a moment is looked at again, and the notes have the Mac and held-out results

- Two identical clicks in one step opened the Mac's Wi‑Fi menu and closed it again: the second is
  skipped with a word on why (count: 2 for a double click).
- macOS Calculator had no focused window for a moment after 「乘」 and the list came back with
  only the menu bar: an empty or failed read is tried once more 0.3 s later.
- Benchmark notes: three more unseen apps on box v9 (16/16, GTK 2 apps 4-10x faster with gail)
  and the Mac mini run with Screen Recording (zoom, Wi‑Fi menu, Calculator checked by eye).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS

bench/computer-use/README.md

@@ -48,3 +48,11 @@ env ANYBOT_BOX_NAME=anybot-box-bench ANYBOT_BOX_VOLUME=anybot-box-bench-home ANY
 改 Chromium 搜尋引擎、Chromium 加書籤、Mousepad 存檔、填表、計算機，「分開」各 3 次、「不分開」各 2 次：25/25 成功、網址打壞 0 次；平均一題「分開」46 秒、「不分開」67 秒。先前最難的改搜尋引擎從 2–6 分鐘（還常失敗）降到 46–147 秒。
 
 找到的：快速模型對編號會對錯（改用名字）、一串按鍵塞不進 8 個的批次、box 上一個鍵一個鍵打進 Chromium 網址列會被剛開啟的頁面和自動完成攪亂（冒號掉、舊字混進來）、box 判斷單行輸入框時多了「#14（」前綴所以全選和貼上沒執行、同時給名字和 (0,0) 時點到角落、快速模型在陌生對話框一直猜名字。
+
+## 再加三題沒見過的 App（v11，box 版本 9 加上 gail）
+
+lxappearance 換佈景、PCManFM 建資料夾、Mousepad 取代存檔（加上 Mousepad 存新檔），兩種模式各 2 次：16/16 成功。GTK 2 的 lxappearance 和 PCManFM 原本在 box 裡一個編號都沒有（少了 gail），「不分開」換佈景 203–455 秒 → 24–78 秒、PCManFM 514–655 秒 → 58–70 秒；「分開」的 PCManFM 從一次中途停下來問、一次 238 秒，變成 54–68 秒兩次都對，全程用檔案管理員。
+
+## Mac mini 上實測（有螢幕錄製權限）
+
+放大看時鐘（2 步、13 秒）、打開 Wi‑Fi 選單看連的網路（答對，選單關掉）、用 macOS 的「計算機」按按鈕算 1234×5678（螢幕上確實顯示 1,234×5,678 = 7,006,652，另外截圖親眼看過）、截圖描述畫面（1 步、22 秒），都沒有改任何設定。看到的問題：同一步重複點兩下把選單又關掉（已擋）、計算機按了「乘」之後有一瞬間讀不到前面的 App（已改成再讀一次）、快速模型照編號按又按錯（出兩次問題後交給 Bot 的模型才順）。

src/agent.rs

@@ -1052,7 +1052,20 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc<
             // (Each call's computer was already kept to the chosen one, see keep_to_chosen.)
             // Whether any step this round failed, was refused or repeated (see StepPlan::after).
             let trouble = std::cell::Cell::new(false);
+            // The previous call of this step, to catch the same click sent twice in a row.
+            let mut previous: Option<(String, String)> = None;
             for asked in calls {
+                // The same click twice in one step opens a menu and closes it again (a Wi‑Fi
+                // menu did exactly that): the second one isn't done.
+                let twice = matches!(asked.name.as_str(), "computer_use_click" | "browser_click") && previous.as_ref() == Some(&(asked.name.clone(), asked.args.clone()));
+                previous = Some((asked.name.clone(), asked.args.clone()));
+                if twice {
+                    send(StreamEvent::ToolStatus(asked.id.clone(), ToolStatus::Failed));
+                    let result = Message::tool_result(&asked.id, &asked.name, "沒有執行：和這一步上一個點擊一模一樣。連點同一個地方兩次會把剛打開的選單又關掉；要點兩下請用 count: 2，看完結果再決定下一步。");
+                    send(StreamEvent::ToolResult(result.clone()));
+                    history.push(result);
+                    continue;
+                }
                 // What runs: the call as asked, on the chosen computer (see keep_to_chosen).
                 let call = ToolCall { args: runs_with.get(&asked.id).cloned().unwrap_or_else(|| asked.args.clone()), ..asked.clone() };
                 // 鬼打牆: the same screen or browser step again and again (a click on the same
@@ -1971,6 +1984,39 @@ mod tests {
         serde_json::json!({"tool_calls": [{"index": 0, "id": id, "function": {"name": name, "arguments": args.to_string()}}]})
     }
 
+    /// The same click twice in one step: the second isn't done (it closed a menu the first had
+    /// just opened). The clicks name a computer that isn't there, so nothing is clicked here.
+    #[test]
+    fn the_same_click_twice_in_a_step_is_done_once() {
+        let click = serde_json::json!({"name": "computer_use_click", "arguments": serde_json::json!({"computer": "沒有這台", "name": "Wi‑Fi"}).to_string()});
+        let two = serde_json::json!({"tool_calls": [
+            {"index": 0, "id": "a", "function": click.clone()},
+            {"index": 1, "id": "b", "function": click},
+        ]});
+        let (port, _) = recording_model(vec![two, serde_json::json!({"content": "好了"})]);
+        let job = Job {
+            req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("打開 Wi‑Fi 選單")], tools: tools::specs(), session: String::new() },
+            ..test_job()
+        };
+        let (tx, rx) = channel();
+        spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default());
+        let mut results = vec![];
+        loop {
+            match rx.recv_timeout(Duration::from_secs(20)).expect("loop stalled") {
+                (_, StreamEvent::ToolResult(m)) => results.push(m.text),
+                (_, StreamEvent::NeedApproval(a)) => {
+                    let _ = a.reply.send(ApprovalAnswer::Reject);
+                }
+                (_, StreamEvent::Done) => break,
+                (_, StreamEvent::Error(e)) => panic!("{e}"),
+                _ => {}
+            }
+        }
+        assert_eq!(results.len(), 2, "{results:?}");
+        assert!(!results[0].contains("一模一樣"), "{results:?}");
+        assert!(results[1].contains("和這一步上一個點擊一模一樣"), "{results:?}");
+    }
+
     #[test]
     fn a_control_is_found_again_by_what_it_is() {
         assert_eq!(control_key("勾選框「自動更新」：1（已勾）"), "勾選框「自動更新」");

src/computer.rs

@@ -902,7 +902,12 @@ fn look(after: Option<String>, pointer: bool) -> Result<(String, Option<String>)
     #[cfg(not(target_os = "linux"))]
     let front = String::new();
     let (found, why) = if controls_allowed() {
-        match crate::desktop::elements(&front).or_else(|e| if front.is_empty() { Err(e) } else { crate::desktop::elements("") }) {
+        let read = || crate::desktop::elements(&front).or_else(|e| if front.is_empty() { Err(e) } else { crate::desktop::elements("") });
+        // Right after a click some apps have no focused window for a moment (macOS Calculator
+        // after 「乘」): the list came back with only the menu bar. One more look shortly after.
+        let first = read();
+        let again = first.is_err() || first.as_ref().is_ok_and(|f| f.0.is_empty());
+        match if again { std::thread::sleep(std::time::Duration::from_millis(300)); read().or(first) } else { first } {
             Ok(f) => (Some(f), String::new()),
             Err(e) => (None, e),
         }