diff --git a/bench/computer-use/README.md b/bench/computer-use/README.md index ac0beaf..54732a7 100644 --- a/bench/computer-use/README.md +++ b/bench/computer-use/README.md @@ -48,3 +48,11 @@ env ANYBOT_BOX_NAME=anybot-box-bench ANYBOT_BOX_VOLUME=anybot-box-bench-home ANY 改 Chromium 搜尋引擎、Chromium 加書籤、Mousepad 存檔、填表、計算機,「分開」各 3 次、「不分開」各 2 次:25/25 成功、網址打壞 0 次;平均一題「分開」46 秒、「不分開」67 秒。先前最難的改搜尋引擎從 2–6 分鐘(還常失敗)降到 46–147 秒。 找到的:快速模型對編號會對錯(改用名字)、一串按鍵塞不進 8 個的批次、box 上一個鍵一個鍵打進 Chromium 網址列會被剛開啟的頁面和自動完成攪亂(冒號掉、舊字混進來)、box 判斷單行輸入框時多了「#14(」前綴所以全選和貼上沒執行、同時給名字和 (0,0) 時點到角落、快速模型在陌生對話框一直猜名字。 + +## 再加三題沒見過的 App(v11,box 版本 9 加上 gail) + +lxappearance 換佈景、PCManFM 建資料夾、Mousepad 取代存檔(加上 Mousepad 存新檔),兩種模式各 2 次:16/16 成功。GTK 2 的 lxappearance 和 PCManFM 原本在 box 裡一個編號都沒有(少了 gail),「不分開」換佈景 203–455 秒 → 24–78 秒、PCManFM 514–655 秒 → 58–70 秒;「分開」的 PCManFM 從一次中途停下來問、一次 238 秒,變成 54–68 秒兩次都對,全程用檔案管理員。 + +## Mac mini 上實測(有螢幕錄製權限) + +放大看時鐘(2 步、13 秒)、打開 Wi‑Fi 選單看連的網路(答對,選單關掉)、用 macOS 的「計算機」按按鈕算 1234×5678(螢幕上確實顯示 1,234×5,678 = 7,006,652,另外截圖親眼看過)、截圖描述畫面(1 步、22 秒),都沒有改任何設定。看到的問題:同一步重複點兩下把選單又關掉(已擋)、計算機按了「乘」之後有一瞬間讀不到前面的 App(已改成再讀一次)、快速模型照編號按又按錯(出兩次問題後交給 Bot 的模型才順)。 diff --git a/src/agent.rs b/src/agent.rs index 2ec16d6..6490c0e 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -1052,7 +1052,20 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< // (Each call's computer was already kept to the chosen one, see keep_to_chosen.) // Whether any step this round failed, was refused or repeated (see StepPlan::after). let trouble = std::cell::Cell::new(false); + // The previous call of this step, to catch the same click sent twice in a row. + let mut previous: Option<(String, String)> = None; for asked in calls { + // The same click twice in one step opens a menu and closes it again (a Wi‑Fi + // menu did exactly that): the second one isn't done. + let twice = matches!(asked.name.as_str(), "computer_use_click" | "browser_click") && previous.as_ref() == Some(&(asked.name.clone(), asked.args.clone())); + previous = Some((asked.name.clone(), asked.args.clone())); + if twice { + send(StreamEvent::ToolStatus(asked.id.clone(), ToolStatus::Failed)); + let result = Message::tool_result(&asked.id, &asked.name, "沒有執行:和這一步上一個點擊一模一樣。連點同一個地方兩次會把剛打開的選單又關掉;要點兩下請用 count: 2,看完結果再決定下一步。"); + send(StreamEvent::ToolResult(result.clone())); + history.push(result); + continue; + } // What runs: the call as asked, on the chosen computer (see keep_to_chosen). let call = ToolCall { args: runs_with.get(&asked.id).cloned().unwrap_or_else(|| asked.args.clone()), ..asked.clone() }; // 鬼打牆: the same screen or browser step again and again (a click on the same @@ -1971,6 +1984,39 @@ mod tests { serde_json::json!({"tool_calls": [{"index": 0, "id": id, "function": {"name": name, "arguments": args.to_string()}}]}) } + /// The same click twice in one step: the second isn't done (it closed a menu the first had + /// just opened). The clicks name a computer that isn't there, so nothing is clicked here. + #[test] + fn the_same_click_twice_in_a_step_is_done_once() { + let click = serde_json::json!({"name": "computer_use_click", "arguments": serde_json::json!({"computer": "沒有這台", "name": "Wi‑Fi"}).to_string()}); + let two = serde_json::json!({"tool_calls": [ + {"index": 0, "id": "a", "function": click.clone()}, + {"index": 1, "id": "b", "function": click}, + ]}); + let (port, _) = recording_model(vec![two, serde_json::json!({"content": "好了"})]); + let job = Job { + req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("打開 Wi‑Fi 選單")], tools: tools::specs(), session: String::new() }, + ..test_job() + }; + let (tx, rx) = channel(); + spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default()); + let mut results = vec![]; + loop { + match rx.recv_timeout(Duration::from_secs(20)).expect("loop stalled") { + (_, StreamEvent::ToolResult(m)) => results.push(m.text), + (_, StreamEvent::NeedApproval(a)) => { + let _ = a.reply.send(ApprovalAnswer::Reject); + } + (_, StreamEvent::Done) => break, + (_, StreamEvent::Error(e)) => panic!("{e}"), + _ => {} + } + } + assert_eq!(results.len(), 2, "{results:?}"); + assert!(!results[0].contains("一模一樣"), "{results:?}"); + assert!(results[1].contains("和這一步上一個點擊一模一樣"), "{results:?}"); + } + #[test] fn a_control_is_found_again_by_what_it_is() { assert_eq!(control_key("勾選框「自動更新」:1(已勾)"), "勾選框「自動更新」"); diff --git a/src/computer.rs b/src/computer.rs index e3cec90..a1b9518 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -902,7 +902,12 @@ fn look(after: Option, pointer: bool) -> Result<(String, Option) #[cfg(not(target_os = "linux"))] let front = String::new(); let (found, why) = if controls_allowed() { - match crate::desktop::elements(&front).or_else(|e| if front.is_empty() { Err(e) } else { crate::desktop::elements("") }) { + let read = || crate::desktop::elements(&front).or_else(|e| if front.is_empty() { Err(e) } else { crate::desktop::elements("") }); + // Right after a click some apps have no focused window for a moment (macOS Calculator + // after 「乘」): the list came back with only the menu bar. One more look shortly after. + let first = read(); + let again = first.is_err() || first.as_ref().is_ok_and(|f| f.0.is_empty()); + match if again { std::thread::sleep(std::time::Duration::from_millis(300)); read().or(first) } else { first } { Ok(f) => (Some(f), String::new()), Err(e) => (None, e), }