diff --git a/src/agent.rs b/src/agent.rs index 7ae3c57..7d30f83 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -234,6 +234,14 @@ fn keep_to_chosen(job: &Job, history: &[Message], args: &str) -> (String, Option } } +/// What element `e7` is, from the latest app list in the chat (「按鈕「7」」). +fn element_named(history: &[Message], r: &str) -> Option { + let tag = format!("{r} "); + history.iter().rev().filter(|m| m.role == Role::Tool).take(12).find_map(|m| { + m.text.lines().find_map(|l| l.trim_start().strip_prefix(&tag).map(|rest| rest.trim().to_string())).filter(|e| !e.is_empty()) + }) +} + /// A step's result, with a word first when the step was moved back to the chosen computer. fn noted(moved: &std::collections::HashMap, id: &str, text: &str) -> String { match moved.get(id) { @@ -508,7 +516,9 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< let what = if is_mac { format!("{} {}", crate::macapps::label(&call.name), crate::macapps::describe(&call.name, &call.args)) } else if is_screen { - format!("{} {}", computer::label(&call.name), computer::describe(&call.name, &call.args)) + // A control by its number: name it from the app's list the bot last read. + let named = tools::args_of(&call.args)["ref"].as_str().map(str::trim).filter(|r| !r.is_empty()).and_then(|r| element_named(&history, r)).map(|e| format!("({e})")).unwrap_or_default(); + format!("{} {}{named}", computer::label(&call.name), computer::describe(&call.name, &call.args)) } else if is_web { web::describe(&call.name, &call.args) } else { @@ -1132,6 +1142,15 @@ mod tests { assert!(!mentions("在 macOS 的登入機制下,幫 Bot 的電腦截圖", &mac, &roles)); } + #[test] + fn a_click_by_number_is_named_in_the_question() { + let list = Message::tool_result("c1", "computer_use_read_app", "[王小明的Mac mini] 「計算機」的畫面:\n 文字「0」\n e4 按鈕「全部清除」\n e7 按鈕「7」"); + assert_eq!(element_named(&[list.clone()], "e7").as_deref(), Some("按鈕「7」")); + assert_eq!(element_named(&[list], "e9"), None); + assert_eq!(computer::describe("computer_use_click", r#"{"ref":"e7"}"#), "點 e7"); + assert_eq!(computer::describe("computer_use_click", r#"{"x":10,"y":20}"#), "點 (10, 20)"); + } + #[test] fn every_computer_answers_to_its_own_name() { // One name per computer: the bot's own goes by the bot's name, the others by theirs, and diff --git a/src/computer.rs b/src/computer.rs index 6890f34..dddc149 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -161,7 +161,12 @@ fn args_of(args: &str) -> Value { pub fn describe(name: &str, args: &str) -> String { let v = args_of(args); - let xy = || format!("({}, {})", v["x"].as_f64().unwrap_or(0.0), v["y"].as_f64().unwrap_or(0.0)); + // By number from the app's list (e7), else where on the screen (it said "(0, 0)" for every + // click by number). + let xy = || match v["ref"].as_str().map(str::trim).filter(|r| !r.is_empty()) { + Some(r) => r.to_string(), + None => format!("({}, {})", v["x"].as_f64().unwrap_or(0.0), v["y"].as_f64().unwrap_or(0.0)), + }; match name { "computer_use_screenshot" => "螢幕截圖".into(), "computer_use_click" => { @@ -172,7 +177,8 @@ pub fn describe(name: &str, args: &str) -> String { "computer_use_typing" => { let t = v["value"].as_str().unwrap_or(""); let short: String = t.chars().take(60).collect(); - format!("「{short}{}」", if t.chars().count() > 60 { "…" } else { "" }) + let at = v["ref"].as_str().filter(|r| !r.trim().is_empty()).map(|r| format!("在 {r} ")).unwrap_or_default(); + format!("{at}「{short}{}」", if t.chars().count() > 60 { "…" } else { "" }) } "computer_use_press_key" => v["key"].as_str().unwrap_or("").to_string(), "computer_use_scroll" => format!("{} {} 格,在 {}", v["direction"].as_str().unwrap_or("down"), v["amount"].as_i64().unwrap_or(3), xy()),