Merge c3/hi-tools: semantic actions (C3)

src/axwrite.rs

@@ -268,6 +268,111 @@ mod mac {
         let v = cfstr(&merged);
         unsafe { AXUIElementSetAttributeValue(focused.0, cfstr("AXValue").0, v.0) == 0 }
     }
+    /// Deepest AXMenuItem (in any open menu) whose words match `want` best (exact first).
+    fn find_menu_item(el: CFTypeRef, want: &str, best: &mut (u8, usize, CFTypeRef), seen: &mut usize, depth: usize) {
+        if *seen >= MAX_NODES || depth > MAX_DEPTH {
+            return;
+        }
+        *seen += 1;
+        if attr_string(el, "AXRole").is_some_and(|r| r == "AXMenuItem" || r == "AXMenuBarItem") {
+            let score = ["AXTitle", "AXDescription"].iter().map(|a| attr_string(el, a).map(|s| name_match(&s, want)).unwrap_or(0)).max().unwrap_or(0);
+            if score > best.0 {
+                *best = (score, depth, el);
+            }
+        }
+        let (_keep, kids) = children(el);
+        for k in kids {
+            find_menu_item(k, want, best, seen, depth + 1);
+        }
+    }
+
+    /// Walk the app's real menu bar: press 檔案, then 新視窗… (what took the bot three
+    /// screenshot rounds per level, menus being invisible to the screenshot numbering).
+    pub fn menu_click(pid: i32, path: &[String]) -> Result<String, String> {
+        let app = app(pid).ok_or("這個 App 沒有輔助使用介面")?;
+        let bar = attr(app.0, "AXMenuBar").ok_or("這個 App 沒有選單列")?;
+        let (_keep, items) = children(bar.0);
+        let want = path[0].trim();
+        let item = items
+            .iter()
+            .filter(|k| attr_string(**k, "AXRole").is_some_and(|r| r == "AXMenuBarItem"))
+            .max_by_key(|k| ["AXTitle", "AXDescription"].iter().map(|a| attr_string(**k, a).map(|s| name_match(&s, want)).unwrap_or(0)).max().unwrap_or(0))
+            .ok_or("選單列是空的")?;
+        let score = ["AXTitle", "AXDescription"].iter().map(|a| attr_string(*item, a).map(|s| name_match(&s, want)).unwrap_or(0)).max().unwrap_or(0);
+        if score == 0 {
+            let names = items.iter().filter_map(|k| attr_string(*k, "AXTitle")).collect::<Vec<_>>().join("、");
+            return Err(format!("選單列裡沒有「{want}」（有：{names}）"));
+        }
+        unsafe { CFRetain(*item) };
+        let r = press(*item, "AXMenuBarItem");
+        unsafe { CFRelease(*item) };
+        r?;
+        std::thread::sleep(std::time::Duration::from_millis(350));
+        for seg in &path[1..] {
+            let mut best = (0u8, 0usize, std::ptr::null());
+            let mut seen = 0;
+            find_menu_item(app.0, seg.trim(), &mut best, &mut seen, 0);
+            if best.2.is_null() {
+                return Err(format!("打開的選單裡沒有「{}」（前面已點：{}）", seg.trim(), path[..path.len() - 1].join("→")));
+            }
+            unsafe { CFRetain(best.2) };
+            let r = press(best.2, "AXMenuItem");
+            unsafe { CFRelease(best.2) };
+            r?;
+            std::thread::sleep(std::time::Duration::from_millis(250));
+        }
+        Ok(path.join(" → "))
+    }
+
+    /// Fill named fields with one AXValue write each (no clicks, no keystrokes). Names match
+    /// title / description / placeholder, as the numbered list shows them.
+    pub fn fill_form(pid: i32, fields: &[(String, String)]) -> Vec<(String, Result<(), String>)> {
+        let app = match app(pid) {
+            Some(a) => a,
+            None => return fields.iter().map(|(n, _)| (n.clone(), Err("這個 App 沒有輔助使用介面".into()))).collect(),
+        };
+        let mut editables: Vec<CFTypeRef> = Vec::new();
+        collect_editables(app.0, &mut editables, &mut 0, 0);
+        let mut out = Vec::new();
+        for (name, value) in fields {
+            let best = editables
+                .iter()
+                .filter_map(|e| {
+                    let s = name_match(&name_of(*e), name);
+                    (s > 0 && is_editable(*e)).then_some((s, *e))
+                })
+                .max_by_key(|(s, _)| *s);
+            out.push(match best {
+                Some((_, e)) => {
+                    let v = cfstr(value);
+                    let ok = unsafe { AXUIElementSetAttributeValue(e, cfstr("AXValue").0, v.0) == 0 };
+                    (name.clone(), if ok { Ok(()) } else { Err("這個欄位不接受直接寫入".into()) })
+                }
+                None => (name.clone(), Err("畫面裡沒有這個名字的欄位（先截圖看清單）".into())),
+            });
+        }
+        out
+    }
+
+    fn is_editable(el: CFTypeRef) -> bool {
+        attr_string(el, "AXRole").is_some_and(|r| EDITABLE_ROLES.contains(&r.as_str()))
+    }
+
+    fn name_of(el: CFTypeRef) -> String {
+        ["AXTitle", "AXDescription", "AXPlaceholderValue", "AXHelp"].iter().filter_map(|a| attr_string(el, a)).find(|s| !s.trim().is_empty()).unwrap_or_default()
+    }
+
+    fn collect_editables(el: CFTypeRef, out: &mut Vec<CFTypeRef>, seen: &mut usize, depth: usize) {
+        if *seen >= MAX_NODES || depth > MAX_DEPTH {
+            return;
+        }
+        *seen += 1;
+        out.push(el);
+        let (_keep, kids) = children(el);
+        for k in kids {
+            collect_editables(k, out, seen, depth + 1);
+        }
+    }
 }
 
 #[cfg(not(target_os = "macos"))]
@@ -282,9 +387,15 @@ mod mac {
     pub fn set_focused_text(_pid: i32, _text: &str, _append: bool) -> bool {
         false
     }
+    pub fn menu_click(_pid: i32, _path: &[String]) -> Result<String, String> {
+        Err("選單直走只在 macOS 有（其他系統照編號點選單）".into())
+    }
+    pub fn fill_form(_pid: i32, fields: &[(String, String)]) -> Vec<(String, Result<(), String>)> {
+        fields.iter().map(|(n, _)| (n.clone(), Err("選欄直填只在 macOS 有（其他系統請逐欄 click＋typing）".into()))).collect()
+    }
 }
 
-pub use mac::{press_at, press_named, set_focused_text};
+pub use mac::{fill_form, menu_click, press_at, press_named, set_focused_text};
 
 #[cfg(test)]
 mod tests {

src/computer.rs

@@ -12,7 +12,7 @@ use base64::Engine;
 use serde_json::{json, Value};
 use std::sync::Mutex;
 
-pub const NAMES: [&str; 12] = ["computer_use_screenshot", "computer_use_zoom", "computer_use_show", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app", "computer_use_batch"];
+pub const NAMES: [&str; 15] = ["computer_use_screenshot", "computer_use_zoom", "computer_use_show", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app", "computer_use_batch", "computer_use_menu_click", "computer_use_fill_form", "computer_use_wait_for"];
 /// Wait after an action before taking the picture that comes back with it (menus open, pages
 /// start to change; estimated). The box waits for the screen to stop changing instead, up to
 /// about this long.
@@ -246,6 +246,9 @@ pub fn label(name: &str) -> &'static str {
         "computer_use_drag" => "拖曳",
         "computer_use_check_permissions" => "檢查螢幕權限",
         "computer_use_open_app" => "打開 App",
+        "computer_use_menu_click" => "走選單",
+        "computer_use_fill_form" => "填表單",
+        "computer_use_wait_for" => "等畫面出現",
         "computer_use_batch" => "依序操作",
         _ => "操作畫面",
     }
@@ -297,6 +300,9 @@ pub fn describe(name: &str, args: &str) -> String {
         "computer_use_check_permissions" => "螢幕錄製、輔助使用".into(),
         "computer_use_open_app" => v["app"].as_str().filter(|a| !a.is_empty()).unwrap_or("").to_string(),
         "computer_use_batch" => format!("依序 {} 個動作", v["actions"].as_array().map(|a| a.len()).unwrap_or(0)),
+        "computer_use_menu_click" => format!("選單 {}", v["menu"].as_array().map(|a| a.iter().filter_map(|x| x.as_str()).collect::<Vec<_>>().join("→")).unwrap_or_default()),
+        "computer_use_fill_form" => format!("{} 個欄位", v["fields"].as_array().map(|a| a.len()).unwrap_or(0)),
+        "computer_use_wait_for" => format!("等「{}」", v["text"].as_str().unwrap_or("")),
         _ => String::new(),
     }
 }
@@ -621,7 +627,7 @@ pub fn run_by(name: &str, args: &str, who: &str) -> (String, bool, Option<String
 /// Steps that use the mouse or keyboard (and come back with the screen after them). A
 /// `computer` call never reaches here: every caller expands it first (`run_by`, the agent loop).
 pub fn moves_things(name: &str) -> bool {
-    matches!(name, "computer_use_click" | "computer_use_typing" | "computer_use_press_key" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_drag" | "computer_use_open_app" | "computer_use_batch")
+    matches!(name, "computer_use_click" | "computer_use_typing" | "computer_use_press_key" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_drag" | "computer_use_open_app" | "computer_use_batch" | "computer_use_menu_click" | "computer_use_fill_form")
 }
 
 /// Ask macOS for a permission (its own prompt, then 系統設定 at that pane) at most once every ten
@@ -891,6 +897,79 @@ fn press_keys(keys: &str, hold: f64) -> Result<(), String> {
 /// Run a planned sequence of actions, then one picture (in the box the box's own batch runs;
 /// here, the local screen). The first action that fails stops the rest — acting on a screen
 /// that isn't where you think it is makes it worse (Anthropic's computer_batch rule).
+/// Words the front app's controls show right now (names and values, lowercased): the cheap
+/// probe for `expect` and `wait_for` — a tree read, no screenshot.
+pub fn screen_words() -> Vec<String> {
+    let mut w = Vec::new();
+    if let Ok((_app, _win, els)) = crate::desktop::elements("") {
+        for e in els {
+            for t in [e.name, e.value] {
+                let t = t.trim().to_lowercase();
+                if !t.is_empty() {
+                    w.push(t);
+                }
+            }
+        }
+    }
+    w
+}
+
+/// Until `needle` shows (or disappears): the box's SETTLE probe made general. Replaces the
+/// screenshot-polling loops that filled whole turns with identical pictures.
+fn wait_for_tool(v: &Value) -> Result<String, String> {
+    let needle = v["text"].as_str().map(str::trim).filter(|s| !s.is_empty()).ok_or("text 要給要等的文字（畫面清單上的名字）")?;
+    let gone = v["gone"].as_bool().unwrap_or(false);
+    let budget = v["timeout_ms"].as_u64().unwrap_or(5000).clamp(200, 30_000);
+    let n = needle.to_lowercase();
+    let t = std::time::Instant::now();
+    loop {
+        let hit = screen_words().iter().any(|w| w.contains(&n));
+        if hit == !gone {
+            return Ok(format!("「{needle}」{}（等了{:.1} 秒）。", if gone { "消失了" } else { "出現了" }, t.elapsed().as_secs_f32()));
+        }
+        if t.elapsed().as_millis() as u64 > budget {
+            return Err(format!("等了 {} 秒，「{needle}」沒有{}：先截圖看畫面現在是什麼樣子，再決定下一步。", budget / 1000, if gone { "消失" } else { "出現" }));
+        }
+        std::thread::sleep(std::time::Duration::from_millis(300));
+    }
+}
+
+/// One call down the app's real menu bar (macOS accessibility): three screenshot rounds per
+/// menu level become one. Other systems: click the numbered menu entries as before.
+fn menu_click_tool(v: &Value) -> Result<String, String> {
+    let path: Vec<String> = v["menu"].as_array().map(|a| a.iter().filter_map(|x| x.as_str()).map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect()).unwrap_or_default();
+    if path.is_empty() {
+        return Err(r#"menu 要給選單路徑陣列，如 ["檔案", "新視窗"]"#.into());
+    }
+    let (app, pid) = mac::front_app().ok_or("看不到最前面的 App：先截圖確認")?;
+    crate::axwrite::menu_click(pid, &path).map(|done| format!("已在「{app}」走選單 {done}。"))
+}
+
+/// Named fields, filled with one accessibility write each; the picture that comes back is the
+/// check. Long forms: this is one step, not one round trip per field.
+fn fill_form_tool(v: &Value) -> Result<String, String> {
+    let fs: Vec<(String, String)> = v["fields"].as_array().map(|a| a.iter().filter_map(|f| Some((f["name"].as_str()?.to_string(), f["value"].as_str()?.to_string()))).collect()).unwrap_or_default();
+    if fs.is_empty() {
+        return Err("fields 要給 [{name, value}…]（name 用清單上的欄位名字）".into());
+    }
+    let (_, pid) = mac::front_app().ok_or("看不到最前面的 App：先截圖確認")?;
+    let rs = crate::axwrite::fill_form(pid, &fs);
+    let okn = rs.iter().filter(|(_, r)| r.is_ok()).count();
+    let lines: Vec<String> = rs.iter().map(|(n, r)| match r {
+        Ok(()) => format!("✓ {n}"),
+        Err(e) => format!("✗ {n}：{e}"),
+    }).collect();
+    if okn == 0 {
+        return Err(format!("一個欄位都填不進去：
+{}
+（先截圖看清單上的欄位名字，改用 click＋typing 逐欄填。）", lines.join("
+")));
+    }
+    Ok(format!("填了 {}/{} 個欄位：
+{}", okn, fs.len(), lines.join("
+")))
+}
+
 pub fn batch(v: &Value, scale: (f64, f64)) -> Result<String, String> {
     let acts = v["actions"].as_array().ok_or("computer_use_batch 要給 actions：動作陣列，每個有 action（click、typing、press_key、scroll、mouse_move、drag）和它的參數")?;
     if acts.is_empty() {
@@ -903,7 +982,19 @@ pub fn batch(v: &Value, scale: (f64, f64)) -> Result<String, String> {
     for (i, a) in acts.iter().enumerate() {
         let name = action_tool(a["action"].as_str().unwrap_or(""))?;
         match act(name, a, scale) {
-            Ok(n) => notes.push(format!("{}. {n}", i + 1)),
+            Ok(n) => {
+                notes.push(format!("{}. {n}", i + 1));
+                // The step promised to make `want` show: check the (cheap) control list, and
+                // stop at the step that missed — the rest of the plan is built on sand.
+                if let Some(want) = a["expect"].as_str().map(str::trim).filter(|s| !s.is_empty()) {
+                    let w = want.to_lowercase();
+                    let hit = screen_words().iter().take(800).any(|x| x.contains(&w));
+                    if !hit {
+                        notes.push(format!("（第 {} 步之後畫面裡沒有「{want}」，後面 {} 步沒有做）", i + 1, acts.len() - i - 1));
+                        return Ok(format!("做到第 {} 步不如預期：\n{}", i + 1, notes.join("\n")));
+                    }
+                }
+            }
             Err(e) => {
                 let e = plain_words(e);
                 if notes.is_empty() {
@@ -1228,6 +1319,9 @@ fn run_inner(name: &str, v: &Value) -> Result<(String, Option<String>), String>
     if !cfg!(any(target_os = "macos", target_os = "linux")) {
         return Err("操作畫面只支援 macOS 和 Linux".into());
     }
+    if name == "computer_use_wait_for" {
+        return Ok((wait_for_tool(v)?, None));
+    }
     #[cfg(target_os = "linux")]
     if !crate::desk::available() {
         return Err(crate::desk::describe().into());
@@ -1264,6 +1358,8 @@ fn run_inner(name: &str, v: &Value) -> Result<(String, Option<String>), String>
         need_controls()?;
     }
     let note = match name {
+        "computer_use_menu_click" => menu_click_tool(v)?,
+        "computer_use_fill_form" => fill_form_tool(v)?,
         "computer_use_batch" => batch(v, scale_here())?,
         _ => act(name, v, scale_here()).map_err(plain_words)?,
     };