// Screen control on 你的電腦 (computer use), like Grok Bot's Computer Use helper: the bot looks at // the screen and uses the mouse and keyboard. Same tool names and arguments as the original's // local daemon (`computer_use_screenshot`, `computer_use_click` {method, x, y, button, count, // modifier_keys}, `computer_use_scroll`, `computer_use_mouse_move`, `computer_use_drag`, // `computer_use_typing` {value}, `computer_use_press_key` {key, hold_duration_ms}). // // Screenshot sizing (local-exec-daemon main.cjs): `zct` fits the long edge to 1024 (Gct) keeping // both edges at least 8 (Hs); images stay under 1 MiB (mg = 1024*1024); JPEG is tried at quality // 85, 70, 55 ($3r), taking the first that fits, else the smallest. use base64::Engine; use serde_json::{json, Value}; use std::sync::Mutex; pub const NAMES: [&str; 9] = ["computer_use_screenshot", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_mouse_button", "computer_use_check_permissions"]; /// The person took control back from the bot (the floating bar): screen actions wait. pub static HELD_BY_PERSON: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); /// When the bot last moved the mouse or typed on this Mac (for the floating bar), and who. pub static LAST_ACTION: Mutex> = Mutex::new(None); /// Longest screenshot edge sent to the model (Gct). pub const MAX_LONG_EDGE: u32 = 1024; /// Shortest edge never goes below this (Hs). pub const MIN_EDGE: u32 = 8; /// Largest screenshot sent (mg = 1024*1024). pub const MAX_IMAGE_BYTES: usize = 1048576; /// JPEG qualities tried in order ($3r). pub const JPEG_QUALITIES: [u8; 3] = [85, 70, 55]; /// Screenshots kept in what's sent to the model; older ones become a note (estimated). pub const KEEP_SCREENSHOTS: usize = 3; /// Most UTF-16 units per typed chunk; characters are never split (CUTypedCharacters.utf16Chunks /// @0x1000beb00: capacity 0x14, a new chunk when the next character would pass 20). #[cfg_attr(not(target_os = "macos"), allow(dead_code))] pub const TYPE_CHUNK_UNITS: usize = 20; /// Virtual key the typed text rides on (postChunks @0x1000beebc: CGEventCreateKeyboardEvent(_, 0x80, _)). #[cfg_attr(not(target_os = "macos"), allow(dead_code))] const TYPE_KEY_CODE: u16 = 128; /// Key down → key up for typed chunks and pressed keys (postChunks / handlePressKey: sleep 0.035 s). pub const KEY_HOLD_SECS: f64 = 0.035; /// Between typed chunks and between keys of one press (postChunks / handlePressKey: sleep 0.02 s). pub const KEY_GAP_SECS: f64 = 0.02; /// Mouse down → up when no hold is given (CUActuator.click @0x100010120: hold > 0 ? hold : 0.03). pub const CLICK_HOLD_SECS: f64 = 0.03; /// Between the clicks of a double/triple click (CUActuator.click: sleep 0.08 s). pub const CLICK_REPEAT_GAP_SECS: f64 = 0.08; /// Interpolated dragged events per path segment (handleDrag @0x10009dee4 passes steps: 6). pub const DRAG_STEPS: usize = 6; /// Hold after pressing, before moving (handleDrag @0x10009dee4 passes holdSeconds: 0.04). pub const DRAG_HOLD_SECS: f64 = 0.04; /// The dragged-event points: each segment split into `DRAG_STEPS` equal parts, ending on its end /// point (CURemoteEventPoster.drag @0x10008c34c: start + delta * i / steps, i = 1…steps). pub fn drag_points(path: &[(f64, f64)]) -> Vec<(f64, f64)> { path.windows(2) .flat_map(|w| { let ((ax, ay), (bx, by)) = (w[0], w[1]); (1..=DRAG_STEPS).map(move |i| { let t = i as f64 / DRAG_STEPS as f64; (ax + (bx - ax) * t, ay + (by - ay) * t) }) }) .collect() } /// Scroll steps when none is given, and the allowed range (handleScroll @0x10009e200: default 3, /// max(amount, 1), min(…, 20)). pub const SCROLL_DEFAULT_AMOUNT: i64 = 3; pub const SCROLL_MAX_AMOUNT: i64 = 20; /// Pixels per scroll step (handleScroll @0x10009e200: (amount + amount) << 3). pub const SCROLL_PIXELS_PER_STEP: i32 = 16; pub fn scroll_amount(amount: Option) -> i32 { amount.unwrap_or(SCROLL_DEFAULT_AMOUNT).clamp(1, SCROLL_MAX_AMOUNT) as i32 } /// (vertical, horizontal) pixel deltas: up and left positive, down and right negative. pub fn scroll_pixels(direction: &str, amount: i32) -> Option<(i32, i32)> { let px = amount * SCROLL_PIXELS_PER_STEP; match direction { "up" => Some((px, 0)), "down" => Some((-px, 0)), "left" => Some((0, px)), "right" => Some((0, -px)), _ => None, } } /// Pause between mouse move / drag steps (estimated). #[cfg_attr(not(target_os = "macos"), allow(dead_code))] const EVENT_GAP_MS: u64 = 8; /// Longest key hold allowed (estimated). const MAX_HOLD_MS: u64 = 5000; /// utf16Chunks: split text into chunks of at most `TYPE_CHUNK_UNITS` UTF-16 units without /// splitting a character (a character longer than that gets its own chunk). #[cfg_attr(not(target_os = "macos"), allow(dead_code))] pub fn utf16_chunks(text: &str) -> Vec> { let mut out: Vec> = vec![]; let mut cur: Vec = Vec::with_capacity(TYPE_CHUNK_UNITS); for c in text.chars() { let mut buf = [0u16; 2]; let units = c.encode_utf16(&mut buf); if !cur.is_empty() && cur.len() + units.len() > TYPE_CHUNK_UNITS { out.push(std::mem::take(&mut cur)); } cur.extend_from_slice(units); } if !cur.is_empty() { out.push(cur); } out } /// "cmd+a delete" → each key of one press, in order (handlePressKey presses a list of keys). pub fn parse_keys(keys: &str) -> Result, String> { let list: Vec<&str> = keys.split_whitespace().collect(); if list.is_empty() { return Err("請給要按的鍵".into()); } list.into_iter().map(parse_combo).collect() } /// Screen points per screenshot pixel, from the last screenshot (x, y). static SCALE: Mutex> = Mutex::new(None); pub fn is_computer_tool(name: &str) -> bool { NAMES.contains(&name) } pub fn label(name: &str) -> &'static str { match name { "computer_use_screenshot" => "看你的螢幕", "computer_use_click" => "點一下", "computer_use_typing" => "打字", "computer_use_press_key" => "按鍵", "computer_use_scroll" => "捲動", "computer_use_mouse_move" => "移動滑鼠", "computer_use_drag" => "拖曳", "computer_use_mouse_button" => "按住或放開滑鼠", "computer_use_check_permissions" => "檢查螢幕權限", _ => "操作畫面", } } fn args_of(args: &str) -> Value { serde_json::from_str(if args.trim().is_empty() { "{}" } else { args }).unwrap_or(Value::Null) } pub fn describe(name: &str, args: &str) -> String { let v = args_of(args); let xy = || format!("({}, {})", v["x"].as_f64().unwrap_or(0.0), v["y"].as_f64().unwrap_or(0.0)); match name { "computer_use_screenshot" => "螢幕截圖".into(), "computer_use_click" => { let b = v["button"].as_str().unwrap_or("left"); let n = v["count"].as_u64().unwrap_or(1); format!("{}{}{} {}", if b == "right" { "右鍵" } else { "" }, if n >= 2 { "連點" } else { "點" }, if n >= 2 { format!(" {n} 下") } else { String::new() }, xy()) } "computer_use_typing" => { let t = v["value"].as_str().unwrap_or(""); let short: String = t.chars().take(60).collect(); format!("「{short}{}」", if t.chars().count() > 60 { "…" } else { "" }) } "computer_use_press_key" => v["key"].as_str().unwrap_or("").to_string(), "computer_use_scroll" => format!("{} {} 格,在 {}", v["direction"].as_str().unwrap_or("down"), v["amount"].as_i64().unwrap_or(3), xy()), "computer_use_mouse_move" => xy(), "computer_use_drag" => format!("{} 個點", v["path"].as_array().map(|a| a.len()).unwrap_or(0)), "computer_use_mouse_button" => format!("{}{}", if v["action"].as_str() == Some("release") { "放開" } else { "按住" }, if v["x"].is_null() { String::new() } else { format!(" {}", xy()) }), "computer_use_check_permissions" => "螢幕錄製、輔助使用".into(), _ => String::new(), } } #[allow(dead_code)] pub fn specs() -> Vec { specs_for(&[crate::model::THIS_COMPUTER_NAME.to_string()], crate::model::THIS_COMPUTER_NAME) } /// The screen tools, with a `computer` choice when the bot has its own desktop too. pub fn specs_for(computers: &[String], default: &str) -> Vec { let mods = json!({"type": "array", "items": {"type": "string", "enum": ["cmd", "shift", "option", "control"]}}); let mut v = vec![ json!({"name": "computer_use_screenshot", "description": "Take a screenshot of a computer's screen to see what's on it. Coordinates for the other computer_use tools are pixels of the latest screenshot of that computer. Take one before clicking, and again to check the result. Needs a model that can see images.", "parameters": {"type": "object", "properties": {}}}), json!({"name": "computer_use_click", "description": "Click on the screen at x, y (pixels of the latest screenshot). background=true clicks a window on the person's Mac without bringing it to the front.", "parameters": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "count": {"type": "integer", "description": "2 for a double click"}, "modifier_keys": mods, "background": {"type": "boolean"}}, "required": ["x", "y"]}}), json!({"name": "computer_use_typing", "description": "Type text where the cursor is (any language).", "parameters": {"type": "object", "properties": {"value": {"type": "string"}}, "required": ["value"]}}), json!({"name": "computer_use_press_key", "description": "Press a key or shortcut, e.g. return, tab, escape, delete, up, cmd+c, cmd+shift+t, cmd+space. Several keys in order: separate with spaces, e.g. \"cmd+a delete\". On the bot's Linux computer cmd means ctrl.", "parameters": {"type": "object", "properties": {"key": {"type": "string"}, "hold_duration_ms": {"type": "integer"}}, "required": ["key"]}}), json!({"name": "computer_use_scroll", "description": "Scroll at x, y (give both or neither; neither = where the pointer is).", "parameters": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}, "direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "amount": {"type": "integer", "description": "Scroll steps of 16 pixels, 1-20, default 3"}, "modifier_keys": mods}, "required": ["direction"]}}), json!({"name": "computer_use_mouse_move", "description": "Move the mouse to x, y (e.g. to show a hover menu).", "parameters": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}), json!({"name": "computer_use_drag", "description": "Drag with the mouse along a path of points (first = press, last = release).", "parameters": {"type": "object", "properties": {"path": {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "button": {"type": "string", "enum": ["left", "right"]}, "modifier_keys": mods}, "required": ["path"]}}), json!({"name": "computer_use_mouse_button", "description": "Press and hold, or release, a mouse button (optionally moving to x, y first). For long presses and custom drags.", "parameters": {"type": "object", "properties": {"action": {"type": "string", "enum": ["press", "release"]}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "x": {"type": "number"}, "y": {"type": "number"}}, "required": ["action"]}}), json!({"name": "computer_use_check_permissions", "description": "Check whether the screen can be seen and the mouse and keyboard used on that computer.", "parameters": {"type": "object", "properties": {}}}), ]; if computers.len() > 1 { for t in v.iter_mut() { t["parameters"]["properties"]["computer"] = json!({"type": "string", "enum": computers, "description": format!("Whose screen. Default: {default}. {} is the bot's own Linux desktop; {} is the person's Mac (use it for the person's own apps).", crate::model::BOT_COMPUTER_NAME, crate::model::THIS_COMPUTER_NAME)}); } } v } /// zct: the size to send. Long edge down to `MAX_LONG_EDGE`, never an edge under `MIN_EDGE`. pub fn fit(w: u32, h: u32) -> (u32, u32, bool) { if w < 1 || h < 1 { return (w, h, false); } let long = w.max(h); if long <= MAX_LONG_EDGE { return (w, h, false); } let short = w.min(h); if short < MIN_EDGE { return (w, h, false); } let mut i = MAX_LONG_EDGE as f64 / long as f64; let mut o = ((w as f64 * i).round() as u32).max(1); let mut a = ((h as f64 * i).round() as u32).max(1); if o < MIN_EDGE || a < MIN_EDGE { i = MIN_EDGE as f64 / short as f64; o = ((w as f64 * i).round() as u32).max(MIN_EDGE); a = ((h as f64 * i).round() as u32).max(MIN_EDGE); } (o, a, o != w || a != h) } /// j3r: JPEG at 85, 70, 55: the first within the budget, else the smallest if it's under the cap. pub fn encode_jpeg(img: &image::RgbImage) -> Option> { let mut best: Option> = None; for q in JPEG_QUALITIES { let mut out = vec![]; let enc = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, q); if img.write_with_encoder(enc).is_err() { continue; } if out.len() <= MAX_IMAGE_BYTES { return Some(out); } if best.as_ref().map_or(true, |b| out.len() < b.len()) { best = Some(out); } } best.filter(|b| b.len() <= MAX_IMAGE_BYTES) } /// Screenshot pixel → screen point. pub fn to_screen(x: f64, y: f64, scale: (f64, f64)) -> (f64, f64) { (x * scale.0, y * scale.1) } /// Key name → (key code, needs shift). US layout codes. pub fn key_code(name: &str) -> Option { let n = name.trim().to_lowercase(); let named = match n.as_str() { "return" | "enter" => 36, "tab" => 48, "space" => 49, "delete" | "backspace" => 51, "escape" | "esc" => 53, "forwarddelete" | "forward_delete" | "del" => 117, "home" => 115, "end" => 119, "pageup" | "page_up" => 116, "pagedown" | "page_down" => 121, "left" | "arrowleft" => 123, "right" | "arrowright" => 124, "down" | "arrowdown" => 125, "up" | "arrowup" => 126, "f1" => 122, "f2" => 120, "f3" => 99, "f4" => 118, "f5" => 96, "f6" => 97, "f7" => 98, "f8" => 100, "f9" => 101, "f10" => 109, "f11" => 103, "f12" => 111, _ => u16::MAX, }; if named != u16::MAX { return Some(named); } let mut cs = n.chars(); let (Some(c), None) = (cs.next(), cs.next()) else { return None }; const LETTERS: [u16; 26] = [0, 11, 8, 2, 14, 3, 5, 4, 34, 38, 40, 37, 46, 45, 31, 35, 12, 15, 1, 17, 32, 9, 13, 7, 16, 6]; const DIGITS: [u16; 10] = [29, 18, 19, 20, 21, 23, 22, 26, 28, 25]; match c { 'a'..='z' => Some(LETTERS[(c as u8 - b'a') as usize]), '0'..='9' => Some(DIGITS[(c as u8 - b'0') as usize]), '-' => Some(27), '=' => Some(24), '[' => Some(33), ']' => Some(30), ';' => Some(41), '\'' => Some(39), ',' => Some(43), '.' => Some(47), '/' => Some(44), '\\' => Some(42), '`' => Some(50), _ => None, } } pub const FLAG_SHIFT: u64 = 1 << 17; pub const FLAG_CONTROL: u64 = 1 << 18; pub const FLAG_OPTION: u64 = 1 << 19; pub const FLAG_CMD: u64 = 1 << 20; pub fn modifier_flag(m: &str) -> Option { match m.trim().to_lowercase().as_str() { "cmd" | "command" | "meta" | "super" | "⌘" => Some(FLAG_CMD), "shift" => Some(FLAG_SHIFT), "option" | "alt" | "opt" => Some(FLAG_OPTION), "control" | "ctrl" => Some(FLAG_CONTROL), _ => None, } } /// "cmd+shift+t" → (key code, flags). pub fn parse_combo(combo: &str) -> Result<(u16, u64), String> { let parts: Vec<&str> = combo.split('+').map(str::trim).filter(|p| !p.is_empty()).collect(); let Some((key, mods)) = parts.split_last() else { return Err("請給要按的鍵".into()) }; let mut flags = 0; for m in mods { flags |= modifier_flag(m).ok_or_else(|| format!("不認得「{m}」這個修飾鍵"))?; } let code = key_code(key).ok_or_else(|| format!("不認得「{key}」這個鍵"))?; Ok((code, flags)) } fn flags_of(v: &Value) -> u64 { v["modifier_keys"].as_array().map(|a| a.iter().filter_map(|m| m.as_str().and_then(modifier_flag)).fold(0, |f, x| f | x)).unwrap_or(0) } /// Run a computer_use tool. Returns (text, ok, screenshot as base64 JPEG). #[allow(dead_code)] pub fn run(name: &str, args: &str) -> (String, bool, Option) { run_by(name, args, "") } /// Same, remembering which bot acted (for the 「正在操作你的電腦」 bar). pub fn run_by(name: &str, args: &str, who: &str) -> (String, bool, Option) { if name != "computer_use_screenshot" && name != "computer_use_check_permissions" { *LAST_ACTION.lock().unwrap() = Some((std::time::Instant::now(), who.to_string())); } let r = run_inner(name, &args_of(args)); match r { Ok((t, img)) => (t, true, img), Err(e) => (e, false, None), } } fn run_inner(name: &str, v: &Value) -> Result<(String, Option), String> { if !cfg!(any(target_os = "macos", target_os = "linux")) { return Err("操作畫面只支援 macOS 和 Linux".into()); } #[cfg(target_os = "linux")] if !crate::desk::available() { return Err(crate::desk::describe().into()); } if name == "computer_use_screenshot" { return screenshot().map(|(t, img)| (t, Some(img))); } #[cfg(target_os = "linux")] if name == "computer_use_check_permissions" { return Ok((crate::desk::describe().into(), None)); } if name == "computer_use_check_permissions" { let see = mac::screen_capture_preflight(); let control = crate::axread::trusted(false); return Ok((format!("螢幕錄製:{}\n輔助使用(滑鼠和鍵盤):{}{}", if see { "已允許" } else { "還沒允許" }, if control { "已允許" } else { "還沒允許" }, if see && control { "" } else { "\n請使用者到「系統設定 → 隱私權與安全性」打開 AnyBot(或「終端機」)。" }), None)); } if HELD_BY_PERSON.load(std::sync::atomic::Ordering::SeqCst) { return Err("使用者正在自己操作這台電腦(已取得控制權)。請等他交還控制權後再操作;可以先截圖看看。".into()); } if cfg!(target_os = "macos") && !crate::axread::trusted(true) { let _ = std::process::Command::new("open").arg("x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility").spawn(); return Err("macOS 還沒讓 AnyBot 使用「輔助使用」,所以不能動滑鼠和鍵盤。我已經打開「系統設定 → 隱私權與安全性 → 輔助使用」:把 AnyBot(或「終端機」)打開,再叫我試一次。只要開一次。".into()); } let scale = || SCALE.lock().unwrap().ok_or_else(|| "請先用 computer_use_screenshot 看一下畫面,座標要用截圖上的位置。".to_string()); let pt = |v: &Value, s: (f64, f64)| -> Result<(f64, f64), String> { let (Some(x), Some(y)) = (v["x"].as_f64(), v["y"].as_f64()) else { return Err("請給 x 和 y".into()) }; Ok(to_screen(x, y, s)) }; match name { "computer_use_click" => { let (x, y) = pt(v, scale()?)?; let button = v["button"].as_str().unwrap_or("left"); let count = v["count"].as_u64().unwrap_or(1).clamp(1, 3) as i64; if v["background"].as_bool().unwrap_or(false) { let (app, pid) = mac::window_at(x, y).ok_or("那個位置沒有視窗")?; mac::click_in_background(pid, x, y, button, count, flags_of(v)); return Ok((format!("已在背景點「{app}」的 ({}, {}),沒有把它拉到前面。要確認結果可以再截圖。", v["x"], v["y"]), None)); } mac::click(x, y, button, count, flags_of(v)); Ok((format!("已{}。要確認結果可以再截圖。", describe(name, &v.to_string())), None)) } "computer_use_mouse_move" => { let (x, y) = pt(v, scale()?)?; mac::move_to(x, y); Ok(("已移動滑鼠。".into(), None)) } "computer_use_scroll" => { // CURemoteToolService.handleScroll @0x10009e200 / closure @0x10009e998. let (has_x, has_y) = (!v["x"].is_null(), !v["y"].is_null()); if has_x != has_y { return Err("x 和 y 要一起給".into()); } let at = if has_x { Some(pt(v, scale()?)?) } else { None }; if !v["amount"].is_null() && v["amount"].as_i64().is_none() { return Err("amount 要是整數".into()); } let amount = scroll_amount(v["amount"].as_i64()); let dir = v["direction"].as_str().unwrap_or("").trim().to_lowercase(); let (dy, dx) = scroll_pixels(&dir, amount).ok_or_else(|| format!("方向「{dir}」要是 up、down、left 或 right"))?; mac::scroll(at, dy, dx, flags_of(v)); Ok((format!("已捲動 {dir} ×{amount}。"), None)) } "computer_use_drag" => { let s = scale()?; let path: Vec<(f64, f64)> = v["path"].as_array().map(|a| a.iter().filter_map(|p| pt(p, s).ok()).collect()).unwrap_or_default(); if path.len() < 2 { return Err("拖曳至少要兩個點(每個點要有數字 x 和 y,用截圖上的座標)".into()); } let button = v["button"].as_str().unwrap_or("left"); mac::drag(&path, button, flags_of(v)); // Like the original's "Dragged from (x, y) to (x, y) with left button." let pts = v["path"].as_array().cloned().unwrap_or_default(); let at = |p: &Value| format!("({:.0}, {:.0})", p["x"].as_f64().unwrap_or(0.0), p["y"].as_f64().unwrap_or(0.0)); let side = match button { "right" => "右鍵", "middle" => "中鍵", _ => "左鍵", }; Ok((format!("已從 {} 拖曳到 {}({side})。", pts.first().map(at).unwrap_or_default(), pts.last().map(at).unwrap_or_default()), None)) } "computer_use_mouse_button" => { let button = v["button"].as_str().unwrap_or("left"); let at = if v["x"].is_null() { None } else { Some(pt(v, scale()?)?) }; let release = v["action"].as_str() == Some("release"); mac::mouse_button(at, button, !release); Ok((if release { "已放開滑鼠鍵。" } else { "已按住滑鼠鍵(記得之後放開)。" }.into(), None)) } "computer_use_typing" => { let text = v["value"].as_str().unwrap_or(""); if text.is_empty() { return Err("請給要打的字".into()); } mac::type_text(text); Ok((format!("已打字({} 個字)。", text.chars().count()), None)) } "computer_use_press_key" => { let keys = parse_keys(v["key"].as_str().unwrap_or(""))?; let hold = v["hold_duration_ms"].as_u64().map(|ms| ms.min(MAX_HOLD_MS) as f64 / 1000.0).filter(|s| *s > 0.0).unwrap_or(KEY_HOLD_SECS); for (i, (code, flags)) in keys.iter().enumerate() { if i > 0 { std::thread::sleep(std::time::Duration::from_secs_f64(KEY_GAP_SECS)); } mac::press(*code, *flags, hold); } Ok((format!("已按 {}。", v["key"].as_str().unwrap_or("")), None)) } other => Err(format!("沒有「{other}」這個工具")), } } /// The app a screen action will land in: the window under the point for clicks, else the /// frontmost app. Used for the per-app permission (「AnyBot 可以看到 {app},並在其中點擊和輸入」). pub fn target_app(name: &str, args: &str) -> Option { let v = args_of(args); let s = (*SCALE.lock().unwrap())?; let point = match name { "computer_use_click" | "computer_use_mouse_move" | "computer_use_scroll" | "computer_use_mouse_button" if !v["x"].is_null() => Some(to_screen(v["x"].as_f64()?, v["y"].as_f64()?, s)), "computer_use_drag" => v["path"].as_array().and_then(|a| a.first()).and_then(|p| Some(to_screen(p["x"].as_f64()?, p["y"].as_f64()?, s))), _ => None, }; match point { Some((x, y)) => mac::window_at(x, y).map(|w| w.0), None => mac::front_app().map(|w| w.0), } } /// Screenshot pixels per screen point (for the demonstration recorder). pub fn points_to_shot() -> f64 { let (pw, _) = mac::screen_points(); let (px, py) = mac::screen_pixels(); let (fw, _, _) = fit(px.max(1), py.max(1)); if pw > 0.0 { fw as f64 / pw } else { 1.0 } } /// The person's own clicks and keys can be recorded (教它一項任務): macOS, Linux on X11. /// Wayland doesn't let apps watch other apps' input. pub fn demo_supported() -> bool { #[cfg(target_os = "linux")] { crate::desk::kind() == crate::desk::Kind::X11 } #[cfg(not(target_os = "linux"))] { cfg!(target_os = "macos") } } /// This system's screen can be seen and used (macOS; Linux on X11 or Wayland). pub fn screen_supported() -> bool { #[cfg(target_os = "linux")] { crate::desk::available() } #[cfg(not(target_os = "linux"))] { cfg!(target_os = "macos") } } pub fn screen_capture_ok() -> bool { mac::screen_capture_preflight() } #[cfg_attr(not(target_os = "macos"), allow(dead_code))] pub fn frontmost_app() -> Option { mac::front_app().map(|w| w.0) } #[cfg_attr(not(target_os = "macos"), allow(dead_code))] pub fn app_at_point(x: f64, y: f64) -> Option { mac::window_at(x, y).map(|w| w.0) } fn screenshot() -> Result<(String, String), String> { if !mac::screen_capture_allowed() { let _ = std::process::Command::new("open").arg("x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture").spawn(); return Err("macOS 還沒讓 AnyBot 錄製螢幕,所以看不到畫面。我已經打開「系統設定 → 隱私權與安全性 → 螢幕與系統錄音」:把 AnyBot(或「終端機」)打開,再叫我試一次。只要開一次。".into()); } let img = capture_screen()?; let (w, h) = img.dimensions(); let (fw, fh, resize) = fit(w, h); let img = if resize { image::imageops::resize(&img, fw, fh, image::imageops::FilterType::Triangle) } else { img }; let jpeg = encode_jpeg(&img).ok_or("截圖太大了,沒辦法送出")?; let (pw, ph) = mac::screen_points(); let scale = (if fw > 0 { pw / fw as f64 } else { 1.0 }, if fh > 0 { ph / fh as f64 } else { 1.0 }); *SCALE.lock().unwrap() = Some(scale); let b64 = base64::engine::general_purpose::STANDARD.encode(&jpeg); Ok((format!("你的螢幕截圖({fw}×{fh})。之後的座標請用這張圖上的像素位置。"), b64)) } /// The whole screen as RGB (backing pixels). pub fn capture_screen() -> Result { #[cfg(target_os = "linux")] return crate::desk::capture(); #[allow(unreachable_code)] { let file = std::env::temp_dir().join(format!("anybot-screen-{}-{}.png", std::process::id(), crate::model::now_ms())); let st = std::process::Command::new("screencapture").args(["-x", "-m", "-t", "png"]).arg(&file).status().map_err(|e| e.to_string())?; let bytes = std::fs::read(&file); let _ = std::fs::remove_file(&file); if !st.success() { return Err("截圖失敗".into()); } Ok(image::load_from_memory(&bytes.map_err(|e| format!("截圖失敗:{e}"))?).map_err(|e| format!("截圖讀不出來:{e}"))?.to_rgb8()) } } #[cfg(target_os = "linux")] use crate::desk as mac; #[cfg(target_os = "macos")] mod mac { use super::*; use std::ffi::c_void; use std::time::Duration; #[repr(C)] #[derive(Clone, Copy)] struct CGPoint { x: f64, y: f64, } #[repr(C)] #[derive(Clone, Copy)] struct CGSize { width: f64, height: f64, } #[repr(C)] #[derive(Clone, Copy)] struct CGRect { origin: CGPoint, size: CGSize, } type Ev = *mut c_void; type Src = *mut c_void; #[link(name = "CoreGraphics", kind = "framework")] extern "C" { fn CGMainDisplayID() -> u32; fn CGDisplayBounds(d: u32) -> CGRect; fn CGPreflightScreenCaptureAccess() -> bool; fn CGRequestScreenCaptureAccess() -> bool; fn CGEventSourceCreate(state: i32) -> Src; fn CGEventSourceSetLocalEventsSuppressionInterval(src: Src, secs: f64); fn CGEventCreateMouseEvent(src: Src, kind: u32, at: CGPoint, button: u32) -> Ev; fn CGEventCreateKeyboardEvent(src: Src, key: u16, down: bool) -> Ev; fn CGEventKeyboardSetUnicodeString(e: Ev, len: usize, s: *const u16); fn CGEventSetFlags(e: Ev, flags: u64); fn CGEventSetIntegerValueField(e: Ev, field: u32, v: i64); fn CGEventPost(tap: u32, e: Ev); fn CGEventCreateScrollWheelEvent(src: Src, units: u32, count: u32, wheel1: i32, ...) -> Ev; fn CGEventPostToPid(pid: i32, e: Ev); fn CGDisplayPixelsWide(d: u32) -> usize; fn CGDisplayPixelsHigh(d: u32) -> usize; fn CGWindowListCopyWindowInfo(option: u32, relative_to: u32) -> *const c_void; fn CGRectMakeWithDictionaryRepresentation(dict: *const c_void, rect: *mut CGRect) -> bool; static kCGWindowOwnerName: *const c_void; static kCGWindowOwnerPID: *const c_void; static kCGWindowBounds: *const c_void; static kCGWindowLayer: *const c_void; } #[link(name = "CoreFoundation", kind = "framework")] extern "C" { fn CFArrayGetCount(a: *const c_void) -> isize; fn CFArrayGetValueAtIndex(a: *const c_void, i: isize) -> *const c_void; fn CFDictionaryGetValue(d: *const c_void, k: *const c_void) -> *const c_void; fn CFNumberGetValue(n: *const c_void, kind: isize, out: *mut c_void) -> bool; fn CFStringGetCString(s: *const c_void, buf: *mut std::ffi::c_char, len: isize, enc: u32) -> bool; } /// On-screen windows front to back: (owner app, pid, bounds), normal layer only. fn windows() -> Vec<(String, i32, CGRect)> { let mut out = vec![]; unsafe { // kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements let list = CGWindowListCopyWindowInfo(1 | 16, 0); if list.is_null() { return out; } for i in 0..CFArrayGetCount(list) { let d = CFArrayGetValueAtIndex(list, i); let mut layer: i32 = -1; let l = CFDictionaryGetValue(d, kCGWindowLayer); if l.is_null() || !CFNumberGetValue(l, 3, &mut layer as *mut i32 as *mut c_void) || layer != 0 { continue; } let mut pid: i32 = 0; let p = CFDictionaryGetValue(d, kCGWindowOwnerPID); if !p.is_null() { CFNumberGetValue(p, 3, &mut pid as *mut i32 as *mut c_void); } let mut name = String::new(); let n = CFDictionaryGetValue(d, kCGWindowOwnerName); if !n.is_null() { let mut buf = [0u8; 256]; if CFStringGetCString(n, buf.as_mut_ptr() as *mut std::ffi::c_char, buf.len() as isize, 0x0800_0100) { let end = buf.iter().position(|b| *b == 0).unwrap_or(buf.len()); name = String::from_utf8_lossy(&buf[..end]).to_string(); } } let mut r = CGRect { origin: CGPoint { x: 0.0, y: 0.0 }, size: CGSize { width: 0.0, height: 0.0 } }; let b = CFDictionaryGetValue(d, kCGWindowBounds); if !b.is_null() { CGRectMakeWithDictionaryRepresentation(b, &mut r); } out.push((name, pid, r)); } CFRelease(list); } out } /// The app whose window is under a screen point. pub fn window_at(x: f64, y: f64) -> Option<(String, i32)> { windows().into_iter().find(|(_, _, r)| x >= r.origin.x && y >= r.origin.y && x < r.origin.x + r.size.width && y < r.origin.y + r.size.height).map(|(n, p, _)| (n, p)) } /// The frontmost app (its frontmost normal window). pub fn front_app() -> Option<(String, i32)> { windows().into_iter().find(|(n, _, r)| !n.is_empty() && r.size.width > 1.0).map(|(n, p, _)| (n, p)) } pub fn screen_capture_preflight() -> bool { unsafe { CGPreflightScreenCaptureAccess() } } pub fn screen_pixels() -> (u32, u32) { let d = unsafe { CGMainDisplayID() }; let (pw, ph) = screen_points(); // CGDisplayPixelsWide reports points on Retina; screencapture gives backing pixels. let (w, h) = unsafe { (CGDisplayPixelsWide(d) as f64, CGDisplayPixelsHigh(d) as f64) }; let scale = backing_scale(); let _ = (w, h); ((pw * scale) as u32, (ph * scale) as u32) } fn backing_scale() -> f64 { static S: std::sync::OnceLock = std::sync::OnceLock::new(); *S.get_or_init(|| { let file = std::env::temp_dir().join(format!("anybot-scale-{}.png", std::process::id())); let ok = std::process::Command::new("screencapture").args(["-x", "-m", "-t", "png", "-R0,0,10,10"]).arg(&file).status().map(|s| s.success()).unwrap_or(false); let w = if ok { image::image_dimensions(&file).map(|d| d.0).unwrap_or(10) } else { 10 }; let _ = std::fs::remove_file(&file); (w as f64 / 10.0).max(1.0) }) } /// A click delivered straight to an app without moving the pointer or raising its window /// (the original's "skylight-no-raise" path does this with a private API). pub fn click_in_background(pid: i32, x: f64, y: f64, button: &str, count: i64, flags: u64) { let (down, up, _, b) = kinds(button); let src = Source::new(); for n in 1..=count { if n > 1 { sleep_secs(CLICK_REPEAT_GAP_SECS); } for (i, kind) in [down, up].into_iter().enumerate() { if i == 1 { sleep_secs(CLICK_HOLD_SECS); } let e = unsafe { CGEventCreateMouseEvent(src.0, kind, CGPoint { x, y }, b) }; if e.is_null() { continue; } unsafe { CGEventSetIntegerValueField(e, CLICK_STATE, n); CGEventSetFlags(e, flags); CGEventPostToPid(pid, e); CFRelease(e); } } } } /// Press or release a mouse button where the pointer is (or at a point). pub fn mouse_button(at: Option<(f64, f64)>, button: &str, press: bool) { let (down, up, _, b) = kinds(button); let p = match at { Some((x, y)) => { move_to(x, y); gap(); CGPoint { x, y } } None => current_point(), }; let src = Source::new(); let e = unsafe { CGEventCreateMouseEvent(src.0, if press { down } else { up }, p, b) }; if !e.is_null() { unsafe { CGEventSetIntegerValueField(e, CLICK_STATE, 1) }; } post(e, 0); } fn current_point() -> CGPoint { extern "C" { fn CGEventCreate(src: Src) -> Ev; fn CGEventGetLocation(e: Ev) -> CGPoint; } unsafe { let e = CGEventCreate(std::ptr::null_mut()); if e.is_null() { return CGPoint { x: 0.0, y: 0.0 }; } let p = CGEventGetLocation(e); CFRelease(e); p } } #[link(name = "CoreFoundation", kind = "framework")] extern "C" { fn CFRelease(x: *const c_void); } const HID_TAP: u32 = 0; /// kCGEventSourceStateHIDSystemState, as CUActuator uses (CGEventSourceCreate(1)). const HID_STATE: i32 = 1; const CLICK_STATE: u32 = 1; const MOVED: u32 = 5; /// kCGScrollEventUnitPixel, as CGEventCreateScrollWheelEvent2(_, 0, 2, …) in the original. const SCROLL_PIXEL: u32 = 0; /// An event source like the original's: HID system state, local-event suppression 0 s, so /// the person's own mouse and keys aren't frozen after each synthetic event. struct Source(Src); impl Source { fn new() -> Self { let s = unsafe { CGEventSourceCreate(HID_STATE) }; if !s.is_null() { unsafe { CGEventSourceSetLocalEventsSuppressionInterval(s, 0.0) }; } Source(s) } } impl Drop for Source { fn drop(&mut self) { if !self.0.is_null() { unsafe { CFRelease(self.0 as *const c_void) } } } } fn post(e: Ev, flags: u64) { if e.is_null() { return; } unsafe { CGEventSetFlags(e, flags); CGEventPost(HID_TAP, e); CFRelease(e); } } fn gap() { std::thread::sleep(Duration::from_millis(EVENT_GAP_MS)); } fn sleep_secs(s: f64) { std::thread::sleep(Duration::from_secs_f64(s)); } /// (down, up, dragged) event kinds and button number. fn kinds(button: &str) -> (u32, u32, u32, u32) { match button { "right" => (3, 4, 7, 1), "middle" => (25, 26, 27, 2), _ => (1, 2, 6, 0), } } pub fn screen_capture_allowed() -> bool { unsafe { CGPreflightScreenCaptureAccess() || CGRequestScreenCaptureAccess() } } pub fn screen_points() -> (f64, f64) { let r = unsafe { CGDisplayBounds(CGMainDisplayID()) }; (r.size.width, r.size.height) } pub fn move_to(x: f64, y: f64) { let src = Source::new(); post(unsafe { CGEventCreateMouseEvent(src.0, MOVED, CGPoint { x, y }, 0) }, 0); } /// CUActuator.click: press, hold 0.03 s, release; 0.08 s between the clicks of a multi-click. pub fn click(x: f64, y: f64, button: &str, count: i64, flags: u64) { let (down, up, _, b) = kinds(button); let at = CGPoint { x, y }; move_to(x, y); gap(); let src = Source::new(); for n in 1..=count { if n > 1 { sleep_secs(CLICK_REPEAT_GAP_SECS); } for (i, kind) in [down, up].into_iter().enumerate() { if i == 1 { sleep_secs(CLICK_HOLD_SECS); } let e = unsafe { CGEventCreateMouseEvent(src.0, kind, at, b) }; if !e.is_null() { unsafe { CGEventSetIntegerValueField(e, CLICK_STATE, n) }; } post(e, flags); } } } /// CURemoteEventPoster.drag(path:button:steps:holdSeconds:): press at the first point (click /// state 1), hold, then walk each segment in `DRAG_STEPS` interpolated dragged events, then /// release at the last point. pub fn drag(path: &[(f64, f64)], button: &str, flags: u64) { let (down, up, dragged, b) = kinds(button); let (x0, y0) = path[0]; move_to(x0, y0); gap(); let src = Source::new(); let event = |kind: u32, x: f64, y: f64| { let e = unsafe { CGEventCreateMouseEvent(src.0, kind, CGPoint { x, y }, b) }; if !e.is_null() { unsafe { CGEventSetIntegerValueField(e, CLICK_STATE, 1) }; } post(e, flags); }; event(down, x0, y0); sleep_secs(DRAG_HOLD_SECS); for (x, y) in drag_points(path) { gap(); event(dragged, x, y); } let (xn, yn) = path[path.len() - 1]; event(up, xn, yn); } /// handleScroll closure @0x10009e998: a mouse-moved event at the spot (the pointer's own /// position when no x/y), then one pixel-unit scroll event with 2 wheels at that spot. pub fn scroll(at: Option<(f64, f64)>, dy: i32, dx: i32, flags: u64) { if let Some((x, y)) = at { move_to(x, y); gap(); } let src = Source::new(); post(unsafe { CGEventCreateScrollWheelEvent(src.0, SCROLL_PIXEL, 2, dy, dx) }, flags); } /// CUTypedCharacters.postChunks: each chunk on key 128, down, 0.035 s, up; 0.02 s between chunks. pub fn type_text(text: &str) { let src = Source::new(); let chunks = utf16_chunks(text); for (i, chunk) in chunks.iter().enumerate() { if i > 0 { sleep_secs(KEY_GAP_SECS); } for down in [true, false] { if !down { sleep_secs(KEY_HOLD_SECS); } let e = unsafe { CGEventCreateKeyboardEvent(src.0, TYPE_KEY_CODE, down) }; if !e.is_null() { unsafe { CGEventKeyboardSetUnicodeString(e, chunk.len(), chunk.as_ptr()) }; } post(e, 0); } } } /// handlePressKey: key down, hold (0.035 s unless told), key up. pub fn press(code: u16, flags: u64, hold_secs: f64) { let src = Source::new(); post(unsafe { CGEventCreateKeyboardEvent(src.0, code, true) }, flags); sleep_secs(hold_secs); post(unsafe { CGEventCreateKeyboardEvent(src.0, code, false) }, flags); } } #[cfg(not(any(target_os = "macos", target_os = "linux")))] mod mac { pub fn screen_capture_allowed() -> bool { false } pub fn screen_points() -> (f64, f64) { (0.0, 0.0) } pub fn move_to(_: f64, _: f64) {} pub fn click(_: f64, _: f64, _: &str, _: i64, _: u64) {} pub fn drag(_: &[(f64, f64)], _: &str, _: u64) {} pub fn scroll(_: Option<(f64, f64)>, _: i32, _: i32, _: u64) {} pub fn type_text(_: &str) {} pub fn press(_: u16, _: u64, _: f64) {} pub fn window_at(_: f64, _: f64) -> Option<(String, i32)> { None } pub fn front_app() -> Option<(String, i32)> { None } pub fn screen_capture_preflight() -> bool { false } pub fn screen_pixels() -> (u32, u32) { (0, 0) } pub fn click_in_background(_: i32, _: f64, _: f64, _: &str, _: i64, _: u64) {} pub fn mouse_button(_: Option<(f64, f64)>, _: &str, _: bool) {} } #[cfg(test)] mod tests { use super::*; #[test] fn screenshots_fit_like_the_original() { assert_eq!(fit(2940, 1912), (1024, 666, true)); assert_eq!(fit(1024, 640), (1024, 640, false)); assert_eq!(fit(800, 600), (800, 600, false)); assert_eq!(fit(4000, 7), (4000, 7, false)); // A very wide strip keeps its short edge at 8. assert_eq!(fit(4000, 10), (3200, 8, true)); } #[test] fn jpeg_stays_under_the_cap() { let img = image::RgbImage::from_fn(1024, 640, |x, y| image::Rgb([(x % 256) as u8, (y % 256) as u8, ((x ^ y) % 256) as u8])); let j = encode_jpeg(&img).unwrap(); assert!(j.len() <= MAX_IMAGE_BYTES && j.starts_with(&[0xFF, 0xD8])); } #[test] fn shortcuts_are_parsed() { assert_eq!(parse_combo("cmd+c").unwrap(), (8, FLAG_CMD)); assert_eq!(parse_combo("cmd+shift+t").unwrap(), (17, FLAG_CMD | FLAG_SHIFT)); assert_eq!(parse_combo("Return").unwrap(), (36, 0)); assert_eq!(parse_combo("ctrl+option+space").unwrap(), (49, FLAG_CONTROL | FLAG_OPTION)); assert!(parse_combo("hyper+x").is_err()); assert!(parse_combo("").is_err()); } #[test] fn typed_text_is_chunked_like_the_original() { let c = utf16_chunks(&"a".repeat(45)); assert_eq!(c.iter().map(Vec::len).collect::>(), vec![20, 20, 5]); // An emoji (2 UTF-16 units) is never split across chunks. let c = utf16_chunks(&format!("{}😀b", "a".repeat(19))); assert_eq!(c.iter().map(Vec::len).collect::>(), vec![19, 3]); assert_eq!(utf16_chunks("你好").len(), 1); assert!(utf16_chunks("").is_empty()); } #[test] fn scrolls_like_the_original() { assert_eq!(scroll_amount(None), 3); assert_eq!(scroll_amount(Some(0)), 1); assert_eq!(scroll_amount(Some(99)), 20); assert_eq!(scroll_pixels("down", 3), Some((-48, 0))); assert_eq!(scroll_pixels("left", 2), Some((0, 32))); assert_eq!(scroll_pixels("sideways", 1), None); } #[test] fn drags_walk_six_steps_per_segment() { let pts = drag_points(&[(0.0, 0.0), (60.0, 0.0), (60.0, 30.0)]); assert_eq!(pts.len(), 2 * DRAG_STEPS); assert_eq!(pts[0], (10.0, 0.0)); assert_eq!(pts[DRAG_STEPS - 1], (60.0, 0.0)); assert_eq!(*pts.last().unwrap(), (60.0, 30.0)); } #[test] fn a_press_can_be_several_keys() { assert_eq!(parse_keys("cmd+a delete").unwrap(), vec![(0, FLAG_CMD), (51, 0)]); assert!(parse_keys(" ").is_err()); } #[test] fn screenshot_pixels_map_to_screen_points() { // A 1470-point-wide screen sent as a 1024-wide picture. let s = (1470.0 / 1024.0, 956.0 / 666.0); let (x, y) = to_screen(512.0, 333.0, s); assert!((x - 735.0).abs() < 0.01 && (y - 478.0).abs() < 0.01); } } #[cfg(test)] mod window_tests { /// `cargo test --release -- --ignored finds_apps_on_screen --nocapture` #[test] #[ignore] fn finds_apps_on_screen() { let front = super::frontmost_app(); println!("front: {front:?}"); println!("at (200,200): {:?}", super::app_at_point(200.0, 200.0)); println!("points→shot: {}", super::points_to_shot()); assert!(front.is_some()); } }