AnyBot/src/computer.rs

// Screen control on 你的電腦 (computer use), like Grok Bot's Computer Use helper: the bot looks at
// the screen and uses the mouse and keyboard. Same tool names and arguments as the original's
// local daemon (`computer_use_screenshot`, `computer_use_click` {method, x, y, button, count,
// modifier_keys}, `computer_use_scroll`, `computer_use_mouse_move`, `computer_use_drag`,
// `computer_use_typing` {value}, `computer_use_press_key` {key, hold_duration_ms}).
//
// Screenshot sizing (local-exec-daemon main.cjs): `zct` fits the long edge to 1024 (Gct) keeping
// both edges at least 8 (Hs); images stay under 1 MiB (mg = 1024*1024); JPEG is tried at quality
// 85, 70, 55 ($3r), taking the first that fits, else the smallest.

use base64::Engine;
use serde_json::{json, Value};
use std::sync::Mutex;

pub const NAMES: [&str; 11] = ["computer_use_screenshot", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_mouse_button", "computer_use_check_permissions", "computer_use_read_app", "computer_use_open_app"];
/// Wait after an action before taking the picture that comes back with it (menus open, pages
/// start to change; estimated).
pub const SETTLE_MS: u64 = 500;

/// The person took control back from the bot (the floating bar): screen actions wait.
pub static HELD_BY_PERSON: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
/// When the bot last moved the mouse or typed on this Mac (for the floating bar), and who.
pub static LAST_ACTION: Mutex<Option<(std::time::Instant, String)>> = Mutex::new(None);

/// Spawn a short-lived helper (`open`, a player, …) and reap it in the background:
/// spawn-without-wait left one zombie per call for the app's whole life.
pub fn spawn_reaped(c: &mut std::process::Command) -> std::io::Result<()> {
    let mut child = c.spawn()?;
    std::thread::spawn(move || {
        let _ = child.wait();
    });
    Ok(())
}
/// Longest screenshot edge sent to the model (Gct).
pub const MAX_LONG_EDGE: u32 = 1024;
/// Shortest edge never goes below this (Hs).
pub const MIN_EDGE: u32 = 8;
/// Largest screenshot sent (mg = 1024*1024).
pub const MAX_IMAGE_BYTES: usize = 1048576;
/// JPEG qualities tried in order ($3r).
pub const JPEG_QUALITIES: [u8; 3] = [85, 70, 55];
/// Screenshots kept in what's sent to the model; older ones become a note (estimated).
pub const KEEP_SCREENSHOTS: usize = 3;
/// Most UTF-16 units per typed chunk; characters are never split (CUTypedCharacters.utf16Chunks
/// @0x1000beb00: capacity 0x14, a new chunk when the next character would pass 20).
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
pub const TYPE_CHUNK_UNITS: usize = 20;
/// Virtual key the typed text rides on (postChunks @0x1000beebc: CGEventCreateKeyboardEvent(_, 0x80, _)).
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
const TYPE_KEY_CODE: u16 = 128;
/// Key down → key up for typed chunks and pressed keys (postChunks / handlePressKey: sleep 0.035 s).
pub const KEY_HOLD_SECS: f64 = 0.035;
/// Between typed chunks and between keys of one press (postChunks / handlePressKey: sleep 0.02 s).
pub const KEY_GAP_SECS: f64 = 0.02;
/// Mouse down → up when no hold is given (CUActuator.click @0x100010120: hold > 0 ? hold : 0.03).
pub const CLICK_HOLD_SECS: f64 = 0.03;
/// Between the clicks of a double/triple click (CUActuator.click: sleep 0.08 s).
pub const CLICK_REPEAT_GAP_SECS: f64 = 0.08;
/// Interpolated dragged events per path segment (handleDrag @0x10009dee4 passes steps: 6).
pub const DRAG_STEPS: usize = 6;
/// Hold after pressing, before moving (handleDrag @0x10009dee4 passes holdSeconds: 0.04).
pub const DRAG_HOLD_SECS: f64 = 0.04;

/// The dragged-event points: each segment split into `DRAG_STEPS` equal parts, ending on its end
/// point (CURemoteEventPoster.drag @0x10008c34c: start + delta * i / steps, i = 1…steps).
pub fn drag_points(path: &[(f64, f64)]) -> Vec<(f64, f64)> {
    path.windows(2)
        .flat_map(|w| {
            let ((ax, ay), (bx, by)) = (w[0], w[1]);
            (1..=DRAG_STEPS).map(move |i| {
                let t = i as f64 / DRAG_STEPS as f64;
                (ax + (bx - ax) * t, ay + (by - ay) * t)
            })
        })
        .collect()
}

/// Scroll steps when none is given, and the allowed range (handleScroll @0x10009e200: default 3,
/// max(amount, 1), min(…, 20)).
pub const SCROLL_DEFAULT_AMOUNT: i64 = 3;
pub const SCROLL_MAX_AMOUNT: i64 = 20;
/// Pixels per scroll step (handleScroll @0x10009e200: (amount + amount) << 3).
pub const SCROLL_PIXELS_PER_STEP: i32 = 16;

pub fn scroll_amount(amount: Option<i64>) -> i32 {
    amount.unwrap_or(SCROLL_DEFAULT_AMOUNT).clamp(1, SCROLL_MAX_AMOUNT) as i32
}

/// (vertical, horizontal) pixel deltas: up and left positive, down and right negative.
pub fn scroll_pixels(direction: &str, amount: i32) -> Option<(i32, i32)> {
    let px = amount * SCROLL_PIXELS_PER_STEP;
    match direction {
        "up" => Some((px, 0)),
        "down" => Some((-px, 0)),
        "left" => Some((0, px)),
        "right" => Some((0, -px)),
        _ => None,
    }
}

/// Pause between mouse move / drag steps (estimated).
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
const EVENT_GAP_MS: u64 = 8;
/// Longest key hold allowed (estimated).
const MAX_HOLD_MS: u64 = 5000;

/// utf16Chunks: split text into chunks of at most `TYPE_CHUNK_UNITS` UTF-16 units without
/// splitting a character (a character longer than that gets its own chunk).
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
pub fn utf16_chunks(text: &str) -> Vec<Vec<u16>> {
    let mut out: Vec<Vec<u16>> = vec![];
    let mut cur: Vec<u16> = Vec::with_capacity(TYPE_CHUNK_UNITS);
    for c in text.chars() {
        let mut buf = [0u16; 2];
        let units = c.encode_utf16(&mut buf);
        if !cur.is_empty() && cur.len() + units.len() > TYPE_CHUNK_UNITS {
            out.push(std::mem::take(&mut cur));
        }
        cur.extend_from_slice(units);
    }
    if !cur.is_empty() {
        out.push(cur);
    }
    out
}

/// One key of a press: a key with its modifiers, or a character typed (「@」「？」 have no key of
/// their own on every keyboard; refusing them sent the bot looking for workarounds).
#[derive(Debug, PartialEq)]
pub enum Key {
    Combo(u16, u64),
    Char(String),
}

/// "cmd+a delete" → each key of one press, in order (handlePressKey presses a list of keys).
pub fn key_plan(keys: &str) -> Result<Vec<Key>, String> {
    let list: Vec<&str> = keys.split_whitespace().collect();
    if list.is_empty() {
        return Err("請給要按的鍵".into());
    }
    list.into_iter()
        .map(|k| match parse_combo(k) {
            // A letter, digit or symbol without a modifier is a character: typed, so an input
            // method (注音) can't turn it into a syllable. Named symbol keys mean their character.
            Ok((c, 0)) if printable_of(c).is_some() => Ok(Key::Char(printable_of(c).unwrap_or_default())),
            Ok((c, f)) => Ok(Key::Combo(c, f)),
            Err(_) if k.chars().count() == 1 => Ok(Key::Char(k.to_string())),
            Err(e) => Err(e),
        })
        .collect()
}

/// The character a printing key gives without shift (US layout), or None for keys like return.
fn printable_of(code: u16) -> Option<String> {
    const LETTERS: [u16; 26] = [0, 11, 8, 2, 14, 3, 5, 4, 34, 38, 40, 37, 46, 45, 31, 35, 12, 15, 1, 17, 32, 9, 13, 7, 16, 6];
    const DIGITS: [u16; 10] = [29, 18, 19, 20, 21, 23, 22, 26, 28, 25];
    if let Some(i) = LETTERS.iter().position(|c| *c == code) {
        return Some(((b'a' + i as u8) as char).to_string());
    }
    if let Some(i) = DIGITS.iter().position(|c| *c == code) {
        return Some(((b'0' + i as u8) as char).to_string());
    }
    let c = match code {
        27 => '-',
        24 => '=',
        33 => '[',
        30 => ']',
        41 => ';',
        39 => '\'',
        43 => ',',
        47 => '.',
        44 => '/',
        42 => '\\',
        50 => '`',
        _ => return None,
    };
    Some(c.to_string())
}

/// The picture the bot and the phone get of this screen: its size, always the same for the same
/// screen. Coordinates on it map to the screen without remembering any earlier screenshot (that
/// hidden state made clicks fail with "screenshot first" and land wrong for a viewer whose
/// picture was a different size).
pub fn shot_size() -> (u32, u32) {
    let (px, py) = mac::screen_pixels();
    let (w, h, _) = fit(px.max(1), py.max(1));
    (w, h)
}

/// Screen points (the mouse's units) per picture pixel.
pub fn scale_here() -> (f64, f64) {
    let (pw, ph) = mac::screen_points();
    let (w, h) = shot_size();
    (pw / w.max(1) as f64, ph / h.max(1) as f64)
}

pub fn is_computer_tool(name: &str) -> bool {
    NAMES.contains(&name)
}

pub fn label(name: &str) -> &'static str {
    match name {
        "computer_use_screenshot" => "看你的螢幕",
        "computer_use_click" => "點一下",
        "computer_use_typing" => "打字",
        "computer_use_press_key" => "按鍵",
        "computer_use_scroll" => "捲動",
        "computer_use_mouse_move" => "移動滑鼠",
        "computer_use_drag" => "拖曳",
        "computer_use_mouse_button" => "按住或放開滑鼠",
        "computer_use_check_permissions" => "檢查螢幕權限",
        "computer_use_read_app" => "讀 App 畫面",
        "computer_use_open_app" => "打開 App",
        _ => "操作畫面",
    }
}

fn args_of(args: &str) -> Value {
    serde_json::from_str(if args.trim().is_empty() { "{}" } else { args }).unwrap_or(Value::Null)
}

pub fn describe(name: &str, args: &str) -> String {
    let v = args_of(args);
    // By number from the app's list (e7), else where on the screen (it said "(0, 0)" for every
    // click by number).
    let xy = || match v["ref"].as_str().map(str::trim).filter(|r| !r.is_empty()) {
        Some(r) => r.to_string(),
        None => format!("({}, {})", v["x"].as_f64().unwrap_or(0.0), v["y"].as_f64().unwrap_or(0.0)),
    };
    match name {
        "computer_use_screenshot" => "螢幕截圖".into(),
        "computer_use_click" => {
            let b = v["button"].as_str().unwrap_or("left");
            let n = v["count"].as_u64().unwrap_or(1);
            format!("{}{}{} {}", if b == "right" { "右鍵" } else { "" }, if n >= 2 { "連點" } else { "點" }, if n >= 2 { format!(" {n} 下") } else { String::new() }, xy())
        }
        "computer_use_typing" => {
            let t = v["value"].as_str().unwrap_or("");
            let short: String = t.chars().take(60).collect();
            let at = v["ref"].as_str().filter(|r| !r.trim().is_empty()).map(|r| format!("在 {r} ")).unwrap_or_default();
            format!("{at}「{short}{}」{}", if t.chars().count() > 60 { "…" } else { "" }, if v["submit"].as_bool().unwrap_or(false) { " 再按 return" } else { "" })
        }
        "computer_use_press_key" => v["key"].as_str().unwrap_or("").to_string(),
        "computer_use_scroll" => format!("{} {} 格，在 {}", v["direction"].as_str().unwrap_or("down"), v["amount"].as_i64().unwrap_or(3), xy()),
        "computer_use_mouse_move" => xy(),
        "computer_use_drag" => format!("{} 個點", v["path"].as_array().map(|a| a.len()).unwrap_or(0)),
        "computer_use_mouse_button" => format!("{}{}", if v["action"].as_str() == Some("release") { "放開" } else { "按住" }, if v["x"].is_null() { String::new() } else { format!(" {}", xy()) }),
        "computer_use_check_permissions" => "螢幕錄製、輔助使用".into(),
        "computer_use_read_app" | "computer_use_open_app" => v["app"].as_str().filter(|a| !a.is_empty()).unwrap_or("最前面的 App").to_string(),
        _ => String::new(),
    }
}

#[allow(dead_code)]
pub fn specs() -> Vec<Value> {
    specs_for(&[], "")
}

/// The screen tools, each with a `computer` choice. One way to act (a real click or keystroke),
/// two ways to see: the picture, or an app's controls as numbered text.
pub fn specs_for(computers: &[String], default: &str) -> Vec<Value> {
    let mods = json!({"type": "array", "items": {"type": "string", "enum": ["cmd", "shift", "option", "control"]}});
    let rf = json!({"type": "string", "description": "A control's number from computer_use_read_app (e.g. e12), instead of x, y"});
    let mut v = vec![
        json!({"name": "computer_use_screenshot", "description": "See a computer's screen. x, y in the other computer_use tools are pixels of this picture. Every action below already returns a fresh picture of the result, so call this only to start or to look again.",
            "parameters": {"type": "object", "properties": {}}}),
        json!({"name": "computer_use_click", "description": "Click at x, y (pixels of the latest picture), or on a control by its number (ref) from computer_use_read_app. Returns the screen after the click.",
            "parameters": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}, "ref": rf, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "count": {"type": "integer", "description": "2 for a double click"}, "modifier_keys": mods}}}),
        json!({"name": "computer_use_typing", "description": "Type text (any language) where the cursor is, or into a field by its number (ref; it is clicked first). submit=true presses return after the text (send a message, run a command). Returns the screen after.",
            "parameters": {"type": "object", "properties": {"value": {"type": "string"}, "ref": rf, "submit": {"type": "boolean"}}, "required": ["value"]}}),
        json!({"name": "computer_use_press_key", "description": "Press a key or shortcut in the app in front: return, tab, escape, delete, up, slash, cmd+c, cmd+shift+t, cmd+space. Several in order: separate with spaces (\"cmd+a delete\"). On a Linux computer cmd means ctrl. Returns the screen after.",
            "parameters": {"type": "object", "properties": {"key": {"type": "string"}, "hold_duration_ms": {"type": "integer"}}, "required": ["key"]}}),
        json!({"name": "computer_use_scroll", "description": "Scroll at x, y or on a control by number (ref); neither = where the pointer is. Returns the screen after.",
            "parameters": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}, "ref": rf, "direction": {"type": "string", "enum": ["up", "down", "left", "right"]}, "amount": {"type": "integer", "description": "Scroll steps of 16 pixels, 1-20, default 3"}, "modifier_keys": mods}, "required": ["direction"]}}),
        json!({"name": "computer_use_open_app", "description": "Open an app, or bring it to the front when it is already open (its name as shown, e.g. Ghostty, Calculator, 計算機, Chromium, xterm). Returns the screen after.",
            "parameters": {"type": "object", "properties": {"app": {"type": "string"}}, "required": ["app"]}}),
        json!({"name": "computer_use_read_app", "description": "Read an app's window as text: buttons, fields, menus, list items and visible text, each control numbered (e1, e2…) for click/typing/scroll's ref. For when a picture can't be taken or isn't enough (exact text, long lists). app: its name; leave out for the app in front. Only on computers that offer it (not the bots' own Linux computer).",
            "parameters": {"type": "object", "properties": {"app": {"type": "string"}}}}),
        json!({"name": "computer_use_mouse_move", "description": "Move the mouse to x, y (e.g. to show a hover menu). Returns the screen after.",
            "parameters": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}),
        json!({"name": "computer_use_drag", "description": "Drag with the mouse along a path of points (first = press, last = release). Returns the screen after.",
            "parameters": {"type": "object", "properties": {"path": {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}, "y": {"type": "number"}}, "required": ["x", "y"]}}, "button": {"type": "string", "enum": ["left", "right"]}, "modifier_keys": mods}, "required": ["path"]}}),
        json!({"name": "computer_use_mouse_button", "description": "Press and hold, or release, a mouse button (optionally moving to x, y first). For long presses and custom drags.",
            "parameters": {"type": "object", "properties": {"action": {"type": "string", "enum": ["press", "release"]}, "button": {"type": "string", "enum": ["left", "right", "middle"]}, "x": {"type": "number"}, "y": {"type": "number"}}, "required": ["action"]}}),
        json!({"name": "computer_use_check_permissions", "description": "Check whether the screen can be seen and the mouse and keyboard used on that computer.",
            "parameters": {"type": "object", "properties": {}}}),
    ];
    // Always named: each computer goes by its own name (the instructions say which is which),
    // so the bot says whose screen it means and the person can tell where it worked.
    if !computers.is_empty() {
        let desc = format!("Whose screen, by the computer's name. Default: {default} (the one the person chose). Use another only when the person names it.");
        for t in v.iter_mut() {
            t["parameters"]["properties"]["computer"] = json!({"type": "string", "enum": computers, "description": desc});
        }
    }
    v
}

/// zct: the size to send. Long edge down to `MAX_LONG_EDGE`, never an edge under `MIN_EDGE`.
pub fn fit(w: u32, h: u32) -> (u32, u32, bool) {
    if w < 1 || h < 1 {
        return (w, h, false);
    }
    let long = w.max(h);
    if long <= MAX_LONG_EDGE {
        return (w, h, false);
    }
    let short = w.min(h);
    if short < MIN_EDGE {
        return (w, h, false);
    }
    let mut i = MAX_LONG_EDGE as f64 / long as f64;
    let mut o = ((w as f64 * i).round() as u32).max(1);
    let mut a = ((h as f64 * i).round() as u32).max(1);
    if o < MIN_EDGE || a < MIN_EDGE {
        i = MIN_EDGE as f64 / short as f64;
        o = ((w as f64 * i).round() as u32).max(MIN_EDGE);
        a = ((h as f64 * i).round() as u32).max(MIN_EDGE);
    }
    (o, a, o != w || a != h)
}

/// j3r: JPEG at 85, 70, 55: the first within the budget, else the smallest if it's under the cap.
pub fn encode_jpeg(img: &image::RgbImage) -> Option<Vec<u8>> {
    let mut best: Option<Vec<u8>> = None;
    for q in JPEG_QUALITIES {
        let mut out = vec![];
        let enc = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, q);
        if img.write_with_encoder(enc).is_err() {
            continue;
        }
        if out.len() <= MAX_IMAGE_BYTES {
            return Some(out);
        }
        if best.as_ref().map_or(true, |b| out.len() < b.len()) {
            best = Some(out);
        }
    }
    best.filter(|b| b.len() <= MAX_IMAGE_BYTES)
}

/// Screenshot pixel → screen point.
pub fn to_screen(x: f64, y: f64, scale: (f64, f64)) -> (f64, f64) {
    (x * scale.0, y * scale.1)
}

/// Key name → (key code, needs shift). US layout codes.
pub fn key_code(name: &str) -> Option<u16> {
    let n = name.trim().to_lowercase();
    let named = match n.as_str() {
        "return" | "enter" => 36,
        "tab" => 48,
        "space" => 49,
        "delete" | "backspace" => 51,
        "escape" | "esc" => 53,
        "forwarddelete" | "forward_delete" | "del" => 117,
        "home" => 115,
        "end" => 119,
        "pageup" | "page_up" => 116,
        "pagedown" | "page_down" => 121,
        "left" | "arrowleft" => 123,
        "right" | "arrowright" => 124,
        "down" | "arrowdown" => 125,
        "up" | "arrowup" => 126,
        "f1" => 122,
        "f2" => 120,
        "f3" => 99,
        "f4" => 118,
        "f5" => 96,
        "f6" => 97,
        "f7" => 98,
        "f8" => 100,
        "f9" => 101,
        "f10" => 109,
        "f11" => 103,
        "f12" => 111,
        // Symbol keys by name (a model writes "slash" as often as "/").
        "slash" => 44,
        "backslash" => 42,
        "comma" => 43,
        "period" | "dot" => 47,
        "semicolon" => 41,
        "quote" | "apostrophe" => 39,
        "minus" | "hyphen" => 27,
        "equal" | "equals" => 24,
        "grave" | "backtick" | "backquote" => 50,
        "bracketleft" | "leftbracket" => 33,
        "bracketright" | "rightbracket" => 30,
        _ => u16::MAX,
    };
    if named != u16::MAX {
        return Some(named);
    }
    let mut cs = n.chars();
    let (Some(c), None) = (cs.next(), cs.next()) else { return None };
    const LETTERS: [u16; 26] = [0, 11, 8, 2, 14, 3, 5, 4, 34, 38, 40, 37, 46, 45, 31, 35, 12, 15, 1, 17, 32, 9, 13, 7, 16, 6];
    const DIGITS: [u16; 10] = [29, 18, 19, 20, 21, 23, 22, 26, 28, 25];
    match c {
        'a'..='z' => Some(LETTERS[(c as u8 - b'a') as usize]),
        '0'..='9' => Some(DIGITS[(c as u8 - b'0') as usize]),
        '-' => Some(27),
        '=' => Some(24),
        '[' => Some(33),
        ']' => Some(30),
        ';' => Some(41),
        '\'' => Some(39),
        ',' => Some(43),
        '.' => Some(47),
        '/' => Some(44),
        '\\' => Some(42),
        '`' => Some(50),
        _ => None,
    }
}

pub const FLAG_SHIFT: u64 = 1 << 17;
pub const FLAG_CONTROL: u64 = 1 << 18;
pub const FLAG_OPTION: u64 = 1 << 19;
pub const FLAG_CMD: u64 = 1 << 20;

pub fn modifier_flag(m: &str) -> Option<u64> {
    match m.trim().to_lowercase().as_str() {
        "cmd" | "command" | "meta" | "super" | "⌘" => Some(FLAG_CMD),
        "shift" => Some(FLAG_SHIFT),
        "option" | "alt" | "opt" => Some(FLAG_OPTION),
        "control" | "ctrl" => Some(FLAG_CONTROL),
        _ => None,
    }
}

/// "cmd+shift+t" → (key code, flags).
pub fn parse_combo(combo: &str) -> Result<(u16, u64), String> {
    let parts: Vec<&str> = combo.split('+').map(str::trim).filter(|p| !p.is_empty()).collect();
    let Some((key, mods)) = parts.split_last() else { return Err("請給要按的鍵".into()) };
    let mut flags = 0;
    for m in mods {
        flags |= modifier_flag(m).ok_or_else(|| format!("不認得「{m}」這個修飾鍵"))?;
    }
    let code = key_code(key).ok_or_else(|| format!("不認得「{key}」這個鍵"))?;
    Ok((code, flags))
}

fn flags_of(v: &Value) -> u64 {
    v["modifier_keys"].as_array().map(|a| a.iter().filter_map(|m| m.as_str().and_then(modifier_flag)).fold(0, |f, x| f | x)).unwrap_or(0)
}

/// Run a computer_use tool. Returns (text, ok, screenshot as base64 JPEG).
#[allow(dead_code)]
pub fn run(name: &str, args: &str) -> (String, bool, Option<String>) {
    run_by(name, args, "")
}

/// Same, remembering which bot acted (for the 「正在操作你的電腦」 bar).
pub fn run_by(name: &str, args: &str, who: &str) -> (String, bool, Option<String>) {
    if moves_things(name) {
        *LAST_ACTION.lock().unwrap_or_else(|e| e.into_inner()) = Some((std::time::Instant::now(), who.to_string()));
    }
    match run_inner(name, &args_of(args)) {
        Ok((t, img)) => (t, true, img),
        Err(e) => (e, false, None),
    }
}

/// Steps that use the mouse or keyboard (and come back with the screen after them).
pub fn moves_things(name: &str) -> bool {
    matches!(name, "computer_use_click" | "computer_use_typing" | "computer_use_press_key" | "computer_use_scroll" | "computer_use_mouse_move" | "computer_use_drag" | "computer_use_mouse_button" | "computer_use_open_app")
}

/// Ask macOS for a permission (its own prompt, then 系統設定 at that pane) at most once every ten
/// minutes. Asking on every step meant a bot working through several steps — or a brain far away
/// working on this computer — covered the screen with prompts and Settings windows, and the
/// person could do nothing else.
pub fn ask_permission(pane: &str) {
    static LAST: std::sync::Mutex<Vec<(String, std::time::Instant)>> = std::sync::Mutex::new(Vec::new());
    {
        let mut l = LAST.lock().unwrap_or_else(|e| e.into_inner());
        if l.iter().any(|(p, t)| p == pane && t.elapsed() < std::time::Duration::from_secs(600)) {
            return;
        }
        l.retain(|(p, _)| p != pane);
        l.push((pane.to_string(), std::time::Instant::now()));
    }
    if !cfg!(target_os = "macos") {
        return;
    }
    match pane {
        "Privacy_Accessibility" => {
            let _ = crate::axread::trusted(true);
        }
        #[cfg(target_os = "macos")]
        "Privacy_ScreenCapture" => {
            let _ = mac::screen_capture_allowed();
        }
        _ => {}
    }
    let _ = spawn_reaped(&mut std::process::Command::new("open").arg(format!("x-apple.systempreferences:com.apple.preference.security?{pane}")));
}

const NO_ACCESSIBILITY: &str = "macOS 還沒讓 AnyBot 使用「輔助使用」，所以不能讀 App，也不能動滑鼠和鍵盤。這台電腦上已經跳出詢問並打開「系統設定 → 隱私權與安全性 → 輔助使用」：請使用者把 AnyBot 打開（只要一次），再叫我試一次。";
const NO_SCREEN_RECORDING: &str = "macOS 還沒讓 AnyBot 錄製螢幕，所以看不到畫面。這台電腦上已經打開「系統設定 → 隱私權與安全性 → 螢幕與系統錄音」：請使用者把 AnyBot 打開（只要一次）。在那之前可以用 computer_use_read_app 讀 App 的文字。";

/// Reading apps and using the mouse and keyboard need 輔助使用 on macOS (other systems: always).
fn controls_allowed() -> bool {
    !cfg!(target_os = "macos") || crate::axread::trusted(false)
}

fn need_controls() -> Result<(), String> {
    if controls_allowed() {
        return Ok(());
    }
    ask_permission("Privacy_Accessibility");
    Err(NO_ACCESSIBILITY.into())
}

/// The app a shown name stands for (「計算機」 → Calculator), from the apps' display names.
#[cfg(target_os = "macos")]
fn app_named(name: &str) -> Option<String> {
    if name.is_empty() || name.contains(['"', '\\', '\'']) {
        return None;
    }
    let q = format!("kMDItemDisplayName == \"{name}\"cd && kMDItemContentType == \"com.apple.application-bundle\"");
    let out = std::process::Command::new("mdfind").arg(q).output().ok()?;
    let path = String::from_utf8_lossy(&out.stdout).lines().next()?.trim().to_string();
    std::path::Path::new(&path).file_stem().map(|s| s.to_string_lossy().to_string())
}

#[cfg(not(target_os = "macos"))]
fn app_named(_name: &str) -> Option<String> {
    None
}

/// Open an app or bring it to the front. A name as the person sees it (「計算機」 on a Chinese
/// Mac) isn't one the system launches by; find the app it is and launch that.
fn open_app(app: &str) -> Result<String, String> {
    let app = app.trim();
    if app.is_empty() {
        return Err("請給要打開的 App 名稱".into());
    }
    crate::desktop::launch(app).or_else(|e| match app_named(app) {
        Some(real) if real != app => crate::desktop::launch(&real),
        _ => Err(e),
    })?;
    Ok(format!("已打開「{app}」，放在最前面。"))
}

/// Bring the app a control belongs to in front, so a click at its spot lands in it (and not in
/// a window lying over it). Nothing to do when it is in front already.
fn bring_forward(app: &str) {
    // macOS only: there "launch" brings a running app forward; on Linux it can start a second
    // copy, and the names a window and its accessibility give differ.
    if !cfg!(target_os = "macos") {
        return;
    }
    let front = mac::front_app().map(|f| f.0).unwrap_or_default();
    if app.is_empty() || front.eq_ignore_ascii_case(app) {
        return;
    }
    if crate::desktop::launch(app).is_ok() {
        std::thread::sleep(std::time::Duration::from_millis(300));
    }
}

/// Where a step acts: a control's number (from computer_use_read_app), or x, y on the picture.
/// Returns the screen point and how to say it (「e7（按鈕「7」）」, 「(512, 300)」).
fn place_of(v: &Value, scale: (f64, f64)) -> Result<Option<(f64, f64, String)>, String> {
    if let Some(r) = v["ref"].as_str().map(str::trim).filter(|r| !r.is_empty()) {
        let s = crate::desktop::spot(r)?;
        bring_forward(&s.app);
        return Ok(Some((s.x, s.y, format!("{}（{}）", crate::desktop::short_ref(r)?, s.what))));
    }
    match (v["x"].as_f64(), v["y"].as_f64()) {
        (Some(x), Some(y)) => {
            let (sx, sy) = to_screen(x, y, scale);
            Ok(Some((sx, sy, format!("({x}, {y})"))))
        }
        (None, None) if v["x"].is_null() && v["y"].is_null() => Ok(None),
        _ => Err("x 和 y 要一起給，而且要是數字（截圖上的位置）".into()),
    }
}

/// Press each key of "cmd+a delete" in order.
fn press_keys(keys: &str, hold: f64) -> Result<(), String> {
    for (i, k) in key_plan(keys)?.iter().enumerate() {
        if i > 0 {
            std::thread::sleep(std::time::Duration::from_secs_f64(KEY_GAP_SECS));
        }
        match k {
            Key::Combo(code, flags) => mac::press(*code, *flags, hold),
            Key::Char(ch) => mac::type_text(ch),
        }
    }
    Ok(())
}

/// Do one mouse or keyboard step at the screen's own coordinates (`scale`: screen points per
/// picture pixel). The bot's tools and a person tapping on the phone's picture both come here:
/// one way of acting on a screen.
pub fn act(name: &str, v: &Value, scale: (f64, f64)) -> Result<String, String> {
    let button = v["button"].as_str().unwrap_or("left");
    match name {
        "computer_use_click" => {
            let (x, y, at) = place_of(v, scale)?.ok_or("請給 x、y（截圖上的位置）或 ref（computer_use_read_app 的編號）")?;
            let count = v["count"].as_u64().unwrap_or(1).clamp(1, 3) as i64;
            mac::click(x, y, button, count, flags_of(v));
            let how = format!("{}{}", if button == "right" { "右鍵" } else { "" }, if count >= 2 { format!("連點 {count} 下") } else { "點".into() });
            Ok(format!("已{how} {at}。"))
        }
        "computer_use_mouse_move" => {
            let (x, y, at) = place_of(v, scale)?.ok_or("請給 x 和 y")?;
            mac::move_to(x, y);
            Ok(format!("已把滑鼠移到 {at}。"))
        }
        "computer_use_scroll" => {
            // CURemoteToolService.handleScroll @0x10009e200 / closure @0x10009e998.
            let at = place_of(v, scale)?;
            if !v["amount"].is_null() && v["amount"].as_i64().is_none() {
                return Err("amount 要是整數".into());
            }
            let amount = scroll_amount(v["amount"].as_i64());
            let dir = v["direction"].as_str().unwrap_or("").trim().to_lowercase();
            let (dy, dx) = scroll_pixels(&dir, amount).ok_or_else(|| format!("方向「{dir}」要是 up、down、left 或 right"))?;
            mac::scroll(at.as_ref().map(|(x, y, _)| (*x, *y)), dy, dx, flags_of(v));
            Ok(format!("已捲動 {dir} ×{amount}{}。", at.map(|(_, _, a)| format!("，在 {a}")).unwrap_or_default()))
        }
        "computer_use_drag" => {
            // Every point must be valid: points that failed to parse used to be dropped in
            // silence, so the drag ran on fewer points and reported success at the wrong place.
            let raw = v["path"].as_array().ok_or("拖曳要給 path：一個點陣，每個點有數字 x 和 y（用截圖上的座標）")?;
            let pts: Vec<(f64, f64)> = raw.iter().map(|p| match (p["x"].as_f64(), p["y"].as_f64()) {
                (Some(x), Some(y)) => Ok((x, y)),
                _ => Err("拖曳的每個點都要有數字 x 和 y（用截圖上的座標）".to_string()),
            }).collect::<Result<_, _>>()?;
            if pts.len() < 2 {
                return Err("拖曳至少要兩個點（每個點要有數字 x 和 y，用截圖上的座標）".into());
            }
            let path: Vec<(f64, f64)> = pts.iter().map(|(x, y)| to_screen(*x, *y, scale)).collect();
            mac::drag(&path, button, flags_of(v));
            // Like the original's "Dragged from (x, y) to (x, y) with left button."
            let side = match button {
                "right" => "右鍵",
                "middle" => "中鍵",
                _ => "左鍵",
            };
            let (a, b) = (pts[0], pts[pts.len() - 1]);
            Ok(format!("已從 ({:.0}, {:.0}) 拖曳到 ({:.0}, {:.0})（{side}）。", a.0, a.1, b.0, b.1))
        }
        "computer_use_mouse_button" => {
            let at = place_of(v, scale)?.map(|(x, y, _)| (x, y));
            let release = v["action"].as_str() == Some("release");
            mac::mouse_button(at, button, !release);
            Ok((if release { "已放開滑鼠鍵。" } else { "已按住滑鼠鍵（記得之後放開）。" }).into())
        }
        "computer_use_typing" => {
            let text = v["value"].as_str().unwrap_or("");
            let submit = v["submit"].as_bool().unwrap_or(false);
            if text.is_empty() && !submit {
                return Err("請給要打的字".into());
            }
            // Into a field by its number: click it first, like a person would.
            let at = place_of(&json!({"ref": v["ref"]}), scale)?;
            if let Some((x, y, _)) = &at {
                mac::click(*x, *y, "left", 1, 0);
                std::thread::sleep(std::time::Duration::from_millis(150));
            }
            if !text.is_empty() {
                mac::type_text(text);
            }
            if submit {
                std::thread::sleep(std::time::Duration::from_millis(60));
                mac::press(36, 0, KEY_HOLD_SECS);
            }
            Ok(format!("已打字（{} 個字）{}{}。", text.chars().count(), at.map(|(_, _, a)| format!("在 {a}")).unwrap_or_default(), if submit { "，再按了 return" } else { "" }))
        }
        "computer_use_press_key" => {
            let hold = v["hold_duration_ms"].as_u64().map(|ms| ms.min(MAX_HOLD_MS) as f64 / 1000.0).filter(|s| *s > 0.0).unwrap_or(KEY_HOLD_SECS);
            press_keys(v["key"].as_str().unwrap_or(""), hold)?;
            Ok(format!("已按 {}。", v["key"].as_str().unwrap_or("")))
        }
        "computer_use_open_app" => open_app(v["app"].as_str().unwrap_or("")),
        other => Err(format!("沒有「{other}」這個工具")),
    }
}

/// What the screen looks like now, after a step: the picture when this computer can take one,
/// else the app in front as numbered text. The bot sees the result of each step in the same
/// answer instead of spending another step (another trip to the model) on a screenshot.
fn observe(note: String) -> (String, Option<String>) {
    std::thread::sleep(std::time::Duration::from_millis(SETTLE_MS));
    if mac::screen_capture_preflight() {
        return match screenshot() {
            Ok((t, img)) => (format!("{note}\n{t}"), Some(img)),
            Err(e) => (format!("{note}\n（看不到結果：{e}）"), None),
        };
    }
    if controls_allowed() {
        if let Ok((app, lines)) = crate::desktop::read_app("") {
            let shown: Vec<&String> = lines.iter().take(120).collect();
            return (format!("{note}\n（這台現在拍不到畫面，改讀最前面的「{app}」；e 編號可以用在 ref）\n{}", shown.iter().map(|l| l.as_str()).collect::<Vec<_>>().join("\n")), None);
        }
    }
    (format!("{note}（這台現在看不到畫面，無法確認結果）"), None)
}

fn run_inner(name: &str, v: &Value) -> Result<(String, Option<String>), String> {
    if !cfg!(any(target_os = "macos", target_os = "linux")) {
        return Err("操作畫面只支援 macOS 和 Linux".into());
    }
    #[cfg(target_os = "linux")]
    if !crate::desk::available() {
        return Err(crate::desk::describe().into());
    }
    match name {
        "computer_use_screenshot" => return screenshot().map(|(t, img)| (t, Some(img))),
        "computer_use_check_permissions" => {
            #[cfg(target_os = "linux")]
            return Ok((crate::desk::describe().into(), None));
            #[allow(unreachable_code)]
            {
                let see = mac::screen_capture_preflight();
                let control = controls_allowed();
                return Ok((format!("螢幕錄製（看畫面）：{}\n輔助使用（滑鼠、鍵盤、讀 App）：{}{}", if see { "已允許" } else { "還沒允許" }, if control { "已允許" } else { "還沒允許" }, if see && control { "" } else { "\n請使用者到「系統設定 → 隱私權與安全性」打開 AnyBot。" }), None));
            }
        }
        "computer_use_read_app" => {
            need_controls()?;
            let (app, lines) = crate::desktop::read_app(v["app"].as_str().unwrap_or("")).map_err(plain_words)?;
            return Ok((if lines.is_empty() {
                format!("「{app}」的畫面讀不到內容（有些 App 不提供輔助使用資訊）。可以改用 computer_use_screenshot 看。")
            } else {
                format!("「{app}」的畫面（e 編號可以用在 click / typing / scroll 的 ref）：\n{}", lines.join("\n"))
            }, None));
        }
        _ => {}
    }
    if !moves_things(name) {
        return Err(format!("沒有「{name}」這個工具"));
    }
    if HELD_BY_PERSON.load(std::sync::atomic::Ordering::SeqCst) {
        return Err("使用者正在自己操作這台電腦（已取得控制權）。請等他交還控制權後再操作；可以先截圖看看。".into());
    }
    if name != "computer_use_open_app" {
        need_controls()?;
    }
    let note = act(name, v, scale_here()).map_err(plain_words)?;
    // Holding a button down is the middle of something (a long press, a drag): no picture yet.
    if name == "computer_use_mouse_button" {
        return Ok((note, None));
    }
    Ok(observe(note))
}

/// agent-desktop's error codes, as words the bot can pass on.
fn plain_words(t: String) -> String {
    if t.contains("PERM_DENIED") {
        ask_permission("Privacy_Accessibility");
        return NO_ACCESSIBILITY.to_string();
    }
    if t.contains("APP_NOT_FOUND") {
        return format!("找不到這個 App（沒有開著，或名字不對）：先用 computer_use_open_app 打開它。（{t}）");
    }
    t
}

/// The phone or another viewer acting on this screen as the person: their tap is on the picture
/// they were sent (`shot_size()`), so it maps exactly like the bot's coordinates. Not refused
/// while the person holds the screen (it is the person), and no picture back (the viewer
/// refreshes its own).
pub fn person_act(name: &str, v: &Value) -> Result<(), String> {
    need_controls()?;
    act(name, v, scale_here()).map(|_| ())
}

/// The app a screen action will land in: the window under the point for clicks, else the
/// frontmost app. Used for the per-app permission (「AnyBot 可以看到 {app}，並在其中點擊和輸入」).
pub fn target_app(name: &str, args: &str) -> Option<String> {
    let v = args_of(args);
    if let Some(app) = v["app"].as_str().filter(|a| !a.trim().is_empty() && matches!(name, "computer_use_read_app" | "computer_use_open_app")) {
        return Some(app.trim().to_string());
    }
    if name == "computer_use_read_app" {
        return frontmost_app();
    }
    if let Some(r) = v["ref"].as_str().filter(|r| !r.trim().is_empty()) {
        return crate::desktop::spot(r).ok().map(|s| s.app).or_else(crate::desktop::last_app);
    }
    let s = scale_here();
    let point = match name {
        "computer_use_click" | "computer_use_mouse_move" | "computer_use_scroll" | "computer_use_mouse_button" if !v["x"].is_null() => Some(to_screen(v["x"].as_f64()?, v["y"].as_f64()?, s)),
        "computer_use_drag" => v["path"].as_array().and_then(|a| a.first()).and_then(|p| Some(to_screen(p["x"].as_f64()?, p["y"].as_f64()?, s))),
        _ => None,
    };
    match point {
        Some((x, y)) => mac::window_at(x, y).map(|w| w.0),
        None => mac::front_app().map(|w| w.0),
    }
}

/// Screenshot pixels per screen point (for the demonstration recorder).
pub fn points_to_shot() -> f64 {
    let (pw, _) = mac::screen_points();
    let (px, py) = mac::screen_pixels();
    let (fw, _, _) = fit(px.max(1), py.max(1));
    if pw > 0.0 { fw as f64 / pw } else { 1.0 }
}

/// The person's own clicks and keys can be recorded (教它一項任務): macOS, Linux on X11.
/// Wayland doesn't let apps watch other apps' input.
pub fn demo_supported() -> bool {
    #[cfg(target_os = "linux")]
    {
        crate::desk::kind() == crate::desk::Kind::X11
    }
    #[cfg(not(target_os = "linux"))]
    {
        cfg!(target_os = "macos")
    }
}

/// This system's screen can be seen and used (macOS; Linux on X11 or Wayland).
pub fn screen_supported() -> bool {
    #[cfg(target_os = "linux")]
    {
        crate::desk::available()
    }
    #[cfg(not(target_os = "linux"))]
    {
        cfg!(target_os = "macos")
    }
}

pub fn screen_capture_ok() -> bool {
    mac::screen_capture_preflight()
}

#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
pub fn frontmost_app() -> Option<String> {
    mac::front_app().map(|w| w.0)
}

#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
pub fn app_at_point(x: f64, y: f64) -> Option<String> {
    mac::window_at(x, y).map(|w| w.0)
}

fn screenshot() -> Result<(String, String), String> {
    if !mac::screen_capture_preflight() {
        ask_permission("Privacy_ScreenCapture");
        return Err(NO_SCREEN_RECORDING.into());
    }
    let img = capture_screen()?;
    // Always the same size for this screen (shot_size), so coordinates map without state.
    let (fw, fh) = shot_size();
    let img = if img.dimensions() != (fw, fh) { image::imageops::resize(&img, fw, fh, image::imageops::FilterType::Triangle) } else { img };
    let jpeg = encode_jpeg(&img).ok_or("截圖太大了，沒辦法送出")?;
    let b64 = base64::engine::general_purpose::STANDARD.encode(&jpeg);
    Ok((format!("螢幕截圖（{fw}×{fh}）。座標用這張圖上的像素。"), b64))
}

/// The whole screen as RGB (backing pixels).
pub fn capture_screen() -> Result<image::RgbImage, String> {
    #[cfg(target_os = "linux")]
    return crate::desk::capture();
    #[allow(unreachable_code)]
    {
        // A random tail: two bots screenshotting the same millisecond must not delete each
        // other's file (pid+time alone collides).
        let file = std::env::temp_dir().join(format!("anybot-screen-{}-{}-{}.png", std::process::id(), crate::model::now_ms(), crate::httpd::random_hex(4)));
        let st = std::process::Command::new("screencapture").args(["-x", "-m", "-t", "png"]).arg(&file).status().map_err(|e| e.to_string())?;
        let bytes = std::fs::read(&file);
        let _ = std::fs::remove_file(&file);
        if !st.success() {
            return Err("截圖失敗".into());
        }
        Ok(image::load_from_memory(&bytes.map_err(|e| format!("截圖失敗：{e}"))?).map_err(|e| format!("截圖讀不出來：{e}"))?.to_rgb8())
    }
}

#[cfg(target_os = "linux")]
use crate::desk as mac;

#[cfg(target_os = "macos")]
mod mac {
    use super::*;
    use std::ffi::c_void;
    use std::time::Duration;

    #[repr(C)]
    #[derive(Clone, Copy)]
    struct CGPoint {
        x: f64,
        y: f64,
    }
    #[repr(C)]
    #[derive(Clone, Copy)]
    struct CGSize {
        width: f64,
        height: f64,
    }
    #[repr(C)]
    #[derive(Clone, Copy)]
    struct CGRect {
        origin: CGPoint,
        size: CGSize,
    }
    type Ev = *mut c_void;

    type Src = *mut c_void;

    #[link(name = "CoreGraphics", kind = "framework")]
    extern "C" {
        fn CGMainDisplayID() -> u32;
        fn CGDisplayBounds(d: u32) -> CGRect;
        fn CGPreflightScreenCaptureAccess() -> bool;
        fn CGRequestScreenCaptureAccess() -> bool;
        fn CGEventSourceCreate(state: i32) -> Src;
        fn CGEventSourceSetLocalEventsSuppressionInterval(src: Src, secs: f64);
        fn CGEventCreateMouseEvent(src: Src, kind: u32, at: CGPoint, button: u32) -> Ev;
        fn CGEventCreateKeyboardEvent(src: Src, key: u16, down: bool) -> Ev;
        fn CGEventKeyboardSetUnicodeString(e: Ev, len: usize, s: *const u16);
        fn CGEventSetFlags(e: Ev, flags: u64);
        fn CGEventSetIntegerValueField(e: Ev, field: u32, v: i64);
        fn CGEventPost(tap: u32, e: Ev);
        fn CGEventCreateScrollWheelEvent(src: Src, units: u32, count: u32, wheel1: i32, ...) -> Ev;
        fn CGDisplayPixelsWide(d: u32) -> usize;
        fn CGDisplayPixelsHigh(d: u32) -> usize;
        fn CGWindowListCopyWindowInfo(option: u32, relative_to: u32) -> *const c_void;
        fn CGRectMakeWithDictionaryRepresentation(dict: *const c_void, rect: *mut CGRect) -> bool;
        static kCGWindowOwnerName: *const c_void;
        static kCGWindowOwnerPID: *const c_void;
        static kCGWindowBounds: *const c_void;
        static kCGWindowLayer: *const c_void;
    }
    #[link(name = "CoreFoundation", kind = "framework")]
    extern "C" {
        fn CFArrayGetCount(a: *const c_void) -> isize;
        fn CFArrayGetValueAtIndex(a: *const c_void, i: isize) -> *const c_void;
        fn CFDictionaryGetValue(d: *const c_void, k: *const c_void) -> *const c_void;
        fn CFNumberGetValue(n: *const c_void, kind: isize, out: *mut c_void) -> bool;
        fn CFStringGetCString(s: *const c_void, buf: *mut std::ffi::c_char, len: isize, enc: u32) -> bool;
    }

    /// On-screen windows front to back: (owner app, pid, bounds), normal layer only.
    fn windows() -> Vec<(String, i32, CGRect)> {
        let mut out = vec![];
        unsafe {
            // kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements
            let list = CGWindowListCopyWindowInfo(1 | 16, 0);
            if list.is_null() {
                return out;
            }
            for i in 0..CFArrayGetCount(list) {
                let d = CFArrayGetValueAtIndex(list, i);
                let mut layer: i32 = -1;
                let l = CFDictionaryGetValue(d, kCGWindowLayer);
                if l.is_null() || !CFNumberGetValue(l, 3, &mut layer as *mut i32 as *mut c_void) || layer != 0 {
                    continue;
                }
                let mut pid: i32 = 0;
                let p = CFDictionaryGetValue(d, kCGWindowOwnerPID);
                if !p.is_null() {
                    CFNumberGetValue(p, 3, &mut pid as *mut i32 as *mut c_void);
                }
                let mut name = String::new();
                let n = CFDictionaryGetValue(d, kCGWindowOwnerName);
                if !n.is_null() {
                    let mut buf = [0u8; 256];
                    if CFStringGetCString(n, buf.as_mut_ptr() as *mut std::ffi::c_char, buf.len() as isize, 0x0800_0100) {
                        let end = buf.iter().position(|b| *b == 0).unwrap_or(buf.len());
                        name = String::from_utf8_lossy(&buf[..end]).to_string();
                    }
                }
                let mut r = CGRect { origin: CGPoint { x: 0.0, y: 0.0 }, size: CGSize { width: 0.0, height: 0.0 } };
                let b = CFDictionaryGetValue(d, kCGWindowBounds);
                if !b.is_null() {
                    CGRectMakeWithDictionaryRepresentation(b, &mut r);
                }
                out.push((name, pid, r));
            }
            CFRelease(list);
        }
        out
    }

    /// The app whose window is under a screen point.
    pub fn window_at(x: f64, y: f64) -> Option<(String, i32)> {
        windows().into_iter().find(|(_, _, r)| x >= r.origin.x && y >= r.origin.y && x < r.origin.x + r.size.width && y < r.origin.y + r.size.height).map(|(n, p, _)| (n, p))
    }

    /// The frontmost app (its frontmost normal window).
    pub fn front_app() -> Option<(String, i32)> {
        windows().into_iter().find(|(n, _, r)| !n.is_empty() && r.size.width > 1.0).map(|(n, p, _)| (n, p))
    }

    pub fn screen_capture_preflight() -> bool {
        unsafe { CGPreflightScreenCaptureAccess() }
    }

    pub fn screen_pixels() -> (u32, u32) {
        let d = unsafe { CGMainDisplayID() };
        let (pw, ph) = screen_points();
        // CGDisplayPixelsWide reports points on Retina; screencapture gives backing pixels.
        let (w, h) = unsafe { (CGDisplayPixelsWide(d) as f64, CGDisplayPixelsHigh(d) as f64) };
        let scale = backing_scale();
        let _ = (w, h);
        ((pw * scale) as u32, (ph * scale) as u32)
    }

    fn backing_scale() -> f64 {
        static S: std::sync::OnceLock<f64> = std::sync::OnceLock::new();
        *S.get_or_init(|| {
            let file = std::env::temp_dir().join(format!("anybot-scale-{}.png", std::process::id()));
            let ok = std::process::Command::new("screencapture").args(["-x", "-m", "-t", "png", "-R0,0,10,10"]).arg(&file).status().map(|s| s.success()).unwrap_or(false);
            let w = if ok { image::image_dimensions(&file).map(|d| d.0).unwrap_or(10) } else { 10 };
            let _ = std::fs::remove_file(&file);
            (w as f64 / 10.0).max(1.0)
        })
    }

    /// Press or release a mouse button where the pointer is (or at a point).
    pub fn mouse_button(at: Option<(f64, f64)>, button: &str, press: bool) {
        let (down, up, _, b) = kinds(button);
        let p = match at {
            Some((x, y)) => {
                move_to(x, y);
                gap();
                CGPoint { x, y }
            }
            None => current_point(),
        };
        let src = Source::new();
        let e = unsafe { CGEventCreateMouseEvent(src.0, if press { down } else { up }, p, b) };
        if !e.is_null() {
            unsafe { CGEventSetIntegerValueField(e, CLICK_STATE, 1) };
        }
        post(e, 0);
    }

    fn current_point() -> CGPoint {
        extern "C" {
            fn CGEventCreate(src: Src) -> Ev;
            fn CGEventGetLocation(e: Ev) -> CGPoint;
        }
        unsafe {
            let e = CGEventCreate(std::ptr::null_mut());
            if e.is_null() {
                return CGPoint { x: 0.0, y: 0.0 };
            }
            let p = CGEventGetLocation(e);
            CFRelease(e);
            p
        }
    }
    #[link(name = "CoreFoundation", kind = "framework")]
    extern "C" {
        fn CFRelease(x: *const c_void);
    }

    const HID_TAP: u32 = 0;
    /// kCGEventSourceStateHIDSystemState, as CUActuator uses (CGEventSourceCreate(1)).
    const HID_STATE: i32 = 1;
    const CLICK_STATE: u32 = 1;
    const MOVED: u32 = 5;
    /// kCGScrollEventUnitPixel, as CGEventCreateScrollWheelEvent2(_, 0, 2, …) in the original.
    const SCROLL_PIXEL: u32 = 0;

    /// An event source like the original's: HID system state, local-event suppression 0 s, so
    /// the person's own mouse and keys aren't frozen after each synthetic event.
    struct Source(Src);
    impl Source {
        fn new() -> Self {
            let s = unsafe { CGEventSourceCreate(HID_STATE) };
            if !s.is_null() {
                unsafe { CGEventSourceSetLocalEventsSuppressionInterval(s, 0.0) };
            }
            Source(s)
        }
    }
    impl Drop for Source {
        fn drop(&mut self) {
            if !self.0.is_null() {
                unsafe { CFRelease(self.0 as *const c_void) }
            }
        }
    }

    fn post(e: Ev, flags: u64) {
        if e.is_null() {
            return;
        }
        unsafe {
            CGEventSetFlags(e, flags);
            CGEventPost(HID_TAP, e);
            CFRelease(e);
        }
    }

    fn gap() {
        std::thread::sleep(Duration::from_millis(EVENT_GAP_MS));
    }

    fn sleep_secs(s: f64) {
        std::thread::sleep(Duration::from_secs_f64(s));
    }

    /// (down, up, dragged) event kinds and button number.
    fn kinds(button: &str) -> (u32, u32, u32, u32) {
        match button {
            "right" => (3, 4, 7, 1),
            "middle" => (25, 26, 27, 2),
            _ => (1, 2, 6, 0),
        }
    }

    pub fn screen_capture_allowed() -> bool {
        unsafe { CGPreflightScreenCaptureAccess() || CGRequestScreenCaptureAccess() }
    }

    pub fn screen_points() -> (f64, f64) {
        let r = unsafe { CGDisplayBounds(CGMainDisplayID()) };
        (r.size.width, r.size.height)
    }

    pub fn move_to(x: f64, y: f64) {
        let src = Source::new();
        post(unsafe { CGEventCreateMouseEvent(src.0, MOVED, CGPoint { x, y }, 0) }, 0);
    }

    /// CUActuator.click: press, hold 0.03 s, release; 0.08 s between the clicks of a multi-click.
    pub fn click(x: f64, y: f64, button: &str, count: i64, flags: u64) {
        let (down, up, _, b) = kinds(button);
        let at = CGPoint { x, y };
        move_to(x, y);
        gap();
        let src = Source::new();
        for n in 1..=count {
            if n > 1 {
                sleep_secs(CLICK_REPEAT_GAP_SECS);
            }
            for (i, kind) in [down, up].into_iter().enumerate() {
                if i == 1 {
                    sleep_secs(CLICK_HOLD_SECS);
                }
                let e = unsafe { CGEventCreateMouseEvent(src.0, kind, at, b) };
                if !e.is_null() {
                    unsafe { CGEventSetIntegerValueField(e, CLICK_STATE, n) };
                }
                post(e, flags);
            }
        }
    }

    /// CURemoteEventPoster.drag(path:button:steps:holdSeconds:): press at the first point (click
    /// state 1), hold, then walk each segment in `DRAG_STEPS` interpolated dragged events, then
    /// release at the last point.
    pub fn drag(path: &[(f64, f64)], button: &str, flags: u64) {
        let (down, up, dragged, b) = kinds(button);
        let (x0, y0) = path[0];
        move_to(x0, y0);
        gap();
        let src = Source::new();
        let event = |kind: u32, x: f64, y: f64| {
            let e = unsafe { CGEventCreateMouseEvent(src.0, kind, CGPoint { x, y }, b) };
            if !e.is_null() {
                unsafe { CGEventSetIntegerValueField(e, CLICK_STATE, 1) };
            }
            post(e, flags);
        };
        event(down, x0, y0);
        sleep_secs(DRAG_HOLD_SECS);
        for (x, y) in drag_points(path) {
            gap();
            event(dragged, x, y);
        }
        let (xn, yn) = path[path.len() - 1];
        event(up, xn, yn);
    }

    /// handleScroll closure @0x10009e998: a mouse-moved event at the spot (the pointer's own
    /// position when no x/y), then one pixel-unit scroll event with 2 wheels at that spot.
    pub fn scroll(at: Option<(f64, f64)>, dy: i32, dx: i32, flags: u64) {
        if let Some((x, y)) = at {
            move_to(x, y);
            gap();
        }
        let src = Source::new();
        post(unsafe { CGEventCreateScrollWheelEvent(src.0, SCROLL_PIXEL, 2, dy, dx) }, flags);
    }

    /// Type text so that any app takes it, whatever the keyboard layout or input method: paste
    /// it (⌘V) and put the clipboard back. Text sent as key events (the original's
    /// CUTypedCharacters, key 128 carrying the characters) went into the input method instead
    /// when one was on — with 注音 every letter vanished while return still arrived, so a
    /// command typed into a terminal ran empty. Falls back to those key events without a
    /// clipboard.
    pub fn type_text(text: &str) {
        if !paste(text) {
            type_as_keys(text);
        }
    }

    /// Wait for the app to read the clipboard before it is put back (Electron apps read it a
    /// moment after ⌘V; estimated).
    const PASTE_READ_MS: u64 = 350;

    fn paste(text: &str) -> bool {
        let Ok(mut cb) = arboard::Clipboard::new() else { return false };
        // What the person had there comes back afterwards (text or a picture).
        let saved_text = cb.get_text().ok();
        let saved_image = if saved_text.is_none() { cb.get_image().ok() } else { None };
        if cb.set_text(text.to_string()).is_err() {
            return false;
        }
        std::thread::sleep(Duration::from_millis(40));
        press(9, FLAG_CMD, KEY_HOLD_SECS);
        std::thread::sleep(Duration::from_millis(PASTE_READ_MS));
        match (saved_text, saved_image) {
            (Some(t), _) => {
                let _ = cb.set_text(t);
            }
            (None, Some(i)) => {
                let _ = cb.set_image(i);
            }
            _ => {
                let _ = cb.clear();
            }
        }
        true
    }

    /// CUTypedCharacters.postChunks: each chunk on key 128, down, 0.035 s, up; 0.02 s between chunks.
    fn type_as_keys(text: &str) {
        let src = Source::new();
        let chunks = utf16_chunks(text);
        for (i, chunk) in chunks.iter().enumerate() {
            if i > 0 {
                sleep_secs(KEY_GAP_SECS);
            }
            for down in [true, false] {
                if !down {
                    sleep_secs(KEY_HOLD_SECS);
                }
                let e = unsafe { CGEventCreateKeyboardEvent(src.0, TYPE_KEY_CODE, down) };
                if !e.is_null() {
                    unsafe { CGEventKeyboardSetUnicodeString(e, chunk.len(), chunk.as_ptr()) };
                }
                post(e, 0);
            }
        }
    }

    /// handlePressKey: key down, hold (0.035 s unless told), key up.
    pub fn press(code: u16, flags: u64, hold_secs: f64) {
        let src = Source::new();
        post(unsafe { CGEventCreateKeyboardEvent(src.0, code, true) }, flags);
        sleep_secs(hold_secs);
        post(unsafe { CGEventCreateKeyboardEvent(src.0, code, false) }, flags);
    }
}


#[cfg(not(any(target_os = "macos", target_os = "linux")))]
mod mac {
    pub fn screen_capture_allowed() -> bool {
        false
    }
    pub fn screen_points() -> (f64, f64) {
        (0.0, 0.0)
    }
    pub fn move_to(_: f64, _: f64) {}
    pub fn click(_: f64, _: f64, _: &str, _: i64, _: u64) {}
    pub fn drag(_: &[(f64, f64)], _: &str, _: u64) {}
    pub fn scroll(_: Option<(f64, f64)>, _: i32, _: i32, _: u64) {}
    pub fn type_text(_: &str) {}
    pub fn press(_: u16, _: u64, _: f64) {}
    pub fn window_at(_: f64, _: f64) -> Option<(String, i32)> {
        None
    }
    pub fn front_app() -> Option<(String, i32)> {
        None
    }
    pub fn screen_capture_preflight() -> bool {
        false
    }
    pub fn screen_pixels() -> (u32, u32) {
        (0, 0)
    }
    pub fn mouse_button(_: Option<(f64, f64)>, _: &str, _: bool) {}
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn screenshots_fit_like_the_original() {
        assert_eq!(fit(2940, 1912), (1024, 666, true));
        assert_eq!(fit(1024, 640), (1024, 640, false));
        assert_eq!(fit(800, 600), (800, 600, false));
        assert_eq!(fit(4000, 7), (4000, 7, false));
        // A very wide strip keeps its short edge at 8.
        assert_eq!(fit(4000, 10), (3200, 8, true));
    }

    #[test]
    fn jpeg_stays_under_the_cap() {
        let img = image::RgbImage::from_fn(1024, 640, |x, y| image::Rgb([(x % 256) as u8, (y % 256) as u8, ((x ^ y) % 256) as u8]));
        let j = encode_jpeg(&img).unwrap();
        assert!(j.len() <= MAX_IMAGE_BYTES && j.starts_with(&[0xFF, 0xD8]));
    }

    #[test]
    fn shortcuts_are_parsed() {
        assert_eq!(parse_combo("cmd+c").unwrap(), (8, FLAG_CMD));
        assert_eq!(parse_combo("cmd+shift+t").unwrap(), (17, FLAG_CMD | FLAG_SHIFT));
        assert_eq!(parse_combo("Return").unwrap(), (36, 0));
        assert_eq!(parse_combo("ctrl+option+space").unwrap(), (49, FLAG_CONTROL | FLAG_OPTION));
        assert!(parse_combo("hyper+x").is_err());
        assert!(parse_combo("").is_err());
    }

    #[test]
    fn typed_text_is_chunked_like_the_original() {
        let c = utf16_chunks(&"a".repeat(45));
        assert_eq!(c.iter().map(Vec::len).collect::<Vec<_>>(), vec![20, 20, 5]);
        // An emoji (2 UTF-16 units) is never split across chunks.
        let c = utf16_chunks(&format!("{}😀b", "a".repeat(19)));
        assert_eq!(c.iter().map(Vec::len).collect::<Vec<_>>(), vec![19, 3]);
        assert_eq!(utf16_chunks("你好").len(), 1);
        assert!(utf16_chunks("").is_empty());
    }

    #[test]
    fn scrolls_like_the_original() {
        assert_eq!(scroll_amount(None), 3);
        assert_eq!(scroll_amount(Some(0)), 1);
        assert_eq!(scroll_amount(Some(99)), 20);
        assert_eq!(scroll_pixels("down", 3), Some((-48, 0)));
        assert_eq!(scroll_pixels("left", 2), Some((0, 32)));
        assert_eq!(scroll_pixels("sideways", 1), None);
    }

    #[test]
    fn drags_walk_six_steps_per_segment() {
        let pts = drag_points(&[(0.0, 0.0), (60.0, 0.0), (60.0, 30.0)]);
        assert_eq!(pts.len(), 2 * DRAG_STEPS);
        assert_eq!(pts[0], (10.0, 0.0));
        assert_eq!(pts[DRAG_STEPS - 1], (60.0, 0.0));
        assert_eq!(*pts.last().unwrap(), (60.0, 30.0));
    }

    #[test]
    fn a_press_can_be_several_keys() {
        assert_eq!(key_plan("cmd+a delete").unwrap(), vec![Key::Combo(0, FLAG_CMD), Key::Combo(51, 0)]);
        assert!(key_plan("  ").is_err());
        // Keys by the names models use, and characters without a key typed instead.
        assert_eq!(key_plan("cmd+grave").unwrap(), vec![Key::Combo(50, FLAG_CMD)]);
        // A printing key alone is its character (typed, so 注音 can't swallow it).
        assert_eq!(key_plan("slash").unwrap(), vec![Key::Char("/".into())]);
        assert_eq!(key_plan("/ @ r return").unwrap(), vec![Key::Char("/".into()), Key::Char("@".into()), Key::Char("r".into()), Key::Combo(36, 0)]);
        assert!(key_plan("hyper+x").is_err());
    }

    #[test]
    fn pictures_and_clicks_share_one_size() {
        // The picture's size depends only on the screen, so a tap or a click maps the same
        // whoever saw which screenshot last (a 2940×1912 Retina screen of 1470×956 points).
        let (w, h, _) = fit(2940, 1912);
        let s = (1470.0 / w as f64, 956.0 / h as f64);
        let (x, y) = to_screen(512.0, 333.0, s);
        assert!((x - 735.0).abs() < 0.01 && (y - 478.0).abs() < 1.0);
    }

    #[test]
    fn screenshot_pixels_map_to_screen_points() {
        // A 1470-point-wide screen sent as a 1024-wide picture.
        let s = (1470.0 / 1024.0, 956.0 / 666.0);
        let (x, y) = to_screen(512.0, 333.0, s);
        assert!((x - 735.0).abs() < 0.01 && (y - 478.0).abs() < 0.01);
    }
}

#[cfg(test)]
mod window_tests {
    /// `cargo test --release -- --ignored finds_apps_on_screen --nocapture`
    #[test]
    #[ignore]
    fn finds_apps_on_screen() {
        let front = super::frontmost_app();
        println!("front: {front:?}");
        println!("at (200,200): {:?}", super::app_at_point(200.0, 200.0));
        println!("points→shot: {}", super::points_to_shot());
        assert!(front.is_some());
    }
}