AnyBot/src/win.rs

//! Windows hands and eyes (C9): SendInput for mouse and keyboard, GDI BitBlt for the screen,
//! Win32 for "what is under that point". The control tree comes from agent-desktop-windows,
//! which speaks the same elements() language as macOS's accessibility here.
#![cfg(target_os = "windows")]
use windows::Win32::Foundation::{CloseHandle, HWND, POINT, RECT};
use windows::Win32::Graphics::Gdi::{
    BitBlt, CreateCompatibleBitmap, CreateCompatibleDC, DeleteDC, DeleteObject, GetDC, GetDIBits, ReleaseDC, SelectObject, BITMAPINFO, BITMAPINFOHEADER, BI_RGB,
    DIB_RGB_COLORS, SRCCOPY,
};
use windows::Win32::UI::Input::KeyboardAndMouse::{
    SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, INPUT_MOUSE, KEYBDINPUT, KEYEVENTF_KEYUP, KEYEVENTF_UNICODE, KEYBD_EVENT_FLAGS, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP,
    MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_MOVE, MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_WHEEL, MOUSEINPUT, MOUSE_EVENT_FLAGS, VIRTUAL_KEY,
};
use windows::Win32::UI::WindowsAndMessaging::{
    GetForegroundWindow, GetSystemMetrics, GetWindowRect, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId, SM_CXSCREEN, SM_CYSCREEN, WindowFromPoint,
};
use windows::Win32::System::Threading::{OpenProcess, QueryFullProcessImageNameW, PROCESS_NAME_FORMAT, PROCESS_QUERY_LIMITED_INFORMATION};

fn mouse(dx: i32, dy: i32, data: u32, flags: MOUSE_EVENT_FLAGS) {
    let mi = MOUSEINPUT { dx, dy, mouseData: data, dwFlags: flags, time: 0, dwExtraInfo: 0 };
    let inp = INPUT { r#type: INPUT_MOUSE, Anonymous: INPUT_0 { mi } };
    let _ = unsafe { SendInput(&[inp], std::mem::size_of::<INPUT>() as i32) };
}

/// Windows wants the whole virtual desktop in 0..65535.
fn norm(x: f64, y: f64) -> (i32, i32) {
    // single-monitor first: the app's own metrics (multi-monitor virtual bounds later)
    let w = unsafe { GetSystemMetrics(SM_CXSCREEN) }.max(1) as f64;
    let h = unsafe { GetSystemMetrics(SM_CYSCREEN) }.max(1) as f64;
    (((x / w) * 65535.0).round() as i32, ((y / h) * 65535.0).round() as i32)
}

fn keys(vk: u16, up: bool, unicode: bool, ch: u16) {
    let ki = KEYBDINPUT {
        wVk: VIRTUAL_KEY(if unicode { 0 } else { vk }),
        wScan: ch,
        dwFlags: if unicode { KEYEVENTF_UNICODE } else { if up { KEYEVENTF_KEYUP } else { KEYBD_EVENT_FLAGS(0) } },
        time: 0,
        dwExtraInfo: 0,
    };
    let inp = INPUT { r#type: INPUT_KEYBOARD, Anonymous: INPUT_0 { ki } };
    let _ = unsafe { SendInput(&[inp], std::mem::size_of::<INPUT>() as i32) };
}

/// macOS key codes (what key_code() hands us) → Windows virtual keys. 0 = no Windows key for it.
fn vk_of(code: u16) -> u16 {
    static VK_TABLE: &[(u16, u16)] = &[
        (0, 0x41), (11, 0x42), (8, 0x43), (2, 0x44), (14, 0x45), (3, 0x46), (5, 0x47), (4, 0x48), (34, 0x49), (38, 0x4A), (40, 0x4B), (37, 0x4C), (46, 0x4D), (45, 0x4E),
        (31, 0x4F), (35, 0x50), (12, 0x51), (15, 0x52), (1, 0x53), (17, 0x54), (32, 0x55), (9, 0x56), (13, 0x57), (7, 0x58), (16, 0x59), (6, 0x5A),
        (18, 0x31), (19, 0x32), (20, 0x33), (21, 0x34), (23, 0x35), (22, 0x36), (26, 0x37), (28, 0x38), (25, 0x39), (29, 0x30),
        (36, 0x0D), (76, 0x03), (48, 0x09), (49, 0x20), (51, 0x08), (53, 0x1B), (24, 0xBB), (27, 0xBD), (33, 0xDB), (30, 0xDD), (41, 0xBA), (39, 0xDE), (43, 0xBC),
        (47, 0xBE), (44, 0xBF), (50, 0xC0), (42, 0xDC), (52, 0xE7), (36, 0x0D), (123, 0x25), (124, 0x27), (126, 0x26), (125, 0x28), (115, 0x24), (119, 0x23), (116, 0x21),
        (121, 0x22), (117, 0x2E), (122, 0x70), (120, 0x71), (99, 0x72), (118, 0x73), (96, 0x74), (97, 0x75), (98, 0x76),
    ];
    VK_TABLE.iter().find(|(c, _)| *c == code).map(|(_, v)| *v).unwrap_or(0)
}

fn modifiers(flags: u64) -> (Vec<u16>, Vec<u16>) {
    let mut down = vec![];
    if flags & 0x20000 != 0 {
        down.push(0x10); // shift
    }
    if flags & 0x40000 != 0 {
        down.push(0x11); // ctrl
    }
    if flags & 0x80000 != 0 {
        down.push(0x12); // alt
    }
    if flags & 0x100000 != 0 {
        down.push(0x5B); // win
    }
    let up: Vec<u16> = down.iter().rev().cloned().collect();
    (down, up)
}

pub fn move_to(x: f64, y: f64) {
    let (nx, ny) = norm(x, y);
    mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE);
}

pub fn click(x: f64, y: f64, button: &str, count: i64, flags: u64) {
    let (down, up) = match button {
        "right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP),
        "middle" => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP),
        _ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP),
    };
    let _ = flags;
    let (nx, ny) = norm(x, y);
    mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE);
    for n in 1..=count.max(1) {
        if n > 1 {
            std::thread::sleep(std::time::Duration::from_millis(80));
        }
        mouse(nx, ny, 0, down);
        std::thread::sleep(std::time::Duration::from_millis(30));
        mouse(nx, ny, 0, up);
    }
}

pub fn drag(points: &[(f64, f64)], button: &str, flags: u64) {
    let (down, up) = match button {
        "right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP),
        _ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP),
    };
    let _ = flags;
    if let Some((x, y)) = points.first() {
        let (nx, ny) = norm(*x, *y);
        mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE);
        mouse(nx, ny, 0, down);
    }
    for (x, y) in points.iter().skip(1) {
        let (nx, ny) = norm(*x, *y);
        mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE);
        std::thread::sleep(std::time::Duration::from_millis(20));
    }
    if let Some((x, y)) = points.last() {
        let (nx, ny) = norm(*x, *y);
        mouse(nx, ny, 0, up);
    }
}

pub fn scroll(at: Option<(f64, f64)>, x: i32, y: i32, flags: u64) {
    let _ = (x, flags);
    if let Some((px, py)) = at {
        move_to(px, py);
    }
    // Windows wheels by 120ths of a line; the harness counts lines.
    mouse(0, 0, ((y * 120) as i32) as u32, MOUSEEVENTF_WHEEL);
}

pub fn type_text(text: &str) {
    for u in text.encode_utf16() {
        keys(0, false, true, u);
        keys(0, true, true, u);
    }
}

pub fn press(code: u16, flags: u64, _hold: f64) {
    let vk = vk_of(code);
    if vk == 0 {
        return;
    }
    let (down, up) = modifiers(flags);
    for k in &down {
        keys(*k, false, false, 0);
    }
    keys(vk, false, false, 0);
    std::thread::sleep(std::time::Duration::from_millis(30));
    keys(vk, true, false, 0);
    for k in &up {
        keys(*k, true, false, 0);
    }
}

fn window_text(h: HWND) -> String {
    let n = unsafe { GetWindowTextLengthW(h) }.max(0) as usize;
    if n == 0 {
        return String::new();
    }
    let mut buf = vec![0u16; n + 1];
    let got = unsafe { GetWindowTextW(h, &mut buf) } as usize;
    String::from_utf16_lossy(&buf[..got])
}

fn process_name(pid: u32) -> String {
    unsafe {
        let Ok(h) = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid) else { return String::new() };
        let mut buf = [0u16; 260];
        let mut len = buf.len() as u32;
        let ok = QueryFullProcessImageNameW(h, PROCESS_NAME_FORMAT(0), windows::core::PWSTR(buf.as_mut_ptr()), &mut len).is_ok();
        let _ = CloseHandle(h);
        if !ok {
            return String::new();
        }
        let full = String::from_utf16_lossy(&buf[..len as usize]);
        std::path::Path::new(&full).file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or(full)
    }
}

fn rect(h: HWND) -> (i32, i32, i32, i32) {
    let mut r = RECT::default();
    let _ = unsafe { GetWindowRect(h, &mut r) };
    (r.left, r.top, r.right, r.bottom)
}

pub fn window_at(x: f64, y: f64) -> Option<(String, i32)> {
    let h = unsafe { WindowFromPoint(POINT { x: x as i32, y: y as i32 }) };
    if h.is_invalid() {
        return None;
    }
    let mut pid: u32 = 0;
    unsafe { GetWindowThreadProcessId(h, Some(&mut pid)) };
    let t = window_text(h);
    let name = if t.is_empty() { process_name(pid) } else { t };
    Some((name, pid as i32))
}

pub fn front_app() -> Option<(String, i32)> {
    let h = unsafe { GetForegroundWindow() };
    if h.is_invalid() {
        return None;
    }
    let mut pid: u32 = 0;
    unsafe { GetWindowThreadProcessId(h, Some(&mut pid)) };
    let t = window_text(h);
    let name = if t.is_empty() { process_name(pid) } else { t };
    Some((name, pid as i32))
}

pub fn screen_capture_allowed() -> bool {
    true
}

pub fn screen_capture_preflight() -> bool {
    true
}

pub fn screen_points() -> (f64, f64) {
    (unsafe { GetSystemMetrics(SM_CXSCREEN) } as f64, unsafe { GetSystemMetrics(SM_CYSCREEN) } as f64)
}

pub fn screen_pixels() -> (u32, u32) {
    let (w, h) = screen_points();
    (w as u32, h as u32)
}

/// The whole main screen, top-down, as pixels (BI_RGB 32-bit → RGB8).
pub fn capture() -> Result<image::RgbImage, String> {
    let w = screen_points().0 as i32;
    let h = screen_points().1 as i32;
    if w <= 0 || h <= 0 {
        return Err("看不到畫面".into());
    }
    unsafe {
        let hdc = GetDC(None);
        let mem = CreateCompatibleDC(Some(hdc));
        let bmp = CreateCompatibleBitmap(hdc, w, h);
        let old = SelectObject(mem, bmp.into());
        let _ = BitBlt(mem, 0, 0, w, h, Some(hdc), 0, 0, SRCCOPY);
        let mut header = BITMAPINFO {
            bmiHeader: BITMAPINFOHEADER {
                biSize: std::mem::size_of::<BITMAPINFOHEADER>() as u32,
                biWidth: w,
                biHeight: -h, // top-down
                biPlanes: 1,
                biBitCount: 32,
                biCompression: BI_RGB.0 as u32,
                ..Default::default()
            },
            ..Default::default()
        };
        let mut pixels = vec![0u8; (w as usize) * (h as usize) * 4];
        let rows = GetDIBits(mem, bmp, 0, h as u32, Some(pixels.as_mut_ptr() as *mut _), &mut header, DIB_RGB_COLORS);
        let _ = SelectObject(mem, old);
        let _ = DeleteObject(bmp.into());
        let _ = DeleteDC(mem);
        ReleaseDC(None, hdc);
        if rows == 0 {
            return Err("截圖失敗".into());
        }
        let mut img = image::RgbImage::new(w as u32, h as u32);
        for y in 0..h as usize {
            for x in 0..w as usize {
                let i = (y * w as usize + x) * 4;
                img.put_pixel(x as u32, y as u32, image::Rgb([pixels[i + 2], pixels[i + 1], pixels[i]]));
            }
        }
        Ok(img)
    }
}