//! Windows hands and eyes (C9): SendInput for mouse and keyboard, GDI BitBlt for the screen, //! Win32 for "what is under that point". The control tree comes from agent-desktop-windows, //! which speaks the same elements() language as macOS's accessibility here. #![cfg(target_os = "windows")] use windows::Win32::Foundation::{CloseHandle, HWND, POINT, RECT}; use windows::Win32::Graphics::Gdi::{ BitBlt, CreateCompatibleBitmap, CreateCompatibleDC, DeleteDC, DeleteObject, GetDC, GetDIBits, ReleaseDC, SelectObject, BITMAPINFO, BITMAPINFOHEADER, BI_RGB, DIB_RGB_COLORS, SRCCOPY, }; use windows::Win32::UI::Input::KeyboardAndMouse::{ SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, INPUT_MOUSE, KEYBDINPUT, KEYEVENTF_KEYUP, KEYEVENTF_UNICODE, KEYBD_EVENT_FLAGS, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP, MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_MOVE, MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_WHEEL, MOUSEINPUT, MOUSE_EVENT_FLAGS, VIRTUAL_KEY, }; use windows::Win32::UI::WindowsAndMessaging::{ GetForegroundWindow, GetSystemMetrics, GetWindowRect, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId, SM_CXSCREEN, SM_CYSCREEN, WindowFromPoint, }; use windows::Win32::System::Threading::{OpenProcess, QueryFullProcessImageNameW, PROCESS_NAME_FORMAT, PROCESS_QUERY_LIMITED_INFORMATION}; fn mouse(dx: i32, dy: i32, data: u32, flags: MOUSE_EVENT_FLAGS) { let mi = MOUSEINPUT { dx, dy, mouseData: data, dwFlags: flags, time: 0, dwExtraInfo: 0 }; let inp = INPUT { r#type: INPUT_MOUSE, Anonymous: INPUT_0 { mi } }; let _ = unsafe { SendInput(&[inp], std::mem::size_of::() as i32) }; } /// Windows wants the whole virtual desktop in 0..65535. fn norm(x: f64, y: f64) -> (i32, i32) { // single-monitor first: the app's own metrics (multi-monitor virtual bounds later) let w = unsafe { GetSystemMetrics(SM_CXSCREEN) }.max(1) as f64; let h = unsafe { GetSystemMetrics(SM_CYSCREEN) }.max(1) as f64; (((x / w) * 65535.0).round() as i32, ((y / h) * 65535.0).round() as i32) } fn keys(vk: u16, up: bool, unicode: bool, ch: u16) { let ki = KEYBDINPUT { wVk: VIRTUAL_KEY(if unicode { 0 } else { vk }), wScan: ch, dwFlags: if unicode { KEYEVENTF_UNICODE } else { if up { KEYEVENTF_KEYUP } else { KEYBD_EVENT_FLAGS(0) } }, time: 0, dwExtraInfo: 0, }; let inp = INPUT { r#type: INPUT_KEYBOARD, Anonymous: INPUT_0 { ki } }; let _ = unsafe { SendInput(&[inp], std::mem::size_of::() as i32) }; } /// macOS key codes (what key_code() hands us) → Windows virtual keys. 0 = no Windows key for it. fn vk_of(code: u16) -> u16 { static VK_TABLE: &[(u16, u16)] = &[ (0, 0x41), (11, 0x42), (8, 0x43), (2, 0x44), (14, 0x45), (3, 0x46), (5, 0x47), (4, 0x48), (34, 0x49), (38, 0x4A), (40, 0x4B), (37, 0x4C), (46, 0x4D), (45, 0x4E), (31, 0x4F), (35, 0x50), (12, 0x51), (15, 0x52), (1, 0x53), (17, 0x54), (32, 0x55), (9, 0x56), (13, 0x57), (7, 0x58), (16, 0x59), (6, 0x5A), (18, 0x31), (19, 0x32), (20, 0x33), (21, 0x34), (23, 0x35), (22, 0x36), (26, 0x37), (28, 0x38), (25, 0x39), (29, 0x30), (36, 0x0D), (76, 0x03), (48, 0x09), (49, 0x20), (51, 0x08), (53, 0x1B), (24, 0xBB), (27, 0xBD), (33, 0xDB), (30, 0xDD), (41, 0xBA), (39, 0xDE), (43, 0xBC), (47, 0xBE), (44, 0xBF), (50, 0xC0), (42, 0xDC), (52, 0xE7), (36, 0x0D), (123, 0x25), (124, 0x27), (126, 0x26), (125, 0x28), (115, 0x24), (119, 0x23), (116, 0x21), (121, 0x22), (117, 0x2E), (122, 0x70), (120, 0x71), (99, 0x72), (118, 0x73), (96, 0x74), (97, 0x75), (98, 0x76), ]; VK_TABLE.iter().find(|(c, _)| *c == code).map(|(_, v)| *v).unwrap_or(0) } fn modifiers(flags: u64) -> (Vec, Vec) { let mut down = vec![]; if flags & 0x20000 != 0 { down.push(0x10); // shift } if flags & 0x40000 != 0 { down.push(0x11); // ctrl } if flags & 0x80000 != 0 { down.push(0x12); // alt } if flags & 0x100000 != 0 { down.push(0x5B); // win } let up: Vec = down.iter().rev().cloned().collect(); (down, up) } pub fn move_to(x: f64, y: f64) { let (nx, ny) = norm(x, y); mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE); } pub fn click(x: f64, y: f64, button: &str, count: i64, flags: u64) { let (down, up) = match button { "right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP), "middle" => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP), _ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP), }; let _ = flags; let (nx, ny) = norm(x, y); mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE); for n in 1..=count.max(1) { if n > 1 { std::thread::sleep(std::time::Duration::from_millis(80)); } mouse(nx, ny, 0, down); std::thread::sleep(std::time::Duration::from_millis(30)); mouse(nx, ny, 0, up); } } pub fn drag(points: &[(f64, f64)], button: &str, flags: u64) { let (down, up) = match button { "right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP), _ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP), }; let _ = flags; if let Some((x, y)) = points.first() { let (nx, ny) = norm(*x, *y); mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE); mouse(nx, ny, 0, down); } for (x, y) in points.iter().skip(1) { let (nx, ny) = norm(*x, *y); mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE); std::thread::sleep(std::time::Duration::from_millis(20)); } if let Some((x, y)) = points.last() { let (nx, ny) = norm(*x, *y); mouse(nx, ny, 0, up); } } pub fn scroll(at: Option<(f64, f64)>, x: i32, y: i32, flags: u64) { let _ = (x, flags); if let Some((px, py)) = at { move_to(px, py); } // Windows wheels by 120ths of a line; the harness counts lines. mouse(0, 0, ((y * 120) as i32) as u32, MOUSEEVENTF_WHEEL); } pub fn type_text(text: &str) { for u in text.encode_utf16() { keys(0, false, true, u); keys(0, true, true, u); } } pub fn press(code: u16, flags: u64, _hold: f64) { let vk = vk_of(code); if vk == 0 { return; } let (down, up) = modifiers(flags); for k in &down { keys(*k, false, false, 0); } keys(vk, false, false, 0); std::thread::sleep(std::time::Duration::from_millis(30)); keys(vk, true, false, 0); for k in &up { keys(*k, true, false, 0); } } fn window_text(h: HWND) -> String { let n = unsafe { GetWindowTextLengthW(h) }.max(0) as usize; if n == 0 { return String::new(); } let mut buf = vec![0u16; n + 1]; let got = unsafe { GetWindowTextW(h, &mut buf) } as usize; String::from_utf16_lossy(&buf[..got]) } fn process_name(pid: u32) -> String { unsafe { let Ok(h) = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid) else { return String::new() }; let mut buf = [0u16; 260]; let mut len = buf.len() as u32; let ok = QueryFullProcessImageNameW(h, PROCESS_NAME_FORMAT(0), windows::core::PWSTR(buf.as_mut_ptr()), &mut len).is_ok(); let _ = CloseHandle(h); if !ok { return String::new(); } let full = String::from_utf16_lossy(&buf[..len as usize]); std::path::Path::new(&full).file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or(full) } } fn rect(h: HWND) -> (i32, i32, i32, i32) { let mut r = RECT::default(); let _ = unsafe { GetWindowRect(h, &mut r) }; (r.left, r.top, r.right, r.bottom) } pub fn window_at(x: f64, y: f64) -> Option<(String, i32)> { let h = unsafe { WindowFromPoint(POINT { x: x as i32, y: y as i32 }) }; if h.is_invalid() { return None; } let mut pid: u32 = 0; unsafe { GetWindowThreadProcessId(h, Some(&mut pid)) }; let t = window_text(h); let name = if t.is_empty() { process_name(pid) } else { t }; Some((name, pid as i32)) } pub fn front_app() -> Option<(String, i32)> { let h = unsafe { GetForegroundWindow() }; if h.is_invalid() { return None; } let mut pid: u32 = 0; unsafe { GetWindowThreadProcessId(h, Some(&mut pid)) }; let t = window_text(h); let name = if t.is_empty() { process_name(pid) } else { t }; Some((name, pid as i32)) } pub fn screen_capture_allowed() -> bool { true } pub fn screen_capture_preflight() -> bool { true } pub fn screen_points() -> (f64, f64) { (unsafe { GetSystemMetrics(SM_CXSCREEN) } as f64, unsafe { GetSystemMetrics(SM_CYSCREEN) } as f64) } pub fn screen_pixels() -> (u32, u32) { let (w, h) = screen_points(); (w as u32, h as u32) } /// The whole main screen, top-down, as pixels (BI_RGB 32-bit → RGB8). pub fn capture() -> Result { let w = screen_points().0 as i32; let h = screen_points().1 as i32; if w <= 0 || h <= 0 { return Err("看不到畫面".into()); } unsafe { let hdc = GetDC(None); let mem = CreateCompatibleDC(Some(hdc)); let bmp = CreateCompatibleBitmap(hdc, w, h); let old = SelectObject(mem, bmp.into()); let _ = BitBlt(mem, 0, 0, w, h, Some(hdc), 0, 0, SRCCOPY); let mut header = BITMAPINFO { bmiHeader: BITMAPINFOHEADER { biSize: std::mem::size_of::() as u32, biWidth: w, biHeight: -h, // top-down biPlanes: 1, biBitCount: 32, biCompression: BI_RGB.0 as u32, ..Default::default() }, ..Default::default() }; let mut pixels = vec![0u8; (w as usize) * (h as usize) * 4]; let rows = GetDIBits(mem, bmp, 0, h as u32, Some(pixels.as_mut_ptr() as *mut _), &mut header, DIB_RGB_COLORS); let _ = SelectObject(mem, old); let _ = DeleteObject(bmp.into()); let _ = DeleteDC(mem); ReleaseDC(None, hdc); if rows == 0 { return Err("截圖失敗".into()); } let mut img = image::RgbImage::new(w as u32, h as u32); for y in 0..h as usize { for x in 0..w as usize { let i = (y * w as usize + x) * 4; img.put_pixel(x as u32, y as u32, image::Rgb([pixels[i + 2], pixels[i + 1], pixels[i]])); } } Ok(img) } }