From f829957b75a726e23762290216228f288b54f167 Mon Sep 17 00:00:00 2001 From: AnyBot Dev Date: Mon, 9 Nov 2026 20:50:44 +0800 Subject: [PATCH] Windows has hands now (C9, engine) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - src/win.rs: the same mac:: API the harness drives, on Win32 — SendInput mouse (absolute, multi-click, drag, wheel), keyboard (macOS key codes mapped to VK, unicode text, modifiers), GDI BitBlt screenshot, WindowFromPoint/GetForegroundWindow for window_at and front_app; compiles clean against the windows crate for x86_64-pc-windows-msvc (API-checked, no hardware here) - the control tree was already speaking agent-desktop-windows - intents answer honestly on Windows ("捷徑目前是 macOS/Linux 的") and don't advertise themselves from the facts line - CI: a report-only Windows compile job for wherever a Windows runner lives - needs a real machine: multi-monitor coordinates, high-DPI, UIA-number clicks, Chinese input --- .gitea/workflows/ci.yml | 9 + Cargo.lock | 162 ++++++++++++++---- Cargo.toml | 1 + docs/plan-2026-11-improvements.md | 2 +- src/computer.rs | 8 +- src/intents.rs | 4 + src/main.rs | 2 + src/win.rs | 270 ++++++++++++++++++++++++++++++ 8 files changed, 420 insertions(+), 38 deletions(-) create mode 100644 src/win.rs diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 454d024..74cc80d 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -31,3 +31,12 @@ jobs: run: cargo clippy --bin anybot 2>&1 | tail -30 - name: Relay build and test run: cd relay && cargo test + + windows-check: + # C9: the Windows engine is real code now; where a Windows runner exists, it must compile. + # Report-only until the release ships Windows binaries. + runs-on: windows + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - run: cargo check --bin anybot diff --git a/Cargo.lock b/Cargo.lock index 414d66a..e9df6ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,8 +116,8 @@ dependencies = [ "accesskit_consumer", "hashbrown 0.17.1", "static_assertions", - "windows", - "windows-core", + "windows 0.62.2", + "windows-core 0.62.2", ] [[package]] @@ -225,7 +225,7 @@ dependencies = [ "thiserror 2.0.21", "tracing", "uiautomation", - "windows", + "windows 0.62.2", "windows-sys 0.61.2", ] @@ -376,6 +376,7 @@ dependencies = [ "wayland-protocols-misc", "wayland-protocols-wlr", "webpki-root-certs", + "windows 0.61.3", "x11rb 0.13.2", "zbus", "zhconv", @@ -965,7 +966,7 @@ dependencies = [ "js-sys", "num-traits", "wasm-bindgen", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -1234,8 +1235,8 @@ dependencies = [ "objc2-core-foundation", "objc2-foundation 0.3.2", "web-sys", - "windows", - "windows-core", + "windows 0.62.2", + "windows-core 0.62.2", ] [[package]] @@ -1960,8 +1961,8 @@ dependencies = [ "read-fonts 0.41.0", "roxmltree 0.21.2", "smallvec", - "windows", - "windows-core", + "windows 0.62.2", + "windows-core 0.62.2", "yeslogic-fontconfig-sys", ] @@ -2075,7 +2076,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ "rustix 1.1.5", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -2259,7 +2260,7 @@ dependencies = [ "log", "presser", "thiserror 2.0.21", - "windows", + "windows 0.62.2", ] [[package]] @@ -2501,7 +2502,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core", + "windows-core 0.62.2", ] [[package]] @@ -2710,7 +2711,7 @@ dependencies = [ "simd_cesu8", "thiserror 2.0.21", "walkdir", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -2833,7 +2834,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -3690,7 +3691,7 @@ dependencies = [ "libc", "redox_syscall 0.5.18", "smallvec", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -5460,8 +5461,8 @@ checksum = "fa2eacf9390a8491bc949b04230f40c12520c6423a2962a6087eea1ef14373af" dependencies = [ "chrono", "uiautomation_derive", - "windows", - "windows-core", + "windows 0.62.2", + "windows-core 0.62.2", ] [[package]] @@ -6157,9 +6158,9 @@ dependencies = [ "web-sys", "wgpu-naga-bridge", "wgpu-types", - "windows", - "windows-core", - "windows-result", + "windows 0.62.2", + "windows-core 0.62.2", + "windows-result 0.4.1", ] [[package]] @@ -6219,16 +6220,38 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections 0.2.0", + "windows-core 0.61.2", + "windows-future 0.2.1", + "windows-link 0.1.3", + "windows-numerics 0.2.0", +] + [[package]] name = "windows" version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" dependencies = [ - "windows-collections", - "windows-core", - "windows-future", - "windows-numerics", + "windows-collections 0.3.2", + "windows-core 0.62.2", + "windows-future 0.3.2", + "windows-numerics 0.3.1", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core 0.61.2", ] [[package]] @@ -6237,7 +6260,20 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" dependencies = [ - "windows-core", + "windows-core 0.62.2", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", ] [[package]] @@ -6248,9 +6284,20 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link", - "windows-result", - "windows-strings", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", + "windows-threading 0.1.0", ] [[package]] @@ -6259,9 +6306,9 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" dependencies = [ - "windows-core", - "windows-link", - "windows-threading", + "windows-core 0.62.2", + "windows-link 0.2.1", + "windows-threading 0.2.1", ] [[package]] @@ -6286,20 +6333,45 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", +] + [[package]] name = "windows-numerics" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" dependencies = [ - "windows-core", - "windows-link", + "windows-core 0.62.2", + "windows-link 0.2.1", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", ] [[package]] @@ -6308,7 +6380,16 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link", + "windows-link 0.2.1", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", ] [[package]] @@ -6317,7 +6398,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -6335,7 +6416,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -6354,13 +6435,22 @@ dependencies = [ "windows_x86_64_msvc", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-threading" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 59e0fae..4111d6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,3 +85,4 @@ arboard = { version = "3.7", default-features = false } [target.'cfg(target_os = "windows")'.dependencies] agent-desktop-windows = { git = "https://github.com/vercel-labs/agent-desktop", tag = "v1.1.0" } +windows = { version = "0.61", features = ["Win32_Foundation", "Win32_Graphics_Gdi", "Win32_UI_Input_KeyboardAndMouse", "Win32_UI_WindowsAndMessaging", "Win32_System_Threading"] } diff --git a/docs/plan-2026-11-improvements.md b/docs/plan-2026-11-improvements.md index b09bcab..b6f72c1 100644 --- a/docs/plan-2026-11-improvements.md +++ b/docs/plan-2026-11-improvements.md @@ -161,7 +161,7 @@ - `teach` 示範錄影→`replay_skill` 已有（時區題 9 步 49s→4 步 15s）。加：重播失敗自動截圖交還模型、示範可直接錄人的真實操作（box 端事件記錄）、技能進 registry 可分享。 - ✅ 完成標準：使用者示範一次→第二次同任務 ≤5 步零模型。 -## C9. Windows 操控面（接 P8） +## C9. Windows 操控面（接 P8）— 🚧 2026-11：引擎已落地（`src/win.rs`：SendInput 滑鼠/鍵盤＋GDI 截圖＋前景視窗，mac code→VK 對照表；控制樹走 agent-desktop-windows；intents 在 Windows 明確降級為 GUI）。待實機驗證：多螢幕/高 DPI 座標、UIA 編號點擊、中文輸入。 - UIA（agent-desktop-windows）讀取＋`SendInput`；`x11.rs/wl.rs` 對應 `win.rs`；facts/意圖表 Windows 版。 - ✅ 完成標準：Windows 手腳可被大腦派畫面步（bench T2/T5 移植版）。 diff --git a/src/computer.rs b/src/computer.rs index cd4cba1..ad530f3 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -1538,6 +1538,8 @@ pub fn plain_picture() -> Result<(u32, u32, Vec), String> { pub fn capture_screen() -> Result { #[cfg(target_os = "linux")] return crate::desk::capture(); + #[cfg(target_os = "windows")] + return crate::win::capture(); #[allow(unreachable_code)] { // A random tail: two bots screenshotting the same millisecond must not delete each @@ -1919,7 +1921,11 @@ mod mac { } -#[cfg(not(any(target_os = "macos", target_os = "linux")))] +#[cfg(target_os = "windows")] +use crate::win as mac; + +/// Any other OS (there is no third one we ship): a bot that can see and do nothing. +#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))] mod mac { pub fn screen_capture_allowed() -> bool { false diff --git a/src/intents.rs b/src/intents.rs index a4a74ee..6caef13 100644 --- a/src/intents.rs +++ b/src/intents.rs @@ -94,6 +94,10 @@ fn is_num(s: &str) -> Option { /// The script for one validated pick. Constants and (checked) numbers only — the model's /// words never reach the shell. pub fn script(name: &str, arg: &str) -> Result { + // Windows hands exist (C9), Windows shortcuts don't yet: the scripts here are sh. + if cfg!(target_os = "windows") { + return Err("系統捷徑目前支援 macOS 和 Linux；這台 Windows 請用截圖與滑鼠。".into()); + } if find(name).is_none() { return Err(format!("沒有捷徑「{name}」。")); } diff --git a/src/main.rs b/src/main.rs index 077c2f7..dca513c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -61,6 +61,8 @@ mod vault; mod vault_store; mod vnc; mod voice; +#[cfg(target_os = "windows")] +mod win; mod tunnel; mod web; mod wire; diff --git a/src/win.rs b/src/win.rs new file mode 100644 index 0000000..8e97980 --- /dev/null +++ b/src/win.rs @@ -0,0 +1,270 @@ +//! Windows hands and eyes (C9): SendInput for mouse and keyboard, GDI BitBlt for the screen, +//! Win32 for "what is under that point". The control tree comes from agent-desktop-windows, +//! which speaks the same elements() language as macOS's accessibility here. +#![cfg(target_os = "windows")] +use windows::Win32::Foundation::{CloseHandle, HWND, POINT, RECT}; +use windows::Win32::Graphics::Gdi::{ + BitBlt, CreateCompatibleBitmap, CreateCompatibleDC, DeleteDC, DeleteObject, GetDC, GetDIBits, ReleaseDC, SelectObject, BITMAPINFO, BITMAPINFOHEADER, BI_RGB, + DIB_RGB_COLORS, SRCCOPY, +}; +use windows::Win32::UI::Input::KeyboardAndMouse::{ + SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, INPUT_MOUSE, KEYBDINPUT, KEYEVENTF_KEYUP, KEYEVENTF_UNICODE, KEYBD_EVENT_FLAGS, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP, + MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_MOVE, MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_WHEEL, MOUSEINPUT, MOUSE_EVENT_FLAGS, VIRTUAL_KEY, +}; +use windows::Win32::UI::WindowsAndMessaging::{ + GetForegroundWindow, GetSystemMetrics, GetWindowRect, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId, SM_CXSCREEN, SM_CYSCREEN, WindowFromPoint, +}; +use windows::Win32::System::Threading::{OpenProcess, QueryFullProcessImageNameW, PROCESS_NAME_FORMAT, PROCESS_QUERY_LIMITED_INFORMATION}; + +fn mouse(dx: i32, dy: i32, data: u32, flags: MOUSE_EVENT_FLAGS) { + let mi = MOUSEINPUT { dx, dy, mouseData: data, dwFlags: flags, time: 0, dwExtraInfo: 0 }; + let inp = INPUT { r#type: INPUT_MOUSE, Anonymous: INPUT_0 { mi } }; + let _ = unsafe { SendInput(&[inp], std::mem::size_of::() as i32) }; +} + +/// Windows wants the whole virtual desktop in 0..65535. +fn norm(x: f64, y: f64) -> (i32, i32) { + // single-monitor first: the app's own metrics (multi-monitor virtual bounds later) + let w = unsafe { GetSystemMetrics(SM_CXSCREEN) }.max(1) as f64; + let h = unsafe { GetSystemMetrics(SM_CYSCREEN) }.max(1) as f64; + (((x / w) * 65535.0).round() as i32, ((y / h) * 65535.0).round() as i32) +} + +fn keys(vk: u16, up: bool, unicode: bool, ch: u16) { + let ki = KEYBDINPUT { + wVk: VIRTUAL_KEY(if unicode { 0 } else { vk }), + wScan: ch, + dwFlags: if unicode { KEYEVENTF_UNICODE } else { if up { KEYEVENTF_KEYUP } else { KEYBD_EVENT_FLAGS(0) } }, + time: 0, + dwExtraInfo: 0, + }; + let inp = INPUT { r#type: INPUT_KEYBOARD, Anonymous: INPUT_0 { ki } }; + let _ = unsafe { SendInput(&[inp], std::mem::size_of::() as i32) }; +} + +/// macOS key codes (what key_code() hands us) → Windows virtual keys. 0 = no Windows key for it. +fn vk_of(code: u16) -> u16 { + static VK_TABLE: &[(u16, u16)] = &[ + (0, 0x41), (11, 0x42), (8, 0x43), (2, 0x44), (14, 0x45), (3, 0x46), (5, 0x47), (4, 0x48), (34, 0x49), (38, 0x4A), (40, 0x4B), (37, 0x4C), (46, 0x4D), (45, 0x4E), + (31, 0x4F), (35, 0x50), (12, 0x51), (15, 0x52), (1, 0x53), (17, 0x54), (32, 0x55), (9, 0x56), (13, 0x57), (7, 0x58), (16, 0x59), (6, 0x5A), + (18, 0x31), (19, 0x32), (20, 0x33), (21, 0x34), (23, 0x35), (22, 0x36), (26, 0x37), (28, 0x38), (25, 0x39), (29, 0x30), + (36, 0x0D), (76, 0x03), (48, 0x09), (49, 0x20), (51, 0x08), (53, 0x1B), (24, 0xBB), (27, 0xBD), (33, 0xDB), (30, 0xDD), (41, 0xBA), (39, 0xDE), (43, 0xBC), + (47, 0xBE), (44, 0xBF), (50, 0xC0), (42, 0xDC), (52, 0xE7), (36, 0x0D), (123, 0x25), (124, 0x27), (126, 0x26), (125, 0x28), (115, 0x24), (119, 0x23), (116, 0x21), + (121, 0x22), (117, 0x2E), (122, 0x70), (120, 0x71), (99, 0x72), (118, 0x73), (96, 0x74), (97, 0x75), (98, 0x76), + ]; + VK_TABLE.iter().find(|(c, _)| *c == code).map(|(_, v)| *v).unwrap_or(0) +} + +fn modifiers(flags: u64) -> (Vec, Vec) { + let mut down = vec![]; + if flags & 0x20000 != 0 { + down.push(0x10); // shift + } + if flags & 0x40000 != 0 { + down.push(0x11); // ctrl + } + if flags & 0x80000 != 0 { + down.push(0x12); // alt + } + if flags & 0x100000 != 0 { + down.push(0x5B); // win + } + let up: Vec = down.iter().rev().cloned().collect(); + (down, up) +} + +pub fn move_to(x: f64, y: f64) { + let (nx, ny) = norm(x, y); + mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE); +} + +pub fn click(x: f64, y: f64, button: &str, count: i64, flags: u64) { + let (down, up) = match button { + "right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP), + "middle" => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP), + _ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP), + }; + let _ = flags; + let (nx, ny) = norm(x, y); + mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE); + for n in 1..=count.max(1) { + if n > 1 { + std::thread::sleep(std::time::Duration::from_millis(80)); + } + mouse(nx, ny, 0, down); + std::thread::sleep(std::time::Duration::from_millis(30)); + mouse(nx, ny, 0, up); + } +} + +pub fn drag(points: &[(f64, f64)], button: &str, flags: u64) { + let (down, up) = match button { + "right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP), + _ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP), + }; + let _ = flags; + if let Some((x, y)) = points.first() { + let (nx, ny) = norm(*x, *y); + mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE); + mouse(nx, ny, 0, down); + } + for (x, y) in points.iter().skip(1) { + let (nx, ny) = norm(*x, *y); + mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + if let Some((x, y)) = points.last() { + let (nx, ny) = norm(*x, *y); + mouse(nx, ny, 0, up); + } +} + +pub fn scroll(at: Option<(f64, f64)>, x: i32, y: i32, flags: u64) { + let _ = (x, flags); + if let Some((px, py)) = at { + move_to(px, py); + } + // Windows wheels by 120ths of a line; the harness counts lines. + mouse(0, 0, ((y * 120) as i32) as u32, MOUSEEVENTF_WHEEL); +} + +pub fn type_text(text: &str) { + for u in text.encode_utf16() { + keys(0, false, true, u); + keys(0, true, true, u); + } +} + +pub fn press(code: u16, flags: u64, _hold: f64) { + let vk = vk_of(code); + if vk == 0 { + return; + } + let (down, up) = modifiers(flags); + for k in &down { + keys(*k, false, false, 0); + } + keys(vk, false, false, 0); + std::thread::sleep(std::time::Duration::from_millis(30)); + keys(vk, true, false, 0); + for k in &up { + keys(*k, true, false, 0); + } +} + +fn window_text(h: HWND) -> String { + let n = unsafe { GetWindowTextLengthW(h) }.max(0) as usize; + if n == 0 { + return String::new(); + } + let mut buf = vec![0u16; n + 1]; + let got = unsafe { GetWindowTextW(h, &mut buf) } as usize; + String::from_utf16_lossy(&buf[..got]) +} + +fn process_name(pid: u32) -> String { + unsafe { + let Ok(h) = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid) else { return String::new() }; + let mut buf = [0u16; 260]; + let mut len = buf.len() as u32; + let ok = QueryFullProcessImageNameW(h, PROCESS_NAME_FORMAT(0), windows::core::PWSTR(buf.as_mut_ptr()), &mut len).is_ok(); + let _ = CloseHandle(h); + if !ok { + return String::new(); + } + let full = String::from_utf16_lossy(&buf[..len as usize]); + std::path::Path::new(&full).file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or(full) + } +} + +fn rect(h: HWND) -> (i32, i32, i32, i32) { + let mut r = RECT::default(); + let _ = unsafe { GetWindowRect(h, &mut r) }; + (r.left, r.top, r.right, r.bottom) +} + +pub fn window_at(x: f64, y: f64) -> Option<(String, i32)> { + let h = unsafe { WindowFromPoint(POINT { x: x as i32, y: y as i32 }) }; + if h.is_invalid() { + return None; + } + let mut pid: u32 = 0; + unsafe { GetWindowThreadProcessId(h, Some(&mut pid)) }; + let t = window_text(h); + let name = if t.is_empty() { process_name(pid) } else { t }; + Some((name, pid as i32)) +} + +pub fn front_app() -> Option<(String, i32)> { + let h = unsafe { GetForegroundWindow() }; + if h.is_invalid() { + return None; + } + let mut pid: u32 = 0; + unsafe { GetWindowThreadProcessId(h, Some(&mut pid)) }; + let t = window_text(h); + let name = if t.is_empty() { process_name(pid) } else { t }; + Some((name, pid as i32)) +} + +pub fn screen_capture_allowed() -> bool { + true +} + +pub fn screen_capture_preflight() -> bool { + true +} + +pub fn screen_points() -> (f64, f64) { + (unsafe { GetSystemMetrics(SM_CXSCREEN) } as f64, unsafe { GetSystemMetrics(SM_CYSCREEN) } as f64) +} + +pub fn screen_pixels() -> (u32, u32) { + let (w, h) = screen_points(); + (w as u32, h as u32) +} + +/// The whole main screen, top-down, as pixels (BI_RGB 32-bit → RGB8). +pub fn capture() -> Result { + let w = screen_points().0 as i32; + let h = screen_points().1 as i32; + if w <= 0 || h <= 0 { + return Err("看不到畫面".into()); + } + unsafe { + let hdc = GetDC(None); + let mem = CreateCompatibleDC(Some(hdc)); + let bmp = CreateCompatibleBitmap(hdc, w, h); + let old = SelectObject(mem, bmp.into()); + let _ = BitBlt(mem, 0, 0, w, h, Some(hdc), 0, 0, SRCCOPY); + let mut header = BITMAPINFO { + bmiHeader: BITMAPINFOHEADER { + biSize: std::mem::size_of::() as u32, + biWidth: w, + biHeight: -h, // top-down + biPlanes: 1, + biBitCount: 32, + biCompression: BI_RGB.0 as u32, + ..Default::default() + }, + ..Default::default() + }; + let mut pixels = vec![0u8; (w as usize) * (h as usize) * 4]; + let rows = GetDIBits(mem, bmp, 0, h as u32, Some(pixels.as_mut_ptr() as *mut _), &mut header, DIB_RGB_COLORS); + let _ = SelectObject(mem, old); + let _ = DeleteObject(bmp.into()); + let _ = DeleteDC(mem); + ReleaseDC(None, hdc); + if rows == 0 { + return Err("截圖失敗".into()); + } + let mut img = image::RgbImage::new(w as u32, h as u32); + for y in 0..h as usize { + for x in 0..w as usize { + let i = (y * w as usize + x) * 4; + img.put_pixel(x as u32, y as u32, image::Rgb([pixels[i + 2], pixels[i + 1], pixels[i]])); + } + } + Ok(img) + } +}