Merge c9/windows: Win32 input + screenshot engine (C9, engine)
This commit is contained in:
daniel wang
commit
861bc59c54
|
|
@ -31,3 +31,12 @@ jobs:
|
|||
run: cargo clippy --bin anybot 2>&1 | tail -30
|
||||
- name: Relay build and test
|
||||
run: cd relay && cargo test
|
||||
|
||||
windows-check:
|
||||
# C9: the Windows engine is real code now; where a Windows runner exists, it must compile.
|
||||
# Report-only until the release ships Windows binaries.
|
||||
runs-on: windows
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: cargo check --bin anybot
|
||||
|
|
|
|||
|
|
@ -116,8 +116,8 @@ dependencies = [
|
|||
"accesskit_consumer",
|
||||
"hashbrown 0.17.1",
|
||||
"static_assertions",
|
||||
"windows",
|
||||
"windows-core",
|
||||
"windows 0.62.2",
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -225,7 +225,7 @@ dependencies = [
|
|||
"thiserror 2.0.21",
|
||||
"tracing",
|
||||
"uiautomation",
|
||||
"windows",
|
||||
"windows 0.62.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
|
|
@ -376,6 +376,7 @@ dependencies = [
|
|||
"wayland-protocols-misc",
|
||||
"wayland-protocols-wlr",
|
||||
"webpki-root-certs",
|
||||
"windows 0.61.3",
|
||||
"x11rb 0.13.2",
|
||||
"zbus",
|
||||
"zhconv",
|
||||
|
|
@ -965,7 +966,7 @@ dependencies = [
|
|||
"js-sys",
|
||||
"num-traits",
|
||||
"wasm-bindgen",
|
||||
"windows-link",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1234,8 +1235,8 @@ dependencies = [
|
|||
"objc2-core-foundation",
|
||||
"objc2-foundation 0.3.2",
|
||||
"web-sys",
|
||||
"windows",
|
||||
"windows-core",
|
||||
"windows 0.62.2",
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1960,8 +1961,8 @@ dependencies = [
|
|||
"read-fonts 0.41.0",
|
||||
"roxmltree 0.21.2",
|
||||
"smallvec",
|
||||
"windows",
|
||||
"windows-core",
|
||||
"windows 0.62.2",
|
||||
"windows-core 0.62.2",
|
||||
"yeslogic-fontconfig-sys",
|
||||
]
|
||||
|
||||
|
|
@ -2075,7 +2076,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8"
|
||||
dependencies = [
|
||||
"rustix 1.1.5",
|
||||
"windows-link",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2259,7 +2260,7 @@ dependencies = [
|
|||
"log",
|
||||
"presser",
|
||||
"thiserror 2.0.21",
|
||||
"windows",
|
||||
"windows 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2501,7 +2502,7 @@ dependencies = [
|
|||
"js-sys",
|
||||
"log",
|
||||
"wasm-bindgen",
|
||||
"windows-core",
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2710,7 +2711,7 @@ dependencies = [
|
|||
"simd_cesu8",
|
||||
"thiserror 2.0.21",
|
||||
"walkdir",
|
||||
"windows-link",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2833,7 +2834,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"windows-link",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3690,7 +3691,7 @@ dependencies = [
|
|||
"libc",
|
||||
"redox_syscall 0.5.18",
|
||||
"smallvec",
|
||||
"windows-link",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -5460,8 +5461,8 @@ checksum = "fa2eacf9390a8491bc949b04230f40c12520c6423a2962a6087eea1ef14373af"
|
|||
dependencies = [
|
||||
"chrono",
|
||||
"uiautomation_derive",
|
||||
"windows",
|
||||
"windows-core",
|
||||
"windows 0.62.2",
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -6157,9 +6158,9 @@ dependencies = [
|
|||
"web-sys",
|
||||
"wgpu-naga-bridge",
|
||||
"wgpu-types",
|
||||
"windows",
|
||||
"windows-core",
|
||||
"windows-result",
|
||||
"windows 0.62.2",
|
||||
"windows-core 0.62.2",
|
||||
"windows-result 0.4.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -6219,16 +6220,38 @@ version = "0.4.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.61.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
|
||||
dependencies = [
|
||||
"windows-collections 0.2.0",
|
||||
"windows-core 0.61.2",
|
||||
"windows-future 0.2.1",
|
||||
"windows-link 0.1.3",
|
||||
"windows-numerics 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.62.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580"
|
||||
dependencies = [
|
||||
"windows-collections",
|
||||
"windows-core",
|
||||
"windows-future",
|
||||
"windows-numerics",
|
||||
"windows-collections 0.3.2",
|
||||
"windows-core 0.62.2",
|
||||
"windows-future 0.3.2",
|
||||
"windows-numerics 0.3.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-collections"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"
|
||||
dependencies = [
|
||||
"windows-core 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -6237,7 +6260,20 @@ version = "0.3.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610"
|
||||
dependencies = [
|
||||
"windows-core",
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
|
||||
dependencies = [
|
||||
"windows-implement",
|
||||
"windows-interface",
|
||||
"windows-link 0.1.3",
|
||||
"windows-result 0.3.4",
|
||||
"windows-strings 0.4.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -6248,9 +6284,20 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
|
|||
dependencies = [
|
||||
"windows-implement",
|
||||
"windows-interface",
|
||||
"windows-link",
|
||||
"windows-result",
|
||||
"windows-strings",
|
||||
"windows-link 0.2.1",
|
||||
"windows-result 0.4.1",
|
||||
"windows-strings 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-future"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
|
||||
dependencies = [
|
||||
"windows-core 0.61.2",
|
||||
"windows-link 0.1.3",
|
||||
"windows-threading 0.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -6259,9 +6306,9 @@ version = "0.3.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb"
|
||||
dependencies = [
|
||||
"windows-core",
|
||||
"windows-link",
|
||||
"windows-threading",
|
||||
"windows-core 0.62.2",
|
||||
"windows-link 0.2.1",
|
||||
"windows-threading 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -6286,20 +6333,45 @@ dependencies = [
|
|||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-numerics"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"
|
||||
dependencies = [
|
||||
"windows-core 0.61.2",
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-numerics"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26"
|
||||
dependencies = [
|
||||
"windows-core",
|
||||
"windows-link",
|
||||
"windows-core 0.62.2",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
|
||||
dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -6308,7 +6380,16 @@ version = "0.4.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-strings"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
|
||||
dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -6317,7 +6398,7 @@ version = "0.5.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -6335,7 +6416,7 @@ version = "0.61.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -6354,13 +6435,22 @@ dependencies = [
|
|||
"windows_x86_64_msvc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-threading"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6"
|
||||
dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-threading"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -85,3 +85,4 @@ arboard = { version = "3.7", default-features = false }
|
|||
|
||||
[target.'cfg(target_os = "windows")'.dependencies]
|
||||
agent-desktop-windows = { git = "https://github.com/vercel-labs/agent-desktop", tag = "v1.1.0" }
|
||||
windows = { version = "0.61", features = ["Win32_Foundation", "Win32_Graphics_Gdi", "Win32_UI_Input_KeyboardAndMouse", "Win32_UI_WindowsAndMessaging", "Win32_System_Threading"] }
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@
|
|||
- `teach` 示範錄影→`replay_skill` 已有（時區題 9 步 49s→4 步 15s）。加：重播失敗自動截圖交還模型、示範可直接錄人的真實操作（box 端事件記錄）、技能進 registry 可分享。
|
||||
- ✅ 完成標準：使用者示範一次→第二次同任務 ≤5 步零模型。
|
||||
|
||||
## C9. Windows 操控面（接 P8）
|
||||
## C9. Windows 操控面（接 P8）— 🚧 2026-11：引擎已落地（`src/win.rs`：SendInput 滑鼠/鍵盤＋GDI 截圖＋前景視窗，mac code→VK 對照表；控制樹走 agent-desktop-windows；intents 在 Windows 明確降級為 GUI）。待實機驗證：多螢幕/高 DPI 座標、UIA 編號點擊、中文輸入。
|
||||
- UIA（agent-desktop-windows）讀取＋`SendInput`；`x11.rs/wl.rs` 對應 `win.rs`；facts/意圖表 Windows 版。
|
||||
- ✅ 完成標準：Windows 手腳可被大腦派畫面步（bench T2/T5 移植版）。
|
||||
|
||||
|
|
|
|||
|
|
@ -1538,6 +1538,8 @@ pub fn plain_picture() -> Result<(u32, u32, Vec<u8>), String> {
|
|||
pub fn capture_screen() -> Result<image::RgbImage, String> {
|
||||
#[cfg(target_os = "linux")]
|
||||
return crate::desk::capture();
|
||||
#[cfg(target_os = "windows")]
|
||||
return crate::win::capture();
|
||||
#[allow(unreachable_code)]
|
||||
{
|
||||
// A random tail: two bots screenshotting the same millisecond must not delete each
|
||||
|
|
@ -1919,7 +1921,11 @@ mod mac {
|
|||
}
|
||||
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "linux")))]
|
||||
#[cfg(target_os = "windows")]
|
||||
use crate::win as mac;
|
||||
|
||||
/// Any other OS (there is no third one we ship): a bot that can see and do nothing.
|
||||
#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]
|
||||
mod mac {
|
||||
pub fn screen_capture_allowed() -> bool {
|
||||
false
|
||||
|
|
|
|||
|
|
@ -94,6 +94,10 @@ fn is_num(s: &str) -> Option<u32> {
|
|||
/// The script for one validated pick. Constants and (checked) numbers only — the model's
|
||||
/// words never reach the shell.
|
||||
pub fn script(name: &str, arg: &str) -> Result<String, String> {
|
||||
// Windows hands exist (C9), Windows shortcuts don't yet: the scripts here are sh.
|
||||
if cfg!(target_os = "windows") {
|
||||
return Err("系統捷徑目前支援 macOS 和 Linux；這台 Windows 請用截圖與滑鼠。".into());
|
||||
}
|
||||
if find(name).is_none() {
|
||||
return Err(format!("沒有捷徑「{name}」。"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -61,6 +61,8 @@ mod vault;
|
|||
mod vault_store;
|
||||
mod vnc;
|
||||
mod voice;
|
||||
#[cfg(target_os = "windows")]
|
||||
mod win;
|
||||
mod tunnel;
|
||||
mod web;
|
||||
mod wire;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,270 @@
|
|||
//! Windows hands and eyes (C9): SendInput for mouse and keyboard, GDI BitBlt for the screen,
|
||||
//! Win32 for "what is under that point". The control tree comes from agent-desktop-windows,
|
||||
//! which speaks the same elements() language as macOS's accessibility here.
|
||||
#![cfg(target_os = "windows")]
|
||||
use windows::Win32::Foundation::{CloseHandle, HWND, POINT, RECT};
|
||||
use windows::Win32::Graphics::Gdi::{
|
||||
BitBlt, CreateCompatibleBitmap, CreateCompatibleDC, DeleteDC, DeleteObject, GetDC, GetDIBits, ReleaseDC, SelectObject, BITMAPINFO, BITMAPINFOHEADER, BI_RGB,
|
||||
DIB_RGB_COLORS, SRCCOPY,
|
||||
};
|
||||
use windows::Win32::UI::Input::KeyboardAndMouse::{
|
||||
SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, INPUT_MOUSE, KEYBDINPUT, KEYEVENTF_KEYUP, KEYEVENTF_UNICODE, KEYBD_EVENT_FLAGS, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP,
|
||||
MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_MOVE, MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_WHEEL, MOUSEINPUT, MOUSE_EVENT_FLAGS, VIRTUAL_KEY,
|
||||
};
|
||||
use windows::Win32::UI::WindowsAndMessaging::{
|
||||
GetForegroundWindow, GetSystemMetrics, GetWindowRect, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId, SM_CXSCREEN, SM_CYSCREEN, WindowFromPoint,
|
||||
};
|
||||
use windows::Win32::System::Threading::{OpenProcess, QueryFullProcessImageNameW, PROCESS_NAME_FORMAT, PROCESS_QUERY_LIMITED_INFORMATION};
|
||||
|
||||
fn mouse(dx: i32, dy: i32, data: u32, flags: MOUSE_EVENT_FLAGS) {
|
||||
let mi = MOUSEINPUT { dx, dy, mouseData: data, dwFlags: flags, time: 0, dwExtraInfo: 0 };
|
||||
let inp = INPUT { r#type: INPUT_MOUSE, Anonymous: INPUT_0 { mi } };
|
||||
let _ = unsafe { SendInput(&[inp], std::mem::size_of::<INPUT>() as i32) };
|
||||
}
|
||||
|
||||
/// Windows wants the whole virtual desktop in 0..65535.
|
||||
fn norm(x: f64, y: f64) -> (i32, i32) {
|
||||
// single-monitor first: the app's own metrics (multi-monitor virtual bounds later)
|
||||
let w = unsafe { GetSystemMetrics(SM_CXSCREEN) }.max(1) as f64;
|
||||
let h = unsafe { GetSystemMetrics(SM_CYSCREEN) }.max(1) as f64;
|
||||
(((x / w) * 65535.0).round() as i32, ((y / h) * 65535.0).round() as i32)
|
||||
}
|
||||
|
||||
fn keys(vk: u16, up: bool, unicode: bool, ch: u16) {
|
||||
let ki = KEYBDINPUT {
|
||||
wVk: VIRTUAL_KEY(if unicode { 0 } else { vk }),
|
||||
wScan: ch,
|
||||
dwFlags: if unicode { KEYEVENTF_UNICODE } else { if up { KEYEVENTF_KEYUP } else { KEYBD_EVENT_FLAGS(0) } },
|
||||
time: 0,
|
||||
dwExtraInfo: 0,
|
||||
};
|
||||
let inp = INPUT { r#type: INPUT_KEYBOARD, Anonymous: INPUT_0 { ki } };
|
||||
let _ = unsafe { SendInput(&[inp], std::mem::size_of::<INPUT>() as i32) };
|
||||
}
|
||||
|
||||
/// macOS key codes (what key_code() hands us) → Windows virtual keys. 0 = no Windows key for it.
|
||||
fn vk_of(code: u16) -> u16 {
|
||||
static VK_TABLE: &[(u16, u16)] = &[
|
||||
(0, 0x41), (11, 0x42), (8, 0x43), (2, 0x44), (14, 0x45), (3, 0x46), (5, 0x47), (4, 0x48), (34, 0x49), (38, 0x4A), (40, 0x4B), (37, 0x4C), (46, 0x4D), (45, 0x4E),
|
||||
(31, 0x4F), (35, 0x50), (12, 0x51), (15, 0x52), (1, 0x53), (17, 0x54), (32, 0x55), (9, 0x56), (13, 0x57), (7, 0x58), (16, 0x59), (6, 0x5A),
|
||||
(18, 0x31), (19, 0x32), (20, 0x33), (21, 0x34), (23, 0x35), (22, 0x36), (26, 0x37), (28, 0x38), (25, 0x39), (29, 0x30),
|
||||
(36, 0x0D), (76, 0x03), (48, 0x09), (49, 0x20), (51, 0x08), (53, 0x1B), (24, 0xBB), (27, 0xBD), (33, 0xDB), (30, 0xDD), (41, 0xBA), (39, 0xDE), (43, 0xBC),
|
||||
(47, 0xBE), (44, 0xBF), (50, 0xC0), (42, 0xDC), (52, 0xE7), (36, 0x0D), (123, 0x25), (124, 0x27), (126, 0x26), (125, 0x28), (115, 0x24), (119, 0x23), (116, 0x21),
|
||||
(121, 0x22), (117, 0x2E), (122, 0x70), (120, 0x71), (99, 0x72), (118, 0x73), (96, 0x74), (97, 0x75), (98, 0x76),
|
||||
];
|
||||
VK_TABLE.iter().find(|(c, _)| *c == code).map(|(_, v)| *v).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn modifiers(flags: u64) -> (Vec<u16>, Vec<u16>) {
|
||||
let mut down = vec![];
|
||||
if flags & 0x20000 != 0 {
|
||||
down.push(0x10); // shift
|
||||
}
|
||||
if flags & 0x40000 != 0 {
|
||||
down.push(0x11); // ctrl
|
||||
}
|
||||
if flags & 0x80000 != 0 {
|
||||
down.push(0x12); // alt
|
||||
}
|
||||
if flags & 0x100000 != 0 {
|
||||
down.push(0x5B); // win
|
||||
}
|
||||
let up: Vec<u16> = down.iter().rev().cloned().collect();
|
||||
(down, up)
|
||||
}
|
||||
|
||||
pub fn move_to(x: f64, y: f64) {
|
||||
let (nx, ny) = norm(x, y);
|
||||
mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE);
|
||||
}
|
||||
|
||||
pub fn click(x: f64, y: f64, button: &str, count: i64, flags: u64) {
|
||||
let (down, up) = match button {
|
||||
"right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP),
|
||||
"middle" => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP),
|
||||
_ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP),
|
||||
};
|
||||
let _ = flags;
|
||||
let (nx, ny) = norm(x, y);
|
||||
mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE);
|
||||
for n in 1..=count.max(1) {
|
||||
if n > 1 {
|
||||
std::thread::sleep(std::time::Duration::from_millis(80));
|
||||
}
|
||||
mouse(nx, ny, 0, down);
|
||||
std::thread::sleep(std::time::Duration::from_millis(30));
|
||||
mouse(nx, ny, 0, up);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn drag(points: &[(f64, f64)], button: &str, flags: u64) {
|
||||
let (down, up) = match button {
|
||||
"right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP),
|
||||
_ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP),
|
||||
};
|
||||
let _ = flags;
|
||||
if let Some((x, y)) = points.first() {
|
||||
let (nx, ny) = norm(*x, *y);
|
||||
mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE);
|
||||
mouse(nx, ny, 0, down);
|
||||
}
|
||||
for (x, y) in points.iter().skip(1) {
|
||||
let (nx, ny) = norm(*x, *y);
|
||||
mouse(nx, ny, 0, MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE);
|
||||
std::thread::sleep(std::time::Duration::from_millis(20));
|
||||
}
|
||||
if let Some((x, y)) = points.last() {
|
||||
let (nx, ny) = norm(*x, *y);
|
||||
mouse(nx, ny, 0, up);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scroll(at: Option<(f64, f64)>, x: i32, y: i32, flags: u64) {
|
||||
let _ = (x, flags);
|
||||
if let Some((px, py)) = at {
|
||||
move_to(px, py);
|
||||
}
|
||||
// Windows wheels by 120ths of a line; the harness counts lines.
|
||||
mouse(0, 0, ((y * 120) as i32) as u32, MOUSEEVENTF_WHEEL);
|
||||
}
|
||||
|
||||
pub fn type_text(text: &str) {
|
||||
for u in text.encode_utf16() {
|
||||
keys(0, false, true, u);
|
||||
keys(0, true, true, u);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn press(code: u16, flags: u64, _hold: f64) {
|
||||
let vk = vk_of(code);
|
||||
if vk == 0 {
|
||||
return;
|
||||
}
|
||||
let (down, up) = modifiers(flags);
|
||||
for k in &down {
|
||||
keys(*k, false, false, 0);
|
||||
}
|
||||
keys(vk, false, false, 0);
|
||||
std::thread::sleep(std::time::Duration::from_millis(30));
|
||||
keys(vk, true, false, 0);
|
||||
for k in &up {
|
||||
keys(*k, true, false, 0);
|
||||
}
|
||||
}
|
||||
|
||||
fn window_text(h: HWND) -> String {
|
||||
let n = unsafe { GetWindowTextLengthW(h) }.max(0) as usize;
|
||||
if n == 0 {
|
||||
return String::new();
|
||||
}
|
||||
let mut buf = vec![0u16; n + 1];
|
||||
let got = unsafe { GetWindowTextW(h, &mut buf) } as usize;
|
||||
String::from_utf16_lossy(&buf[..got])
|
||||
}
|
||||
|
||||
fn process_name(pid: u32) -> String {
|
||||
unsafe {
|
||||
let Ok(h) = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid) else { return String::new() };
|
||||
let mut buf = [0u16; 260];
|
||||
let mut len = buf.len() as u32;
|
||||
let ok = QueryFullProcessImageNameW(h, PROCESS_NAME_FORMAT(0), windows::core::PWSTR(buf.as_mut_ptr()), &mut len).is_ok();
|
||||
let _ = CloseHandle(h);
|
||||
if !ok {
|
||||
return String::new();
|
||||
}
|
||||
let full = String::from_utf16_lossy(&buf[..len as usize]);
|
||||
std::path::Path::new(&full).file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or(full)
|
||||
}
|
||||
}
|
||||
|
||||
fn rect(h: HWND) -> (i32, i32, i32, i32) {
|
||||
let mut r = RECT::default();
|
||||
let _ = unsafe { GetWindowRect(h, &mut r) };
|
||||
(r.left, r.top, r.right, r.bottom)
|
||||
}
|
||||
|
||||
pub fn window_at(x: f64, y: f64) -> Option<(String, i32)> {
|
||||
let h = unsafe { WindowFromPoint(POINT { x: x as i32, y: y as i32 }) };
|
||||
if h.is_invalid() {
|
||||
return None;
|
||||
}
|
||||
let mut pid: u32 = 0;
|
||||
unsafe { GetWindowThreadProcessId(h, Some(&mut pid)) };
|
||||
let t = window_text(h);
|
||||
let name = if t.is_empty() { process_name(pid) } else { t };
|
||||
Some((name, pid as i32))
|
||||
}
|
||||
|
||||
pub fn front_app() -> Option<(String, i32)> {
|
||||
let h = unsafe { GetForegroundWindow() };
|
||||
if h.is_invalid() {
|
||||
return None;
|
||||
}
|
||||
let mut pid: u32 = 0;
|
||||
unsafe { GetWindowThreadProcessId(h, Some(&mut pid)) };
|
||||
let t = window_text(h);
|
||||
let name = if t.is_empty() { process_name(pid) } else { t };
|
||||
Some((name, pid as i32))
|
||||
}
|
||||
|
||||
pub fn screen_capture_allowed() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub fn screen_capture_preflight() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
pub fn screen_points() -> (f64, f64) {
|
||||
(unsafe { GetSystemMetrics(SM_CXSCREEN) } as f64, unsafe { GetSystemMetrics(SM_CYSCREEN) } as f64)
|
||||
}
|
||||
|
||||
pub fn screen_pixels() -> (u32, u32) {
|
||||
let (w, h) = screen_points();
|
||||
(w as u32, h as u32)
|
||||
}
|
||||
|
||||
/// The whole main screen, top-down, as pixels (BI_RGB 32-bit → RGB8).
|
||||
pub fn capture() -> Result<image::RgbImage, String> {
|
||||
let w = screen_points().0 as i32;
|
||||
let h = screen_points().1 as i32;
|
||||
if w <= 0 || h <= 0 {
|
||||
return Err("看不到畫面".into());
|
||||
}
|
||||
unsafe {
|
||||
let hdc = GetDC(None);
|
||||
let mem = CreateCompatibleDC(Some(hdc));
|
||||
let bmp = CreateCompatibleBitmap(hdc, w, h);
|
||||
let old = SelectObject(mem, bmp.into());
|
||||
let _ = BitBlt(mem, 0, 0, w, h, Some(hdc), 0, 0, SRCCOPY);
|
||||
let mut header = BITMAPINFO {
|
||||
bmiHeader: BITMAPINFOHEADER {
|
||||
biSize: std::mem::size_of::<BITMAPINFOHEADER>() as u32,
|
||||
biWidth: w,
|
||||
biHeight: -h, // top-down
|
||||
biPlanes: 1,
|
||||
biBitCount: 32,
|
||||
biCompression: BI_RGB.0 as u32,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let mut pixels = vec![0u8; (w as usize) * (h as usize) * 4];
|
||||
let rows = GetDIBits(mem, bmp, 0, h as u32, Some(pixels.as_mut_ptr() as *mut _), &mut header, DIB_RGB_COLORS);
|
||||
let _ = SelectObject(mem, old);
|
||||
let _ = DeleteObject(bmp.into());
|
||||
let _ = DeleteDC(mem);
|
||||
ReleaseDC(None, hdc);
|
||||
if rows == 0 {
|
||||
return Err("截圖失敗".into());
|
||||
}
|
||||
let mut img = image::RgbImage::new(w as u32, h as u32);
|
||||
for y in 0..h as usize {
|
||||
for x in 0..w as usize {
|
||||
let i = (y * w as usize + x) * 4;
|
||||
img.put_pixel(x as u32, y as u32, image::Rgb([pixels[i + 2], pixels[i + 1], pixels[i]]));
|
||||
}
|
||||
}
|
||||
Ok(img)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue