642 lines
29 KiB
Rust
642 lines
29 KiB
Rust
// 示範教學: record the person doing a task on this Mac so a bot can learn the steps and do it
|
||
// again later. The original: 「錄下你執行任務的過程。{Bot} 會學習這些步驟，之後可以自行再次執行。」
|
||
// and 「{Bot} 正在觀看並學習」 (it records the screen with ffmpeg plus an input-event log; see
|
||
// local-exec-daemon saveRecordingDataPackage {inputEvents, displayWidth, apiWidth…}).
|
||
// Here a listen-only event tap collects clicks, keys and scrolls (nothing is blocked or changed),
|
||
// and they're written up as numbered steps with screenshot coordinates.
|
||
|
||
use serde::{Deserialize, Serialize};
|
||
use std::sync::Mutex;
|
||
use std::time::Instant;
|
||
|
||
/// Two clicks closer than this (and near each other) are a double click (estimated).
|
||
const DOUBLE_CLICK_MS: u128 = 400;
|
||
/// Scrolls closer than this are one step (estimated).
|
||
const SCROLL_MERGE_MS: u128 = 1500;
|
||
|
||
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq)]
|
||
pub struct Demo {
|
||
pub name: String,
|
||
pub created: String,
|
||
pub steps: Vec<String>,
|
||
/// The screen recording (mp4), like the original's ffmpeg recording; empty when none.
|
||
#[serde(default)]
|
||
pub video: String,
|
||
/// The screen at each step (a JPEG per step; empty when not captured).
|
||
#[serde(default)]
|
||
pub shots: Vec<String>,
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
enum Raw {
|
||
Click { x: f64, y: f64, right: bool, app: String, at: Instant },
|
||
Key { text: String, code: i64, flags: u64, app: String },
|
||
Scroll { dy: i64, app: String, at: Instant },
|
||
}
|
||
|
||
static EVENTS: Mutex<Vec<Raw>> = Mutex::new(Vec::new());
|
||
/// The frame number showing on screen when each event happened (same order as EVENTS).
|
||
static EVENT_FRAMES: Mutex<Vec<usize>> = Mutex::new(Vec::new());
|
||
static FRAMES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||
static FRAME_STOP: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
|
||
static FRAME_HANDLE: Mutex<Option<std::thread::JoinHandle<()>>> = Mutex::new(None);
|
||
static REC_DIR: Mutex<Option<std::path::PathBuf>> = Mutex::new(None);
|
||
/// Screen pictures per second while recording (estimated; the original records video).
|
||
pub const FRAMES_PER_SEC: u64 = 2;
|
||
/// Width of the recorded pictures (the original's render proxy is 1080p; smaller here).
|
||
pub const FRAME_WIDTH: u32 = 1280;
|
||
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
||
static RUNLOOP: Mutex<usize> = Mutex::new(0);
|
||
pub static RECORDING: Mutex<Option<(String, Instant)>> = Mutex::new(None);
|
||
|
||
pub fn recording() -> Option<(String, Instant)> {
|
||
RECORDING.lock().unwrap().clone()
|
||
}
|
||
|
||
/// Start recording for a bot. Err when macOS hasn't allowed listening to input yet.
|
||
pub fn start(bot_name: &str) -> Result<(), String> {
|
||
if RECORDING.lock().unwrap().is_some() {
|
||
return Err("已經在錄了".into());
|
||
}
|
||
EVENTS.lock().unwrap().clear();
|
||
EVENT_FRAMES.lock().unwrap().clear();
|
||
// Zero the frame counter before the listener starts: events that arrive in the first
|
||
// half second used to carry the *previous* recording's frame numbers (start_frames
|
||
// cleared it only after mac::start), mapping steps to pictures from an old demo.
|
||
FRAMES.store(0, std::sync::atomic::Ordering::SeqCst);
|
||
mac::start()?;
|
||
*RECORDING.lock().unwrap() = Some((bot_name.to_string(), Instant::now()));
|
||
start_frames();
|
||
Ok(())
|
||
}
|
||
|
||
fn demos_dir() -> std::path::PathBuf {
|
||
crate::model::State::path().parent().map(|p| p.to_path_buf()).unwrap_or_else(std::env::temp_dir).join("demos")
|
||
}
|
||
|
||
/// Pictures of the screen while recording: frame_00000.jpg, frame_00001.jpg…
|
||
fn start_frames() {
|
||
let dir = demos_dir().join(chrono::Local::now().format("%Y%m%d-%H%M%S").to_string());
|
||
if std::fs::create_dir_all(&dir).is_err() {
|
||
return;
|
||
}
|
||
*REC_DIR.lock().unwrap() = Some(dir.clone());
|
||
FRAMES.store(0, std::sync::atomic::Ordering::SeqCst);
|
||
FRAME_STOP.store(false, std::sync::atomic::Ordering::SeqCst);
|
||
*FRAME_HANDLE.lock().unwrap() = Some(std::thread::spawn(move || {
|
||
while !FRAME_STOP.load(std::sync::atomic::Ordering::SeqCst) {
|
||
let t0 = Instant::now();
|
||
if crate::computer::screen_supported() && crate::computer::screen_capture_ok() {
|
||
if let Ok(img) = crate::computer::capture_screen() {
|
||
{
|
||
let (w, h) = img.dimensions();
|
||
let small = if w > FRAME_WIDTH { image::imageops::resize(&img, FRAME_WIDTH, (h as u64 * FRAME_WIDTH as u64 / w.max(1) as u64) as u32, image::imageops::FilterType::Triangle) } else { img };
|
||
let n = FRAMES.load(std::sync::atomic::Ordering::SeqCst);
|
||
if small.save(dir.join(format!("frame_{n:05}.jpg"))).is_ok() {
|
||
FRAMES.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
let wait = std::time::Duration::from_millis(1000 / FRAMES_PER_SEC);
|
||
if let Some(rest) = wait.checked_sub(t0.elapsed()) {
|
||
std::thread::sleep(rest);
|
||
}
|
||
}
|
||
}));
|
||
}
|
||
|
||
/// Stop the pictures and turn them into a video when ffmpeg is there. Returns (dir, video).
|
||
fn stop_frames() -> Option<(std::path::PathBuf, String)> {
|
||
FRAME_STOP.store(true, std::sync::atomic::Ordering::SeqCst);
|
||
// Let the capturing thread finish its current picture and exit (it ends within a second).
|
||
// The old fixed 700 ms sleep raced slow captures (dirty files after the video was made)
|
||
// and froze the window for its whole duration.
|
||
if let Some(h) = FRAME_HANDLE.lock().unwrap().take() {
|
||
let _ = h.join();
|
||
}
|
||
let dir = REC_DIR.lock().unwrap().take()?;
|
||
// Encoding runs off the window thread (ffmpeg on a long demo takes seconds; the window
|
||
// used to freeze for all of it). The mp4 file appears when it is done.
|
||
if dir.join("frame_00000.jpg").exists() {
|
||
let d = dir.clone();
|
||
std::thread::spawn(move || {
|
||
if make_video(&d).is_some() {
|
||
// The video holds the motion; drop the single pictures it consumed.
|
||
if let Ok(rd) = std::fs::read_dir(&d) {
|
||
for e in rd.flatten() {
|
||
if e.file_name().to_string_lossy().starts_with("frame_") {
|
||
let _ = std::fs::remove_file(e.path());
|
||
}
|
||
}
|
||
}
|
||
}
|
||
});
|
||
return Some((dir.clone(), dir.join("recording.mp4").display().to_string()));
|
||
}
|
||
Some((dir, String::new()))
|
||
}
|
||
|
||
/// frame_*.jpg → recording.mp4 (H.264, like the original's render proxy).
|
||
pub fn make_video(dir: &std::path::Path) -> Option<String> {
|
||
if !dir.join("frame_00000.jpg").exists() {
|
||
return None;
|
||
}
|
||
let out = dir.join("recording.mp4");
|
||
let ok = std::process::Command::new("ffmpeg")
|
||
.env("PATH", crate::cloudbox::tool_path())
|
||
.args(["-hide_banner", "-loglevel", "error", "-y", "-framerate", &FRAMES_PER_SEC.to_string(), "-i"])
|
||
.arg(dir.join("frame_%05d.jpg"))
|
||
.args(["-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2,format=yuv420p", "-c:v", "libx264", "-preset", "veryfast", "-movflags", "+faststart"])
|
||
.arg(&out)
|
||
.stdin(std::process::Stdio::null())
|
||
.stdout(std::process::Stdio::null())
|
||
.stderr(std::process::Stdio::null())
|
||
.status()
|
||
.map(|s| s.success())
|
||
.unwrap_or(false);
|
||
ok.then(|| out.display().to_string())
|
||
}
|
||
|
||
/// Stop and write up the steps.
|
||
pub fn finish(name: &str) -> Demo {
|
||
mac::stop();
|
||
// The listener stops asynchronously (the X11 one polls every 150 ms): give the last
|
||
// events of the demo time to arrive — they used to be cut off the recording.
|
||
std::thread::sleep(std::time::Duration::from_millis(300));
|
||
*RECORDING.lock().unwrap() = None;
|
||
let raw = std::mem::take(&mut *EVENTS.lock().unwrap());
|
||
let frames_at = std::mem::take(&mut *EVENT_FRAMES.lock().unwrap());
|
||
let compiled = compile(&raw, crate::computer::points_to_shot());
|
||
let rec = stop_frames();
|
||
let mut shots = vec![];
|
||
let mut video = String::new();
|
||
if let Some((dir, v)) = &rec {
|
||
video = v.clone();
|
||
for (i, (_, first)) in compiled.iter().enumerate() {
|
||
// The picture just before the step happened (what the person was looking at).
|
||
let f = frames_at.get(*first).copied().unwrap_or(0);
|
||
let src = dir.join(format!("frame_{f:05}.jpg"));
|
||
let dst = dir.join(format!("step_{:02}.jpg", i + 1));
|
||
shots.push(if std::fs::copy(&src, &dst).is_ok() { dst.display().to_string() } else { String::new() });
|
||
}
|
||
// The frame pictures are removed by the encoder thread once the video is written.
|
||
}
|
||
Demo { name: name.trim().to_string(), created: chrono::Local::now().format("%Y-%m-%d %H:%M").to_string(), steps: compiled.into_iter().map(|c| c.0).collect(), video, shots }
|
||
}
|
||
|
||
pub fn cancel() {
|
||
mac::stop();
|
||
*RECORDING.lock().unwrap() = None;
|
||
EVENTS.lock().unwrap().clear();
|
||
EVENT_FRAMES.lock().unwrap().clear();
|
||
if let Some((dir, _)) = stop_frames() {
|
||
let _ = std::fs::remove_dir_all(dir);
|
||
}
|
||
}
|
||
|
||
/// The view_demo_step tool: the screen at one step of a demonstration, for the bot to look at.
|
||
pub fn view_step_spec() -> serde_json::Value {
|
||
serde_json::json!({"name": "view_demo_step", "description": "See what the person's screen looked like at one step of a demonstration they recorded for you (a picture), to find the right buttons when you repeat it.",
|
||
"parameters": {"type": "object", "properties": {"demo": {"type": "string", "description": "The demonstration's name"}, "step": {"type": "integer", "description": "Step number, from 1"}}, "required": ["demo", "step"]}})
|
||
}
|
||
|
||
/// Returns (text, ok, picture as base64 JPEG).
|
||
pub fn view_step(demos: &[Demo], args: &str) -> (String, bool, Option<String>) {
|
||
use base64::Engine;
|
||
let v: serde_json::Value = serde_json::from_str(args).unwrap_or_default();
|
||
let name = v["demo"].as_str().unwrap_or("").trim();
|
||
let Some(d) = demos.iter().find(|d| d.name == name).or_else(|| demos.iter().find(|d| d.name.contains(name) && !name.is_empty())) else {
|
||
return (format!("沒有叫「{name}」的示範。有：{}", demos.iter().map(|d| d.name.as_str()).collect::<Vec<_>>().join("、")), false, None);
|
||
};
|
||
let n = v["step"].as_u64().unwrap_or(0) as usize;
|
||
if n == 0 || n > d.steps.len() {
|
||
return (format!("「{}」有 {} 個步驟，請給 1 到 {}。", d.name, d.steps.len(), d.steps.len()), false, None);
|
||
}
|
||
let text = format!("「{}」第 {n} 步：{}", d.name, d.steps[n - 1]);
|
||
let Some(path) = d.shots.get(n - 1).filter(|p| !p.is_empty()) else {
|
||
return (format!("{text}\n（這一步沒有存到畫面）"), true, None);
|
||
};
|
||
match std::fs::read(path).ok().and_then(|b| image::load_from_memory(&b).ok()) {
|
||
Some(img) => {
|
||
let img = img.to_rgb8();
|
||
let (w, h) = img.dimensions();
|
||
let (fw, fh, resize) = crate::computer::fit(w, h);
|
||
let img = if resize { image::imageops::resize(&img, fw, fh, image::imageops::FilterType::Triangle) } else { img };
|
||
match crate::computer::encode_jpeg(&img) {
|
||
Some(j) => (format!("{text}\n這是使用者做這一步之前的畫面（{fw}×{fh}，座標和步驟裡的一樣）。"), true, Some(base64::engine::general_purpose::STANDARD.encode(j))),
|
||
None => (format!("{text}\n（畫面太大，沒辦法送出）"), true, None),
|
||
}
|
||
}
|
||
None => (format!("{text}\n（畫面檔案不見了）"), true, None),
|
||
}
|
||
}
|
||
|
||
fn key_name(code: i64) -> Option<&'static str> {
|
||
Some(match code {
|
||
36 => "return",
|
||
48 => "tab",
|
||
49 => "space",
|
||
51 => "delete",
|
||
53 => "escape",
|
||
117 => "forwarddelete",
|
||
123 => "left",
|
||
124 => "right",
|
||
125 => "down",
|
||
126 => "up",
|
||
115 => "home",
|
||
119 => "end",
|
||
116 => "pageup",
|
||
121 => "pagedown",
|
||
_ => return None,
|
||
})
|
||
}
|
||
|
||
const CMD: u64 = 1 << 20;
|
||
const SHIFT: u64 = 1 << 17;
|
||
const CTRL: u64 = 1 << 18;
|
||
const OPT: u64 = 1 << 19;
|
||
|
||
/// Raw events → steps a bot can follow.
|
||
fn compile(raw: &[Raw], to_shot: f64) -> Vec<(String, usize)> {
|
||
let mut steps: Vec<(String, usize)> = vec![];
|
||
let mut typed_from = 0usize;
|
||
let mut typed = String::new();
|
||
let mut typed_app = String::new();
|
||
let mut last_app = String::new();
|
||
let flush = |steps: &mut Vec<(String, usize)>, typed: &mut String, app: &str, from: usize| {
|
||
if !typed.is_empty() {
|
||
steps.push((format!("在「{app}」輸入「{typed}」"), from));
|
||
typed.clear();
|
||
}
|
||
};
|
||
let mut i = 0;
|
||
while i < raw.len() {
|
||
match &raw[i] {
|
||
Raw::Click { x, y, right, app, at } => {
|
||
flush(&mut steps, &mut typed, &typed_app, typed_from);
|
||
let start = i;
|
||
let mut count = 1;
|
||
while let Some(Raw::Click { x: x2, y: y2, right: r2, at: t2, .. }) = raw.get(i + 1) {
|
||
if r2 == right && (x2 - x).abs() < 6.0 && (y2 - y).abs() < 6.0 && t2.duration_since(*at).as_millis() < DOUBLE_CLICK_MS * count as u128 {
|
||
count += 1;
|
||
i += 1;
|
||
} else {
|
||
break;
|
||
}
|
||
}
|
||
let verb = match (right, count) {
|
||
(true, _) => "按右鍵".to_string(),
|
||
(false, 1) => "點".to_string(),
|
||
(false, n) => format!("連點 {n} 下"),
|
||
};
|
||
let switch = if *app != last_app && !app.is_empty() { format!("切到「{app}」，") } else { String::new() };
|
||
steps.push((format!("{switch}在「{app}」{verb} ({:.0}, {:.0})", x * to_shot, y * to_shot), start));
|
||
last_app = app.clone();
|
||
}
|
||
Raw::Key { text, code, flags, app } => {
|
||
let mods: Vec<&str> = [(CMD, "cmd"), (CTRL, "control"), (OPT, "option")].iter().filter(|(f, _)| flags & f != 0).map(|(_, n)| *n).collect();
|
||
if mods.is_empty() && *code == 51 && !typed.is_empty() {
|
||
typed.pop();
|
||
} else if mods.is_empty() && key_name(*code).is_none() && !text.is_empty() && !text.chars().any(char::is_control) {
|
||
if typed.is_empty() {
|
||
typed_app = app.clone();
|
||
typed_from = i;
|
||
}
|
||
typed.push_str(text);
|
||
} else {
|
||
flush(&mut steps, &mut typed, &typed_app, typed_from);
|
||
let key = key_name(*code).map(String::from).unwrap_or_else(|| text.to_lowercase());
|
||
if key.is_empty() {
|
||
i += 1;
|
||
continue;
|
||
}
|
||
let mut combo: Vec<String> = mods.iter().map(|m| m.to_string()).collect();
|
||
if flags & SHIFT != 0 && !mods.is_empty() {
|
||
combo.push("shift".into());
|
||
}
|
||
combo.push(key);
|
||
steps.push((format!("按 {}", combo.join("+")), i));
|
||
}
|
||
last_app = app.clone();
|
||
}
|
||
Raw::Scroll { dy, app, at } => {
|
||
flush(&mut steps, &mut typed, &typed_app, typed_from);
|
||
let start = i;
|
||
let mut total = *dy;
|
||
let mut last = *at;
|
||
while let Some(Raw::Scroll { dy: d2, at: t2, .. }) = raw.get(i + 1) {
|
||
if t2.duration_since(last).as_millis() < SCROLL_MERGE_MS && d2.signum() == dy.signum() {
|
||
total += d2;
|
||
last = *t2;
|
||
i += 1;
|
||
} else {
|
||
break;
|
||
}
|
||
}
|
||
steps.push((format!("在「{app}」往{}捲動", if total < 0 { "下" } else { "上" }), start));
|
||
}
|
||
}
|
||
i += 1;
|
||
}
|
||
flush(&mut steps, &mut typed, &typed_app, typed_from);
|
||
steps
|
||
}
|
||
|
||
/// The steps as told to the bot.
|
||
pub fn prompt_section(demos: &[Demo]) -> String {
|
||
if demos.is_empty() {
|
||
return String::new();
|
||
}
|
||
let mut s = String::from("\n\n你學過的示範（使用者在他的 Mac 上做給你看的步驟）。使用者要你做同樣的事時，照著在「你的電腦」上做：開始先截圖一次，之後每一步做完都會附上新畫面，座標是錄製時截圖上的位置，畫面不同就找對應的按鈕；輸入的內容可以依新的需求調整。想看某一步當時的畫面，用 view_demo_step。");
|
||
for d in demos {
|
||
s.push_str(&format!("\n### {}（{}）\n", d.name, d.created));
|
||
for (i, st) in d.steps.iter().enumerate() {
|
||
s.push_str(&format!("{}. {st}\n", i + 1));
|
||
}
|
||
}
|
||
s
|
||
}
|
||
|
||
fn push(e: Raw) {
|
||
// AnyBot's own window (its Stop button, the bar) isn't part of the task.
|
||
let own = match &e {
|
||
Raw::Click { app, .. } | Raw::Key { app, .. } | Raw::Scroll { app, .. } => app.to_lowercase().contains("anybot"),
|
||
};
|
||
if !own {
|
||
EVENTS.lock().unwrap().push(e);
|
||
// The last picture taken before this moment.
|
||
EVENT_FRAMES.lock().unwrap().push(FRAMES.load(std::sync::atomic::Ordering::SeqCst).saturating_sub(1));
|
||
}
|
||
}
|
||
|
||
#[cfg(target_os = "macos")]
|
||
mod mac {
|
||
use super::*;
|
||
use std::ffi::c_void;
|
||
|
||
#[repr(C)]
|
||
#[derive(Clone, Copy)]
|
||
struct CGPoint {
|
||
x: f64,
|
||
y: f64,
|
||
}
|
||
type Cb = extern "C" fn(*mut c_void, u32, *mut c_void, *mut c_void) -> *mut c_void;
|
||
|
||
#[link(name = "CoreGraphics", kind = "framework")]
|
||
extern "C" {
|
||
fn CGEventTapCreate(tap: u32, place: u32, options: u32, mask: u64, cb: Cb, user: *mut c_void) -> *mut c_void;
|
||
fn CGEventTapEnable(tap: *mut c_void, enable: bool);
|
||
fn CGEventGetLocation(e: *mut c_void) -> CGPoint;
|
||
fn CGEventGetFlags(e: *mut c_void) -> u64;
|
||
fn CGEventGetIntegerValueField(e: *mut c_void, field: u32) -> i64;
|
||
fn CGEventKeyboardGetUnicodeString(e: *mut c_void, max: usize, actual: *mut usize, buf: *mut u16);
|
||
}
|
||
#[link(name = "CoreFoundation", kind = "framework")]
|
||
extern "C" {
|
||
fn CFMachPortCreateRunLoopSource(alloc: *const c_void, port: *mut c_void, order: isize) -> *mut c_void;
|
||
fn CFRunLoopGetCurrent() -> *mut c_void;
|
||
fn CFRunLoopAddSource(rl: *mut c_void, src: *mut c_void, mode: *const c_void);
|
||
fn CFRunLoopRun();
|
||
fn CFRunLoopStop(rl: *mut c_void);
|
||
static kCFRunLoopCommonModes: *const c_void;
|
||
}
|
||
|
||
static TAP: Mutex<usize> = Mutex::new(0);
|
||
// The foreground app barely moves between keystrokes, and the window-list query is the
|
||
// slowest call in the tap callback — slow callbacks make macOS disable the tap (events
|
||
// lost until it is re-enabled). Answer keystrokes from a short cache.
|
||
static FRONT_CACHE: Mutex<Option<(Instant, String)>> = Mutex::new(None);
|
||
fn front_cached() -> String {
|
||
let mut c = FRONT_CACHE.lock().unwrap_or_else(|e| e.into_inner());
|
||
if let Some((t, n)) = c.as_ref() {
|
||
if t.elapsed() < std::time::Duration::from_millis(400) {
|
||
return n.clone();
|
||
}
|
||
}
|
||
let n = crate::computer::frontmost_app().unwrap_or_default();
|
||
*c = Some((Instant::now(), n.clone()));
|
||
n
|
||
}
|
||
|
||
extern "C" fn callback(_proxy: *mut c_void, kind: u32, e: *mut c_void, _user: *mut c_void) -> *mut c_void {
|
||
unsafe {
|
||
match kind {
|
||
1 | 3 => {
|
||
let p = CGEventGetLocation(e);
|
||
let app = crate::computer::app_at_point(p.x, p.y).unwrap_or_default();
|
||
push(Raw::Click { x: p.x, y: p.y, right: kind == 3, app, at: Instant::now() });
|
||
}
|
||
10 => {
|
||
let mut buf = [0u16; 8];
|
||
let mut n = 0usize;
|
||
CGEventKeyboardGetUnicodeString(e, buf.len(), &mut n, buf.as_mut_ptr());
|
||
let text = String::from_utf16_lossy(&buf[..n.min(buf.len())]);
|
||
let code = CGEventGetIntegerValueField(e, 9);
|
||
let app = front_cached();
|
||
push(Raw::Key { text, code, flags: CGEventGetFlags(e), app });
|
||
}
|
||
22 => {
|
||
let dy = CGEventGetIntegerValueField(e, 11);
|
||
if dy != 0 {
|
||
let p = CGEventGetLocation(e);
|
||
let app = crate::computer::app_at_point(p.x, p.y).unwrap_or_default();
|
||
push(Raw::Scroll { dy, app, at: Instant::now() });
|
||
}
|
||
}
|
||
// The system turned the tap off (slow callback): turn it back on.
|
||
0xFFFF_FFFE | 0xFFFF_FFFF => {
|
||
let t = *TAP.lock().unwrap();
|
||
if t != 0 {
|
||
CGEventTapEnable(t as *mut c_void, true);
|
||
}
|
||
}
|
||
_ => {}
|
||
}
|
||
}
|
||
e
|
||
}
|
||
|
||
pub fn start() -> Result<(), String> {
|
||
let (tx, rx) = std::sync::mpsc::channel();
|
||
std::thread::spawn(move || unsafe {
|
||
let mask: u64 = (1 << 1) | (1 << 3) | (1 << 10) | (1 << 22);
|
||
// kCGSessionEventTap, head insert, listen only.
|
||
let tap = CGEventTapCreate(1, 0, 1, mask, callback, std::ptr::null_mut());
|
||
if tap.is_null() {
|
||
let _ = tx.send(false);
|
||
return;
|
||
}
|
||
*TAP.lock().unwrap() = tap as usize;
|
||
let src = CFMachPortCreateRunLoopSource(std::ptr::null(), tap, 0);
|
||
let rl = CFRunLoopGetCurrent();
|
||
CFRunLoopAddSource(rl, src, kCFRunLoopCommonModes);
|
||
CGEventTapEnable(tap, true);
|
||
*RUNLOOP.lock().unwrap() = rl as usize;
|
||
let _ = tx.send(true);
|
||
CFRunLoopRun();
|
||
CGEventTapEnable(tap, false);
|
||
*TAP.lock().unwrap() = 0;
|
||
});
|
||
match rx.recv_timeout(std::time::Duration::from_secs(3)) {
|
||
Ok(true) => Ok(()),
|
||
_ => {
|
||
let _ = crate::computer::spawn_reaped(std::process::Command::new("open").arg("x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent"));
|
||
Err("macOS 還沒讓 AnyBot 觀看你的操作。我已經打開「系統設定 → 隱私權與安全性 → 輸入監控」：把 AnyBot（或「終端機」）打開，再按一次「開始錄製」。".into())
|
||
}
|
||
}
|
||
}
|
||
|
||
pub fn stop() {
|
||
let rl = std::mem::take(&mut *RUNLOOP.lock().unwrap());
|
||
if rl != 0 {
|
||
unsafe { CFRunLoopStop(rl as *mut c_void) };
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Linux: the X RECORD extension shows every click and key (nothing is blocked or changed).
|
||
#[cfg(target_os = "linux")]
|
||
mod mac {
|
||
use super::*;
|
||
use std::sync::atomic::{AtomicBool, Ordering};
|
||
|
||
static STOP: AtomicBool = AtomicBool::new(false);
|
||
|
||
pub fn start() -> Result<(), String> {
|
||
if crate::desk::kind() != crate::desk::Kind::X11 {
|
||
return Err(if crate::desk::wayland() {
|
||
"在 Wayland 上，系統不讓 App 看到你按了哪裡、打了什麼，所以沒辦法錄製示範（Bot 還是可以操作畫面）。要錄示範，請在登入畫面改選 Xorg 工作階段。"
|
||
} else {
|
||
"錄製示範需要桌面畫面（X11）。"
|
||
}
|
||
.into());
|
||
}
|
||
STOP.store(false, Ordering::SeqCst);
|
||
let (tx, rx) = std::sync::mpsc::channel();
|
||
std::thread::spawn(move || {
|
||
let tx2 = tx.clone();
|
||
let r = crate::x11::record(
|
||
|input| push(to_raw(input)),
|
||
move || {
|
||
let _ = tx2.send(Ok(()));
|
||
!STOP.load(Ordering::SeqCst)
|
||
},
|
||
);
|
||
if let Err(e) = r {
|
||
let _ = tx.send(Err(e));
|
||
}
|
||
});
|
||
// On a timeout the record thread may still be spinning up: close the gate so its
|
||
// keep_going() ends it — a thread left running meant a second RECORD context
|
||
// (double events) on the next try, plus a leaked X connection.
|
||
match rx.recv_timeout(std::time::Duration::from_secs(3)) {
|
||
Ok(Ok(())) => Ok(()),
|
||
Ok(Err(e)) => Err(e),
|
||
Err(_) => {
|
||
STOP.store(true, Ordering::SeqCst);
|
||
Err("沒辦法開始錄製（X11 沒有回應）".into())
|
||
}
|
||
}
|
||
}
|
||
|
||
/// X input → the same events the macOS tap gives (key codes and flags in macOS terms, so
|
||
/// the steps read the same; Linux's Ctrl shows as "control").
|
||
pub fn to_raw(i: crate::x11::Input) -> Raw {
|
||
let app = |x: f64, y: f64| crate::x11::window_at(x, y).map(|a| a.0).or_else(|| crate::x11::front_app().map(|a| a.0)).unwrap_or_default();
|
||
match i {
|
||
crate::x11::Input::Click { x, y, right } => Raw::Click { x, y, right, app: app(x, y), at: Instant::now() },
|
||
crate::x11::Input::Scroll { dy } => Raw::Scroll { dy, app: crate::x11::front_app().map(|a| a.0).unwrap_or_default(), at: Instant::now() },
|
||
crate::x11::Input::Key { text, keysym, mods } => {
|
||
let code = crate::x11::keysym_name(keysym)
|
||
.and_then(crate::computer::key_code)
|
||
.or_else(|| crate::x11::char_of_keysym(keysym).and_then(|c| crate::computer::key_code(&c.to_lowercase().to_string())))
|
||
.map(|c| c as i64)
|
||
.unwrap_or(-1);
|
||
let mut flags = 0;
|
||
if mods & 1 != 0 {
|
||
flags |= SHIFT;
|
||
}
|
||
if mods & 4 != 0 {
|
||
flags |= CTRL;
|
||
}
|
||
if mods & 8 != 0 {
|
||
flags |= OPT;
|
||
}
|
||
if mods & 64 != 0 {
|
||
flags |= CMD;
|
||
}
|
||
let text = if keysym == 0xff0d { "\r".into() } else { text };
|
||
Raw::Key { text, code, flags, app: crate::x11::front_app().map(|a| a.0).unwrap_or_default() }
|
||
}
|
||
}
|
||
}
|
||
|
||
pub fn stop() {
|
||
STOP.store(true, Ordering::SeqCst);
|
||
}
|
||
}
|
||
|
||
#[cfg(not(any(target_os = "macos", target_os = "linux")))]
|
||
mod mac {
|
||
pub fn start() -> Result<(), String> {
|
||
Err("這個系統還不能錄製示範".into())
|
||
}
|
||
pub fn stop() {}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn frames_become_a_video() {
|
||
let dir = std::env::temp_dir().join(format!("anybot-demo-test-{}", std::process::id()));
|
||
std::fs::create_dir_all(&dir).unwrap();
|
||
for i in 0..4u8 {
|
||
image::RgbImage::from_pixel(64, 40, image::Rgb([i * 60, 100, 200])).save(dir.join(format!("frame_{i:05}.jpg"))).unwrap();
|
||
}
|
||
let v = make_video(&dir);
|
||
if std::process::Command::new("ffmpeg").env("PATH", crate::cloudbox::tool_path()).arg("-version").output().is_ok() {
|
||
let v = v.expect("video made");
|
||
assert!(std::fs::metadata(&v).unwrap().len() > 0);
|
||
}
|
||
let demo = Demo { name: "報表".into(), created: String::new(), steps: vec!["點 (1, 2)".into()], video: String::new(), shots: vec![dir.join("frame_00002.jpg").display().to_string()] };
|
||
let (t, ok, img) = view_step(std::slice::from_ref(&demo), r#"{"demo":"報表","step":1}"#);
|
||
assert!(ok && img.is_some() && t.contains("第 1 步"), "{t}");
|
||
assert!(!view_step(&[demo], r#"{"demo":"報表","step":5}"#).1);
|
||
let _ = std::fs::remove_dir_all(&dir);
|
||
}
|
||
|
||
#[test]
|
||
fn steps_read_like_instructions() {
|
||
let t0 = Instant::now();
|
||
let ms = |n: u64| t0 + std::time::Duration::from_millis(n);
|
||
let raw = vec![
|
||
Raw::Click { x: 100.0, y: 50.0, right: false, app: "Safari".into(), at: ms(0) },
|
||
Raw::Key { text: "h".into(), code: 4, flags: 0, app: "Safari".into() },
|
||
Raw::Key { text: "i".into(), code: 34, flags: 0, app: "Safari".into() },
|
||
Raw::Key { text: "x".into(), code: 7, flags: 0, app: "Safari".into() },
|
||
Raw::Key { text: "".into(), code: 51, flags: 0, app: "Safari".into() },
|
||
Raw::Key { text: "\r".into(), code: 36, flags: 0, app: "Safari".into() },
|
||
Raw::Key { text: "c".into(), code: 8, flags: CMD, app: "Safari".into() },
|
||
Raw::Click { x: 300.0, y: 200.0, right: false, app: "備忘錄".into(), at: ms(2000) },
|
||
Raw::Click { x: 301.0, y: 201.0, right: false, app: "備忘錄".into(), at: ms(2200) },
|
||
Raw::Scroll { dy: -3, app: "備忘錄".into(), at: ms(3000) },
|
||
Raw::Scroll { dy: -2, app: "備忘錄".into(), at: ms(3300) },
|
||
];
|
||
let s: Vec<String> = compile(&raw, 0.5).into_iter().map(|c| c.0).collect();
|
||
assert_eq!(
|
||
s,
|
||
vec![
|
||
"切到「Safari」，在「Safari」點 (50, 25)",
|
||
"在「Safari」輸入「hi」",
|
||
"按 return",
|
||
"按 cmd+c",
|
||
"切到「備忘錄」，在「備忘錄」連點 2 下 (150, 100)",
|
||
"在「備忘錄」往下捲動",
|
||
]
|
||
);
|
||
}
|
||
}
|