// 示範教學: record the person doing a task on this Mac so a bot can learn the steps and do it // again later. The original: 「錄下你執行任務的過程。{Bot} 會學習這些步驟,之後可以自行再次執行。」 // and 「{Bot} 正在觀看並學習」 (it records the screen with ffmpeg plus an input-event log; see // local-exec-daemon saveRecordingDataPackage {inputEvents, displayWidth, apiWidth…}). // Here a listen-only event tap collects clicks, keys and scrolls (nothing is blocked or changed), // and they're written up as numbered steps with screenshot coordinates. use serde::{Deserialize, Serialize}; use std::sync::Mutex; use std::time::Instant; /// Two clicks closer than this (and near each other) are a double click (estimated). const DOUBLE_CLICK_MS: u128 = 400; /// Scrolls closer than this are one step (estimated). const SCROLL_MERGE_MS: u128 = 1500; #[derive(Clone, Serialize, Deserialize, Debug, PartialEq)] pub struct Demo { pub name: String, pub created: String, pub steps: Vec, /// The screen recording (mp4), like the original's ffmpeg recording; empty when none. #[serde(default)] pub video: String, /// The screen at each step (a JPEG per step; empty when not captured). #[serde(default)] pub shots: Vec, } #[derive(Clone, Debug)] enum Raw { Click { x: f64, y: f64, right: bool, app: String, at: Instant }, Key { text: String, code: i64, flags: u64, app: String }, Scroll { dy: i64, app: String, at: Instant }, } static EVENTS: Mutex> = Mutex::new(Vec::new()); /// The frame number showing on screen when each event happened (same order as EVENTS). static EVENT_FRAMES: Mutex> = Mutex::new(Vec::new()); static FRAMES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); static FRAME_STOP: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); static FRAME_HANDLE: Mutex>> = Mutex::new(None); static REC_DIR: Mutex> = Mutex::new(None); /// Screen pictures per second while recording (estimated; the original records video). pub const FRAMES_PER_SEC: u64 = 2; /// Width of the recorded pictures (the original's render proxy is 1080p; smaller here). pub const FRAME_WIDTH: u32 = 1280; #[cfg_attr(not(target_os = "macos"), allow(dead_code))] static RUNLOOP: Mutex = Mutex::new(0); pub static RECORDING: Mutex> = Mutex::new(None); pub fn recording() -> Option<(String, Instant)> { RECORDING.lock().unwrap().clone() } /// Start recording for a bot. Err when macOS hasn't allowed listening to input yet. pub fn start(bot_name: &str) -> Result<(), String> { if RECORDING.lock().unwrap().is_some() { return Err("已經在錄了".into()); } EVENTS.lock().unwrap().clear(); EVENT_FRAMES.lock().unwrap().clear(); // Zero the frame counter before the listener starts: events that arrive in the first // half second used to carry the *previous* recording's frame numbers (start_frames // cleared it only after mac::start), mapping steps to pictures from an old demo. FRAMES.store(0, std::sync::atomic::Ordering::SeqCst); mac::start()?; *RECORDING.lock().unwrap() = Some((bot_name.to_string(), Instant::now())); start_frames(); Ok(()) } fn demos_dir() -> std::path::PathBuf { crate::model::State::path().parent().map(|p| p.to_path_buf()).unwrap_or_else(std::env::temp_dir).join("demos") } /// Pictures of the screen while recording: frame_00000.jpg, frame_00001.jpg… fn start_frames() { let dir = demos_dir().join(chrono::Local::now().format("%Y%m%d-%H%M%S").to_string()); if std::fs::create_dir_all(&dir).is_err() { return; } *REC_DIR.lock().unwrap() = Some(dir.clone()); FRAMES.store(0, std::sync::atomic::Ordering::SeqCst); FRAME_STOP.store(false, std::sync::atomic::Ordering::SeqCst); *FRAME_HANDLE.lock().unwrap() = Some(std::thread::spawn(move || { while !FRAME_STOP.load(std::sync::atomic::Ordering::SeqCst) { let t0 = Instant::now(); if crate::computer::screen_supported() && crate::computer::screen_capture_ok() { if let Ok(img) = crate::computer::capture_screen() { { let (w, h) = img.dimensions(); let small = if w > FRAME_WIDTH { image::imageops::resize(&img, FRAME_WIDTH, (h as u64 * FRAME_WIDTH as u64 / w.max(1) as u64) as u32, image::imageops::FilterType::Triangle) } else { img }; let n = FRAMES.load(std::sync::atomic::Ordering::SeqCst); if small.save(dir.join(format!("frame_{n:05}.jpg"))).is_ok() { FRAMES.fetch_add(1, std::sync::atomic::Ordering::SeqCst); } } } } let wait = std::time::Duration::from_millis(1000 / FRAMES_PER_SEC); if let Some(rest) = wait.checked_sub(t0.elapsed()) { std::thread::sleep(rest); } } })); } /// Stop the pictures and turn them into a video when ffmpeg is there. Returns (dir, video). fn stop_frames() -> Option<(std::path::PathBuf, String)> { FRAME_STOP.store(true, std::sync::atomic::Ordering::SeqCst); // Let the capturing thread finish its current picture and exit (it ends within a second). // The old fixed 700 ms sleep raced slow captures (dirty files after the video was made) // and froze the window for its whole duration. if let Some(h) = FRAME_HANDLE.lock().unwrap().take() { let _ = h.join(); } let dir = REC_DIR.lock().unwrap().take()?; // Encoding runs off the window thread (ffmpeg on a long demo takes seconds; the window // used to freeze for all of it). The mp4 file appears when it is done. if dir.join("frame_00000.jpg").exists() { let d = dir.clone(); std::thread::spawn(move || { if make_video(&d).is_some() { // The video holds the motion; drop the single pictures it consumed. if let Ok(rd) = std::fs::read_dir(&d) { for e in rd.flatten() { if e.file_name().to_string_lossy().starts_with("frame_") { let _ = std::fs::remove_file(e.path()); } } } } }); return Some((dir.clone(), dir.join("recording.mp4").display().to_string())); } Some((dir, String::new())) } /// frame_*.jpg → recording.mp4 (H.264, like the original's render proxy). pub fn make_video(dir: &std::path::Path) -> Option { if !dir.join("frame_00000.jpg").exists() { return None; } let out = dir.join("recording.mp4"); let ok = std::process::Command::new("ffmpeg") .env("PATH", crate::cloudbox::tool_path()) .args(["-hide_banner", "-loglevel", "error", "-y", "-framerate", &FRAMES_PER_SEC.to_string(), "-i"]) .arg(dir.join("frame_%05d.jpg")) .args(["-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2,format=yuv420p", "-c:v", "libx264", "-preset", "veryfast", "-movflags", "+faststart"]) .arg(&out) .stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) .status() .map(|s| s.success()) .unwrap_or(false); ok.then(|| out.display().to_string()) } /// Stop and write up the steps. pub fn finish(name: &str) -> Demo { mac::stop(); // The listener stops asynchronously (the X11 one polls every 150 ms): give the last // events of the demo time to arrive — they used to be cut off the recording. std::thread::sleep(std::time::Duration::from_millis(300)); *RECORDING.lock().unwrap() = None; let raw = std::mem::take(&mut *EVENTS.lock().unwrap()); let frames_at = std::mem::take(&mut *EVENT_FRAMES.lock().unwrap()); let compiled = compile(&raw, crate::computer::points_to_shot()); let rec = stop_frames(); let mut shots = vec![]; let mut video = String::new(); if let Some((dir, v)) = &rec { video = v.clone(); for (i, (_, first)) in compiled.iter().enumerate() { // The picture just before the step happened (what the person was looking at). let f = frames_at.get(*first).copied().unwrap_or(0); let src = dir.join(format!("frame_{f:05}.jpg")); let dst = dir.join(format!("step_{:02}.jpg", i + 1)); shots.push(if std::fs::copy(&src, &dst).is_ok() { dst.display().to_string() } else { String::new() }); } // The frame pictures are removed by the encoder thread once the video is written. } Demo { name: name.trim().to_string(), created: chrono::Local::now().format("%Y-%m-%d %H:%M").to_string(), steps: compiled.into_iter().map(|c| c.0).collect(), video, shots } } pub fn cancel() { mac::stop(); *RECORDING.lock().unwrap() = None; EVENTS.lock().unwrap().clear(); EVENT_FRAMES.lock().unwrap().clear(); if let Some((dir, _)) = stop_frames() { let _ = std::fs::remove_dir_all(dir); } } /// The view_demo_step tool: the screen at one step of a demonstration, for the bot to look at. pub fn view_step_spec() -> serde_json::Value { serde_json::json!({"name": "view_demo_step", "description": "See what the person's screen looked like at one step of a demonstration they recorded for you (a picture), to find the right buttons when you repeat it.", "parameters": {"type": "object", "properties": {"demo": {"type": "string", "description": "The demonstration's name"}, "step": {"type": "integer", "description": "Step number, from 1"}}, "required": ["demo", "step"]}}) } /// Returns (text, ok, picture as base64 JPEG). pub fn view_step(demos: &[Demo], args: &str) -> (String, bool, Option) { use base64::Engine; let v: serde_json::Value = serde_json::from_str(args).unwrap_or_default(); let name = v["demo"].as_str().unwrap_or("").trim(); let Some(d) = demos.iter().find(|d| d.name == name).or_else(|| demos.iter().find(|d| d.name.contains(name) && !name.is_empty())) else { return (format!("沒有叫「{name}」的示範。有:{}", demos.iter().map(|d| d.name.as_str()).collect::>().join("、")), false, None); }; let n = v["step"].as_u64().unwrap_or(0) as usize; if n == 0 || n > d.steps.len() { return (format!("「{}」有 {} 個步驟,請給 1 到 {}。", d.name, d.steps.len(), d.steps.len()), false, None); } let text = format!("「{}」第 {n} 步:{}", d.name, d.steps[n - 1]); let Some(path) = d.shots.get(n - 1).filter(|p| !p.is_empty()) else { return (format!("{text}\n(這一步沒有存到畫面)"), true, None); }; match std::fs::read(path).ok().and_then(|b| image::load_from_memory(&b).ok()) { Some(img) => { let img = img.to_rgb8(); let (w, h) = img.dimensions(); let (fw, fh, resize) = crate::computer::fit(w, h); let img = if resize { image::imageops::resize(&img, fw, fh, image::imageops::FilterType::Triangle) } else { img }; match crate::computer::encode_jpeg(&img) { Some(j) => (format!("{text}\n這是使用者做這一步之前的畫面({fw}×{fh},座標和步驟裡的一樣)。"), true, Some(base64::engine::general_purpose::STANDARD.encode(j))), None => (format!("{text}\n(畫面太大,沒辦法送出)"), true, None), } } None => (format!("{text}\n(畫面檔案不見了)"), true, None), } } fn key_name(code: i64) -> Option<&'static str> { Some(match code { 36 => "return", 48 => "tab", 49 => "space", 51 => "delete", 53 => "escape", 117 => "forwarddelete", 123 => "left", 124 => "right", 125 => "down", 126 => "up", 115 => "home", 119 => "end", 116 => "pageup", 121 => "pagedown", _ => return None, }) } const CMD: u64 = 1 << 20; const SHIFT: u64 = 1 << 17; const CTRL: u64 = 1 << 18; const OPT: u64 = 1 << 19; /// Raw events → steps a bot can follow. fn compile(raw: &[Raw], to_shot: f64) -> Vec<(String, usize)> { let mut steps: Vec<(String, usize)> = vec![]; let mut typed_from = 0usize; let mut typed = String::new(); let mut typed_app = String::new(); let mut last_app = String::new(); let flush = |steps: &mut Vec<(String, usize)>, typed: &mut String, app: &str, from: usize| { if !typed.is_empty() { steps.push((format!("在「{app}」輸入「{typed}」"), from)); typed.clear(); } }; let mut i = 0; while i < raw.len() { match &raw[i] { Raw::Click { x, y, right, app, at } => { flush(&mut steps, &mut typed, &typed_app, typed_from); let start = i; let mut count = 1; while let Some(Raw::Click { x: x2, y: y2, right: r2, at: t2, .. }) = raw.get(i + 1) { if r2 == right && (x2 - x).abs() < 6.0 && (y2 - y).abs() < 6.0 && t2.duration_since(*at).as_millis() < DOUBLE_CLICK_MS * count as u128 { count += 1; i += 1; } else { break; } } let verb = match (right, count) { (true, _) => "按右鍵".to_string(), (false, 1) => "點".to_string(), (false, n) => format!("連點 {n} 下"), }; let switch = if *app != last_app && !app.is_empty() { format!("切到「{app}」,") } else { String::new() }; steps.push((format!("{switch}在「{app}」{verb} ({:.0}, {:.0})", x * to_shot, y * to_shot), start)); last_app = app.clone(); } Raw::Key { text, code, flags, app } => { let mods: Vec<&str> = [(CMD, "cmd"), (CTRL, "control"), (OPT, "option")].iter().filter(|(f, _)| flags & f != 0).map(|(_, n)| *n).collect(); if mods.is_empty() && *code == 51 && !typed.is_empty() { typed.pop(); } else if mods.is_empty() && key_name(*code).is_none() && !text.is_empty() && !text.chars().any(char::is_control) { if typed.is_empty() { typed_app = app.clone(); typed_from = i; } typed.push_str(text); } else { flush(&mut steps, &mut typed, &typed_app, typed_from); let key = key_name(*code).map(String::from).unwrap_or_else(|| text.to_lowercase()); if key.is_empty() { i += 1; continue; } let mut combo: Vec = mods.iter().map(|m| m.to_string()).collect(); if flags & SHIFT != 0 && !mods.is_empty() { combo.push("shift".into()); } combo.push(key); steps.push((format!("按 {}", combo.join("+")), i)); } last_app = app.clone(); } Raw::Scroll { dy, app, at } => { flush(&mut steps, &mut typed, &typed_app, typed_from); let start = i; let mut total = *dy; let mut last = *at; while let Some(Raw::Scroll { dy: d2, at: t2, .. }) = raw.get(i + 1) { if t2.duration_since(last).as_millis() < SCROLL_MERGE_MS && d2.signum() == dy.signum() { total += d2; last = *t2; i += 1; } else { break; } } steps.push((format!("在「{app}」往{}捲動", if total < 0 { "下" } else { "上" }), start)); } } i += 1; } flush(&mut steps, &mut typed, &typed_app, typed_from); steps } /// The steps as told to the bot. pub fn prompt_section(demos: &[Demo]) -> String { if demos.is_empty() { return String::new(); } let mut s = String::from("\n\n你學過的示範(使用者在他的 Mac 上做給你看的步驟)。使用者要你做同樣的事時,照著在「你的電腦」上做:開始先截圖一次,之後每一步做完都會附上新畫面,座標是錄製時截圖上的位置,畫面不同就找對應的按鈕;輸入的內容可以依新的需求調整。想看某一步當時的畫面,用 view_demo_step。"); for d in demos { s.push_str(&format!("\n### {}({})\n", d.name, d.created)); for (i, st) in d.steps.iter().enumerate() { s.push_str(&format!("{}. {st}\n", i + 1)); } } s } fn push(e: Raw) { // AnyBot's own window (its Stop button, the bar) isn't part of the task. let own = match &e { Raw::Click { app, .. } | Raw::Key { app, .. } | Raw::Scroll { app, .. } => app.to_lowercase().contains("anybot"), }; if !own { EVENTS.lock().unwrap().push(e); // The last picture taken before this moment. EVENT_FRAMES.lock().unwrap().push(FRAMES.load(std::sync::atomic::Ordering::SeqCst).saturating_sub(1)); } } #[cfg(target_os = "macos")] mod mac { use super::*; use std::ffi::c_void; #[repr(C)] #[derive(Clone, Copy)] struct CGPoint { x: f64, y: f64, } type Cb = extern "C" fn(*mut c_void, u32, *mut c_void, *mut c_void) -> *mut c_void; #[link(name = "CoreGraphics", kind = "framework")] extern "C" { fn CGEventTapCreate(tap: u32, place: u32, options: u32, mask: u64, cb: Cb, user: *mut c_void) -> *mut c_void; fn CGEventTapEnable(tap: *mut c_void, enable: bool); fn CGEventGetLocation(e: *mut c_void) -> CGPoint; fn CGEventGetFlags(e: *mut c_void) -> u64; fn CGEventGetIntegerValueField(e: *mut c_void, field: u32) -> i64; fn CGEventKeyboardGetUnicodeString(e: *mut c_void, max: usize, actual: *mut usize, buf: *mut u16); } #[link(name = "CoreFoundation", kind = "framework")] extern "C" { fn CFMachPortCreateRunLoopSource(alloc: *const c_void, port: *mut c_void, order: isize) -> *mut c_void; fn CFRunLoopGetCurrent() -> *mut c_void; fn CFRunLoopAddSource(rl: *mut c_void, src: *mut c_void, mode: *const c_void); fn CFRunLoopRun(); fn CFRunLoopStop(rl: *mut c_void); static kCFRunLoopCommonModes: *const c_void; } static TAP: Mutex = Mutex::new(0); // The foreground app barely moves between keystrokes, and the window-list query is the // slowest call in the tap callback — slow callbacks make macOS disable the tap (events // lost until it is re-enabled). Answer keystrokes from a short cache. static FRONT_CACHE: Mutex> = Mutex::new(None); fn front_cached() -> String { let mut c = FRONT_CACHE.lock().unwrap_or_else(|e| e.into_inner()); if let Some((t, n)) = c.as_ref() { if t.elapsed() < std::time::Duration::from_millis(400) { return n.clone(); } } let n = crate::computer::frontmost_app().unwrap_or_default(); *c = Some((Instant::now(), n.clone())); n } extern "C" fn callback(_proxy: *mut c_void, kind: u32, e: *mut c_void, _user: *mut c_void) -> *mut c_void { unsafe { match kind { 1 | 3 => { let p = CGEventGetLocation(e); let app = crate::computer::app_at_point(p.x, p.y).unwrap_or_default(); push(Raw::Click { x: p.x, y: p.y, right: kind == 3, app, at: Instant::now() }); } 10 => { let mut buf = [0u16; 8]; let mut n = 0usize; CGEventKeyboardGetUnicodeString(e, buf.len(), &mut n, buf.as_mut_ptr()); let text = String::from_utf16_lossy(&buf[..n.min(buf.len())]); let code = CGEventGetIntegerValueField(e, 9); let app = front_cached(); push(Raw::Key { text, code, flags: CGEventGetFlags(e), app }); } 22 => { let dy = CGEventGetIntegerValueField(e, 11); if dy != 0 { let p = CGEventGetLocation(e); let app = crate::computer::app_at_point(p.x, p.y).unwrap_or_default(); push(Raw::Scroll { dy, app, at: Instant::now() }); } } // The system turned the tap off (slow callback): turn it back on. 0xFFFF_FFFE | 0xFFFF_FFFF => { let t = *TAP.lock().unwrap(); if t != 0 { CGEventTapEnable(t as *mut c_void, true); } } _ => {} } } e } pub fn start() -> Result<(), String> { let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || unsafe { let mask: u64 = (1 << 1) | (1 << 3) | (1 << 10) | (1 << 22); // kCGSessionEventTap, head insert, listen only. let tap = CGEventTapCreate(1, 0, 1, mask, callback, std::ptr::null_mut()); if tap.is_null() { let _ = tx.send(false); return; } *TAP.lock().unwrap() = tap as usize; let src = CFMachPortCreateRunLoopSource(std::ptr::null(), tap, 0); let rl = CFRunLoopGetCurrent(); CFRunLoopAddSource(rl, src, kCFRunLoopCommonModes); CGEventTapEnable(tap, true); *RUNLOOP.lock().unwrap() = rl as usize; let _ = tx.send(true); CFRunLoopRun(); CGEventTapEnable(tap, false); *TAP.lock().unwrap() = 0; }); match rx.recv_timeout(std::time::Duration::from_secs(3)) { Ok(true) => Ok(()), _ => { let _ = crate::computer::spawn_reaped(&mut std::process::Command::new("open").arg("x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent")); Err("macOS 還沒讓 AnyBot 觀看你的操作。我已經打開「系統設定 → 隱私權與安全性 → 輸入監控」:把 AnyBot(或「終端機」)打開,再按一次「開始錄製」。".into()) } } } pub fn stop() { let rl = std::mem::take(&mut *RUNLOOP.lock().unwrap()); if rl != 0 { unsafe { CFRunLoopStop(rl as *mut c_void) }; } } } /// Linux: the X RECORD extension shows every click and key (nothing is blocked or changed). #[cfg(target_os = "linux")] mod mac { use super::*; use std::sync::atomic::{AtomicBool, Ordering}; static STOP: AtomicBool = AtomicBool::new(false); pub fn start() -> Result<(), String> { if crate::desk::kind() != crate::desk::Kind::X11 { return Err(if crate::desk::wayland() { "在 Wayland 上,系統不讓 App 看到你按了哪裡、打了什麼,所以沒辦法錄製示範(Bot 還是可以操作畫面)。要錄示範,請在登入畫面改選 Xorg 工作階段。" } else { "錄製示範需要桌面畫面(X11)。" } .into()); } STOP.store(false, Ordering::SeqCst); let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { let tx2 = tx.clone(); let r = crate::x11::record( |input| push(to_raw(input)), move || { let _ = tx2.send(Ok(())); !STOP.load(Ordering::SeqCst) }, ); if let Err(e) = r { let _ = tx.send(Err(e)); } }); // On a timeout the record thread may still be spinning up: close the gate so its // keep_going() ends it — a thread left running meant a second RECORD context // (double events) on the next try, plus a leaked X connection. match rx.recv_timeout(std::time::Duration::from_secs(3)) { Ok(Ok(())) => Ok(()), Ok(Err(e)) => Err(e), Err(_) => { STOP.store(true, Ordering::SeqCst); Err("沒辦法開始錄製(X11 沒有回應)".into()) } } } /// X input → the same events the macOS tap gives (key codes and flags in macOS terms, so /// the steps read the same; Linux's Ctrl shows as "control"). pub fn to_raw(i: crate::x11::Input) -> Raw { let app = |x: f64, y: f64| crate::x11::window_at(x, y).map(|a| a.0).or_else(|| crate::x11::front_app().map(|a| a.0)).unwrap_or_default(); match i { crate::x11::Input::Click { x, y, right } => Raw::Click { x, y, right, app: app(x, y), at: Instant::now() }, crate::x11::Input::Scroll { dy } => Raw::Scroll { dy, app: crate::x11::front_app().map(|a| a.0).unwrap_or_default(), at: Instant::now() }, crate::x11::Input::Key { text, keysym, mods } => { let code = crate::x11::keysym_name(keysym) .and_then(crate::computer::key_code) .or_else(|| crate::x11::char_of_keysym(keysym).and_then(|c| crate::computer::key_code(&c.to_lowercase().to_string()))) .map(|c| c as i64) .unwrap_or(-1); let mut flags = 0; if mods & 1 != 0 { flags |= SHIFT; } if mods & 4 != 0 { flags |= CTRL; } if mods & 8 != 0 { flags |= OPT; } if mods & 64 != 0 { flags |= CMD; } let text = if keysym == 0xff0d { "\r".into() } else { text }; Raw::Key { text, code, flags, app: crate::x11::front_app().map(|a| a.0).unwrap_or_default() } } } } pub fn stop() { STOP.store(true, Ordering::SeqCst); } } #[cfg(not(any(target_os = "macos", target_os = "linux")))] mod mac { pub fn start() -> Result<(), String> { Err("這個系統還不能錄製示範".into()) } pub fn stop() {} } #[cfg(test)] mod tests { use super::*; #[test] fn frames_become_a_video() { let dir = std::env::temp_dir().join(format!("anybot-demo-test-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); for i in 0..4u8 { image::RgbImage::from_pixel(64, 40, image::Rgb([i * 60, 100, 200])).save(dir.join(format!("frame_{i:05}.jpg"))).unwrap(); } let v = make_video(&dir); if std::process::Command::new("ffmpeg").env("PATH", crate::cloudbox::tool_path()).arg("-version").output().is_ok() { let v = v.expect("video made"); assert!(std::fs::metadata(&v).unwrap().len() > 0); } let demo = Demo { name: "報表".into(), created: String::new(), steps: vec!["點 (1, 2)".into()], video: String::new(), shots: vec![dir.join("frame_00002.jpg").display().to_string()] }; let (t, ok, img) = view_step(&[demo.clone()], r#"{"demo":"報表","step":1}"#); assert!(ok && img.is_some() && t.contains("第 1 步"), "{t}"); assert!(!view_step(&[demo], r#"{"demo":"報表","step":5}"#).1); let _ = std::fs::remove_dir_all(&dir); } #[test] fn steps_read_like_instructions() { let t0 = Instant::now(); let ms = |n: u64| t0 + std::time::Duration::from_millis(n); let raw = vec![ Raw::Click { x: 100.0, y: 50.0, right: false, app: "Safari".into(), at: ms(0) }, Raw::Key { text: "h".into(), code: 4, flags: 0, app: "Safari".into() }, Raw::Key { text: "i".into(), code: 34, flags: 0, app: "Safari".into() }, Raw::Key { text: "x".into(), code: 7, flags: 0, app: "Safari".into() }, Raw::Key { text: "".into(), code: 51, flags: 0, app: "Safari".into() }, Raw::Key { text: "\r".into(), code: 36, flags: 0, app: "Safari".into() }, Raw::Key { text: "c".into(), code: 8, flags: CMD, app: "Safari".into() }, Raw::Click { x: 300.0, y: 200.0, right: false, app: "備忘錄".into(), at: ms(2000) }, Raw::Click { x: 301.0, y: 201.0, right: false, app: "備忘錄".into(), at: ms(2200) }, Raw::Scroll { dy: -3, app: "備忘錄".into(), at: ms(3000) }, Raw::Scroll { dy: -2, app: "備忘錄".into(), at: ms(3300) }, ]; let s: Vec = compile(&raw, 0.5).into_iter().map(|c| c.0).collect(); assert_eq!( s, vec![ "切到「Safari」,在「Safari」點 (50, 25)", "在「Safari」輸入「hi」", "按 return", "按 cmd+c", "切到「備忘錄」,在「備忘錄」連點 2 下 (150, 100)", "在「備忘錄」往下捲動", ] ); } }