// Linux Wayland on GNOME and KDE: the desktop portals, the way these desktops let an app see the // screen and use the mouse and keyboard. // - RemoteDesktop (+ ScreenCast for the screens): the desktop asks the person once // ("AnyBot wants to control your screen"); the answer is remembered with a restore token // kept in AnyBot's folder, so later sessions start without asking (portal version 2+; // older desktops such as Ubuntu 22.04 ask once each time AnyBot starts). // - Screenshot: a still picture of the screen, without a dialog. // Characters the keyboard layout has are sent as keysyms; others (Chinese…) go through the // clipboard and Ctrl+V, because these desktops only press keys their layout has. use std::sync::mpsc::{channel, Sender}; use std::sync::Mutex; use std::time::Duration; use ashpd::desktop::remote_desktop::{Axis, DeviceType, KeyState, RemoteDesktop, SelectDevicesOptions}; use ashpd::desktop::screencast::{Screencast, SelectSourcesOptions, SourceType}; use ashpd::desktop::PersistMode; #[derive(Debug)] enum Cmd { Motion(f64, f64), Button(i32, bool), Keysym(u32, bool), Wheel(Axis, i32), } /// A screen the session can point at: its PipeWire node and logical place. #[derive(Clone, Copy, Debug, Default)] pub struct Screen { node: u32, x: i32, y: i32, w: i32, h: i32, } struct Live { tx: Sender, screens: Vec, } static LIVE: Mutex> = Mutex::new(None); static LAST_ERROR: Mutex = Mutex::new(String::new()); fn token_file() -> std::path::PathBuf { crate::model::State::path().parent().map(|d| d.join("wayland-remote-token.txt")).unwrap_or_default() } /// The portal offers remote control here (doesn't ask the person anything). pub fn available() -> bool { futures_lite::future::block_on(async { match RemoteDesktop::new().await { Ok(rd) => rd.version() >= 1, Err(_) => false, } }) } /// Why the last start failed (for the bot's message). pub fn last_error() -> String { LAST_ERROR.lock().unwrap().clone() } /// Start the session if it isn't running (the first time, the desktop asks the person). fn ensure() -> Option> { if let Some(l) = LIVE.lock().unwrap().as_ref() { return Some(l.screens.clone()); } let (ready_tx, ready_rx) = channel::, String>>(); let (tx, rx) = channel::(); std::thread::spawn(move || { futures_lite::future::block_on(async move { let started = async { let rd = RemoteDesktop::new().await.map_err(|e| e.to_string())?; let session = rd.create_session(Default::default()).await.map_err(|e| e.to_string())?; let token = std::fs::read_to_string(token_file()).ok().map(|s| s.trim().to_string()).filter(|s| !s.is_empty()); // Version 2 can remember the person's answer (older desktops ask every time AnyBot starts). let mut devices = SelectDevicesOptions::default().set_devices(DeviceType::Keyboard | DeviceType::Pointer); if rd.version() >= 2 { devices = devices.set_persist_mode(PersistMode::ExplicitlyRevoked).set_restore_token(token.as_deref()); } rd.select_devices(&session, devices) .await .map_err(|e| e.to_string())?; let sc = Screencast::new().await.map_err(|e| e.to_string())?; sc.select_sources(&session, SelectSourcesOptions::default().set_sources(ashpd::enumflags2::BitFlags::from(SourceType::Monitor)).set_multiple(true)) .await .map_err(|e| e.to_string())?; let res = rd.start(&session, None, Default::default()).await.map_err(|e| e.to_string())?.response().map_err(|e| match e { ashpd::Error::Response(_) => "你在系統視窗裡沒有允許 AnyBot 操作畫面".to_string(), e => e.to_string(), })?; // GNOME's dialog has an 「允許遠端操作」 switch that starts off: sharing alone lets // AnyBot see but not use the screen. if !res.devices().contains(DeviceType::Pointer) || !res.devices().contains(DeviceType::Keyboard) { let _ = session.close().await; return Err("系統視窗裡的「允許遠端操作」沒有打開,所以 AnyBot 只能看、不能操作。請再試一次,這次先打開「允許遠端操作」再按「分享」".to_string()); } if let Some(t) = res.restore_token() { let _ = std::fs::write(token_file(), t); } let screens: Vec = res .streams() .iter() .map(|s| { let (x, y) = s.position().unwrap_or((0, 0)); let (w, h) = s.size().unwrap_or((0, 0)); Screen { node: s.pipe_wire_node_id(), x, y, w, h } }) .collect(); if screens.is_empty() { return Err("系統沒有給 AnyBot 任何螢幕".to_string()); } Ok::<_, String>((rd, session, screens)) }; let (rd, session, screens) = match started.await { Ok(v) => v, Err(e) => { let _ = ready_tx.send(Err(e)); return; } }; // GNOME drops the first event of a new virtual keyboard: wake it with a Shift tap // (no visible effect) before AnyBot types anything. for p in [KeyState::Pressed, KeyState::Released] { let _ = rd.notify_keyboard_keysym(&session, 0xffe1, p, Default::default()).await; } async_io::Timer::after(Duration::from_millis(150)).await; let _ = ready_tx.send(Ok(screens.clone())); // Serve commands until AnyBot stops using the session (or the desktop ends it). while let Ok(cmd) = rx.recv() { let r = match cmd { Cmd::Motion(x, y) => { let s = screens.iter().find(|s| x >= s.x as f64 && y >= s.y as f64 && x < (s.x + s.w) as f64 && y < (s.y + s.h) as f64).or(screens.first()).copied().unwrap_or_default(); // Twice: a new virtual pointer can miss its first move, and a click must // never land where the mouse was before. let (lx, ly) = ((x - s.x as f64).max(0.0), (y - s.y as f64).max(0.0)); let _ = rd.notify_pointer_motion_absolute(&session, s.node, lx, ly, Default::default()).await; rd.notify_pointer_motion_absolute(&session, s.node, lx, ly, Default::default()).await } Cmd::Button(b, p) => rd.notify_pointer_button(&session, b, if p { KeyState::Pressed } else { KeyState::Released }, Default::default()).await, Cmd::Keysym(k, p) => rd.notify_keyboard_keysym(&session, k as i32, if p { KeyState::Pressed } else { KeyState::Released }, Default::default()).await, Cmd::Wheel(axis, steps) => rd.notify_pointer_axis_discrete(&session, axis, steps, Default::default()).await, }; if r.is_err() { break; } } let _ = session.close().await; *LIVE.lock().unwrap() = None; }); }); match ready_rx.recv_timeout(Duration::from_secs(120)) { Ok(Ok(screens)) => { *LIVE.lock().unwrap() = Some(Live { tx, screens: screens.clone() }); Some(screens) } Ok(Err(e)) => { *LAST_ERROR.lock().unwrap() = e; None } Err(_) => { *LAST_ERROR.lock().unwrap() = "等太久了,系統的允許視窗沒有回應".into(); None } } } fn send(cmd: Cmd) { if ensure().is_none() { return; } let ok = LIVE.lock().unwrap().as_ref().map(|l| l.tx.send(cmd).is_ok()).unwrap_or(false); if !ok { *LIVE.lock().unwrap() = None; } // Commands are asynchronous; a short gap keeps press/release order sane for the desktop. std::thread::sleep(Duration::from_millis(4)); } /// The screens' bounding box in logical points. pub fn screen_points() -> (f64, f64) { let Some(screens) = ensure() else { return (0.0, 0.0) }; let w = screens.iter().map(|s| s.x + s.w).max().unwrap_or(0); let h = screens.iter().map(|s| s.y + s.h).max().unwrap_or(0); (w as f64, h as f64) } static PIXELS: Mutex<(u32, u32)> = Mutex::new((0, 0)); pub fn screen_pixels() -> (u32, u32) { let p = *PIXELS.lock().unwrap(); if p.0 > 0 { return p; } let (w, h) = screen_points(); (w as u32, h as u32) } /// A `file://` URI's path. pub fn file_path_of(uri: &str) -> Option { let rest = uri.strip_prefix("file://")?; let bytes = rest.as_bytes(); let mut out = Vec::with_capacity(bytes.len()); let mut i = 0; while i < bytes.len() { if bytes[i] == b'%' && i + 2 < bytes.len() { if let Ok(b) = u8::from_str_radix(std::str::from_utf8(&bytes[i + 1..i + 3]).ok()?, 16) { out.push(b); i += 3; continue; } } out.push(bytes[i]); i += 1; } Some(std::path::PathBuf::from(String::from_utf8(out).ok()?)) } pub fn capture() -> Result { if ensure().is_none() { return Err(format!("AnyBot 還不能看這台電腦的畫面:{}。", last_error())); } let uri = futures_lite::future::block_on(async { let req = ashpd::desktop::screenshot::Screenshot::request().interactive(false).modal(false).send().await.map_err(|e| e.to_string())?; req.response().map(|s| s.uri().as_str().to_string()).map_err(|e| e.to_string()) }) .map_err(|e| format!("Wayland 截圖失敗:{e}"))?; let path = file_path_of(&uri).ok_or("截圖的位置看不懂")?; let img = image::open(&path).map_err(|e| e.to_string()); // The desktop saved it (e.g. in 圖片/Screenshots); it was only for the bot. let _ = std::fs::remove_file(&path); let img = img?.to_rgb8(); *PIXELS.lock().unwrap() = img.dimensions(); Ok(img) } pub fn motion(x: f64, y: f64) { send(Cmd::Motion(x, y)); } /// 1 left, 2 middle, 3 right → Linux button codes. pub fn button(b: u8, press: bool) { let code = match b { 3 => 0x111, 2 => 0x112, _ => 0x110, }; send(Cmd::Button(code, press)); } pub fn wheel(up: i32, left: i32) { if up != 0 { send(Cmd::Wheel(Axis::Vertical, -up)); } if left != 0 { send(Cmd::Wheel(Axis::Horizontal, -left)); } } pub fn key(sym: u32, press: bool) { send(Cmd::Keysym(sym, press)); } /// Typed directly: plain keyboard characters. The rest go through the clipboard. pub fn typable(c: char) -> bool { c == '\n' || c == '\t' || (' '..='~').contains(&c) } fn tap(sym: u32) { key(sym, true); std::thread::sleep(Duration::from_millis(8)); key(sym, false); std::thread::sleep(Duration::from_millis(8)); } pub fn type_text(text: &str) { let mut rest = String::new(); let flush_paste = |s: &mut String| { if s.is_empty() { return; } paste(s); s.clear(); }; for c in text.chars() { if typable(c) { flush_paste(&mut rest); tap(crate::x11::keysym_for_char(c)); } else { rest.push(c); } } flush_paste(&mut rest); } /// Put text on the clipboard, press Ctrl+V, then put back what was there. fn paste(text: &str) { use std::io::Write; use std::process::{Command, Stdio}; let old = Command::new("wl-paste").args(["-n", "-t", "text"]).stderr(Stdio::null()).output().ok().filter(|o| o.status.success()).map(|o| o.stdout); let set = |bytes: &[u8]| -> bool { let Ok(mut child) = Command::new("wl-copy").stdin(Stdio::piped()).stdout(Stdio::null()).stderr(Stdio::null()).spawn() else { return false }; let ok = child.stdin.take().map(|mut s| s.write_all(bytes).is_ok()).unwrap_or(false); ok && child.wait().map(|s| s.success()).unwrap_or(false) }; if !set(text.as_bytes()) { *LAST_ERROR.lock().unwrap() = "要打中文需要 wl-clipboard(sudo apt install wl-clipboard)".into(); return; } std::thread::sleep(Duration::from_millis(60)); key(0xffe3, true); tap(0x76); key(0xffe3, false); std::thread::sleep(Duration::from_millis(250)); if let Some(old) = old { set(&old); } } #[cfg(test)] mod tests { use super::*; #[test] fn uri_paths_and_typing() { assert_eq!(file_path_of("file:///home/a/Pictures/Screenshot%20From%202026.png").unwrap(), std::path::PathBuf::from("/home/a/Pictures/Screenshot From 2026.png")); assert_eq!(file_path_of("file:///tmp/%E5%9C%96.png").unwrap(), std::path::PathBuf::from("/tmp/圖.png")); assert!(file_path_of("https://x").is_none()); assert!(typable('a') && typable('\n') && !typable('你') && !typable('é')); } }