333 lines
13 KiB
Rust
333 lines
13 KiB
Rust
// Linux Wayland on GNOME and KDE: the desktop portals, the way these desktops let an app see the
|
||
// screen and use the mouse and keyboard.
|
||
// - RemoteDesktop (+ ScreenCast for the screens): the desktop asks the person once
|
||
// ("AnyBot wants to control your screen"); the answer is remembered with a restore token
|
||
// kept in AnyBot's folder, so later sessions start without asking (portal version 2+;
|
||
// older desktops such as Ubuntu 22.04 ask once each time AnyBot starts).
|
||
// - Screenshot: a still picture of the screen, without a dialog.
|
||
// Characters the keyboard layout has are sent as keysyms; others (Chinese…) go through the
|
||
// clipboard and Ctrl+V, because these desktops only press keys their layout has.
|
||
|
||
use std::sync::mpsc::{channel, Sender};
|
||
use std::sync::Mutex;
|
||
use std::time::Duration;
|
||
|
||
use ashpd::desktop::remote_desktop::{Axis, DeviceType, KeyState, RemoteDesktop, SelectDevicesOptions};
|
||
use ashpd::desktop::screencast::{Screencast, SelectSourcesOptions, SourceType};
|
||
use ashpd::desktop::PersistMode;
|
||
|
||
#[derive(Debug)]
|
||
enum Cmd {
|
||
Motion(f64, f64),
|
||
Button(i32, bool),
|
||
Keysym(u32, bool),
|
||
Wheel(Axis, i32),
|
||
}
|
||
|
||
/// A screen the session can point at: its PipeWire node and logical place.
|
||
#[derive(Clone, Copy, Debug, Default)]
|
||
pub struct Screen {
|
||
node: u32,
|
||
x: i32,
|
||
y: i32,
|
||
w: i32,
|
||
h: i32,
|
||
}
|
||
|
||
struct Live {
|
||
tx: Sender<Cmd>,
|
||
screens: Vec<Screen>,
|
||
}
|
||
|
||
static LIVE: Mutex<Option<Live>> = Mutex::new(None);
|
||
static LAST_ERROR: Mutex<String> = Mutex::new(String::new());
|
||
|
||
fn token_file() -> std::path::PathBuf {
|
||
crate::model::State::path().parent().map(|d| d.join("wayland-remote-token.txt")).unwrap_or_default()
|
||
}
|
||
|
||
/// The portal offers remote control here (doesn't ask the person anything).
|
||
pub fn available() -> bool {
|
||
futures_lite::future::block_on(async {
|
||
match RemoteDesktop::new().await {
|
||
Ok(rd) => rd.version() >= 1,
|
||
Err(_) => false,
|
||
}
|
||
})
|
||
}
|
||
|
||
/// Why the last start failed (for the bot's message).
|
||
pub fn last_error() -> String {
|
||
LAST_ERROR.lock().unwrap().clone()
|
||
}
|
||
|
||
/// Start the session if it isn't running (the first time, the desktop asks the person).
|
||
fn ensure() -> Option<Vec<Screen>> {
|
||
if let Some(l) = LIVE.lock().unwrap().as_ref() {
|
||
return Some(l.screens.clone());
|
||
}
|
||
let (ready_tx, ready_rx) = channel::<Result<Vec<Screen>, String>>();
|
||
let (tx, rx) = channel::<Cmd>();
|
||
std::thread::spawn(move || {
|
||
futures_lite::future::block_on(async move {
|
||
let started = async {
|
||
let rd = RemoteDesktop::new().await.map_err(|e| e.to_string())?;
|
||
let session = rd.create_session(Default::default()).await.map_err(|e| e.to_string())?;
|
||
let token = std::fs::read_to_string(token_file()).ok().map(|s| s.trim().to_string()).filter(|s| !s.is_empty());
|
||
// Version 2 can remember the person's answer (older desktops ask every time AnyBot starts).
|
||
let mut devices = SelectDevicesOptions::default().set_devices(DeviceType::Keyboard | DeviceType::Pointer);
|
||
if rd.version() >= 2 {
|
||
devices = devices.set_persist_mode(PersistMode::ExplicitlyRevoked).set_restore_token(token.as_deref());
|
||
}
|
||
rd.select_devices(&session, devices)
|
||
.await
|
||
.map_err(|e| e.to_string())?;
|
||
let sc = Screencast::new().await.map_err(|e| e.to_string())?;
|
||
sc.select_sources(&session, SelectSourcesOptions::default().set_sources(ashpd::enumflags2::BitFlags::from(SourceType::Monitor)).set_multiple(true))
|
||
.await
|
||
.map_err(|e| e.to_string())?;
|
||
let res = rd.start(&session, None, Default::default()).await.map_err(|e| e.to_string())?.response().map_err(|e| match e {
|
||
ashpd::Error::Response(_) => "你在系統視窗裡沒有允許 AnyBot 操作畫面".to_string(),
|
||
e => e.to_string(),
|
||
})?;
|
||
// GNOME's dialog has an 「允許遠端操作」 switch that starts off: sharing alone lets
|
||
// AnyBot see but not use the screen.
|
||
if !res.devices().contains(DeviceType::Pointer) || !res.devices().contains(DeviceType::Keyboard) {
|
||
let _ = session.close().await;
|
||
return Err("系統視窗裡的「允許遠端操作」沒有打開,所以 AnyBot 只能看、不能操作。請再試一次,這次先打開「允許遠端操作」再按「分享」".to_string());
|
||
}
|
||
if let Some(t) = res.restore_token() {
|
||
let _ = std::fs::write(token_file(), t);
|
||
}
|
||
let screens: Vec<Screen> = res
|
||
.streams()
|
||
.iter()
|
||
.map(|s| {
|
||
let (x, y) = s.position().unwrap_or((0, 0));
|
||
let (w, h) = s.size().unwrap_or((0, 0));
|
||
Screen { node: s.pipe_wire_node_id(), x, y, w, h }
|
||
})
|
||
.collect();
|
||
if screens.is_empty() {
|
||
return Err("系統沒有給 AnyBot 任何螢幕".to_string());
|
||
}
|
||
Ok::<_, String>((rd, session, screens))
|
||
};
|
||
let (rd, session, screens) = match started.await {
|
||
Ok(v) => v,
|
||
Err(e) => {
|
||
let _ = ready_tx.send(Err(e));
|
||
return;
|
||
}
|
||
};
|
||
// GNOME drops the first event of a new virtual keyboard: wake it with a Shift tap
|
||
// (no visible effect) before AnyBot types anything.
|
||
for p in [KeyState::Pressed, KeyState::Released] {
|
||
let _ = rd.notify_keyboard_keysym(&session, 0xffe1, p, Default::default()).await;
|
||
}
|
||
async_io::Timer::after(Duration::from_millis(150)).await;
|
||
let _ = ready_tx.send(Ok(screens.clone()));
|
||
// Serve commands until AnyBot stops using the session (or the desktop ends it).
|
||
while let Ok(cmd) = rx.recv() {
|
||
let r = match cmd {
|
||
Cmd::Motion(x, y) => {
|
||
let s = screens.iter().find(|s| x >= s.x as f64 && y >= s.y as f64 && x < (s.x + s.w) as f64 && y < (s.y + s.h) as f64).or(screens.first()).copied().unwrap_or_default();
|
||
// Twice: a new virtual pointer can miss its first move, and a click must
|
||
// never land where the mouse was before.
|
||
let (lx, ly) = ((x - s.x as f64).max(0.0), (y - s.y as f64).max(0.0));
|
||
let _ = rd.notify_pointer_motion_absolute(&session, s.node, lx, ly, Default::default()).await;
|
||
rd.notify_pointer_motion_absolute(&session, s.node, lx, ly, Default::default()).await
|
||
}
|
||
Cmd::Button(b, p) => rd.notify_pointer_button(&session, b, if p { KeyState::Pressed } else { KeyState::Released }, Default::default()).await,
|
||
Cmd::Keysym(k, p) => rd.notify_keyboard_keysym(&session, k as i32, if p { KeyState::Pressed } else { KeyState::Released }, Default::default()).await,
|
||
Cmd::Wheel(axis, steps) => rd.notify_pointer_axis_discrete(&session, axis, steps, Default::default()).await,
|
||
};
|
||
if r.is_err() {
|
||
break;
|
||
}
|
||
}
|
||
let _ = session.close().await;
|
||
*LIVE.lock().unwrap() = None;
|
||
});
|
||
});
|
||
match ready_rx.recv_timeout(Duration::from_secs(120)) {
|
||
Ok(Ok(screens)) => {
|
||
*LIVE.lock().unwrap() = Some(Live { tx, screens: screens.clone() });
|
||
Some(screens)
|
||
}
|
||
Ok(Err(e)) => {
|
||
*LAST_ERROR.lock().unwrap() = e;
|
||
None
|
||
}
|
||
Err(_) => {
|
||
*LAST_ERROR.lock().unwrap() = "等太久了,系統的允許視窗沒有回應".into();
|
||
None
|
||
}
|
||
}
|
||
}
|
||
|
||
fn send(cmd: Cmd) {
|
||
if ensure().is_none() {
|
||
return;
|
||
}
|
||
let ok = LIVE.lock().unwrap().as_ref().map(|l| l.tx.send(cmd).is_ok()).unwrap_or(false);
|
||
if !ok {
|
||
*LIVE.lock().unwrap() = None;
|
||
}
|
||
// Commands are asynchronous; a short gap keeps press/release order sane for the desktop.
|
||
std::thread::sleep(Duration::from_millis(4));
|
||
}
|
||
|
||
/// The screens' bounding box in logical points.
|
||
pub fn screen_points() -> (f64, f64) {
|
||
let Some(screens) = ensure() else { return (0.0, 0.0) };
|
||
let w = screens.iter().map(|s| s.x + s.w).max().unwrap_or(0);
|
||
let h = screens.iter().map(|s| s.y + s.h).max().unwrap_or(0);
|
||
(w as f64, h as f64)
|
||
}
|
||
|
||
static PIXELS: Mutex<(u32, u32)> = Mutex::new((0, 0));
|
||
|
||
pub fn screen_pixels() -> (u32, u32) {
|
||
let p = *PIXELS.lock().unwrap();
|
||
if p.0 > 0 {
|
||
return p;
|
||
}
|
||
let (w, h) = screen_points();
|
||
(w as u32, h as u32)
|
||
}
|
||
|
||
/// A `file://` URI's path.
|
||
pub fn file_path_of(uri: &str) -> Option<std::path::PathBuf> {
|
||
let rest = uri.strip_prefix("file://")?;
|
||
let bytes = rest.as_bytes();
|
||
let mut out = Vec::with_capacity(bytes.len());
|
||
let mut i = 0;
|
||
while i < bytes.len() {
|
||
if bytes[i] == b'%' && i + 2 < bytes.len() {
|
||
if let Ok(b) = u8::from_str_radix(std::str::from_utf8(&bytes[i + 1..i + 3]).ok()?, 16) {
|
||
out.push(b);
|
||
i += 3;
|
||
continue;
|
||
}
|
||
}
|
||
out.push(bytes[i]);
|
||
i += 1;
|
||
}
|
||
Some(std::path::PathBuf::from(String::from_utf8(out).ok()?))
|
||
}
|
||
|
||
pub fn capture() -> Result<image::RgbImage, String> {
|
||
if ensure().is_none() {
|
||
return Err(format!("AnyBot 還不能看這台電腦的畫面:{}。", last_error()));
|
||
}
|
||
let uri = futures_lite::future::block_on(async {
|
||
let req = ashpd::desktop::screenshot::Screenshot::request().interactive(false).modal(false).send().await.map_err(|e| e.to_string())?;
|
||
req.response().map(|s| s.uri().as_str().to_string()).map_err(|e| e.to_string())
|
||
})
|
||
.map_err(|e| format!("Wayland 截圖失敗:{e}"))?;
|
||
let path = file_path_of(&uri).ok_or("截圖的位置看不懂")?;
|
||
let img = image::open(&path).map_err(|e| e.to_string());
|
||
// The desktop saved it (e.g. in 圖片/Screenshots); it was only for the bot.
|
||
let _ = std::fs::remove_file(&path);
|
||
let img = img?.to_rgb8();
|
||
*PIXELS.lock().unwrap() = img.dimensions();
|
||
Ok(img)
|
||
}
|
||
|
||
pub fn motion(x: f64, y: f64) {
|
||
send(Cmd::Motion(x, y));
|
||
}
|
||
|
||
/// 1 left, 2 middle, 3 right → Linux button codes.
|
||
pub fn button(b: u8, press: bool) {
|
||
let code = match b {
|
||
3 => 0x111,
|
||
2 => 0x112,
|
||
_ => 0x110,
|
||
};
|
||
send(Cmd::Button(code, press));
|
||
}
|
||
|
||
pub fn wheel(up: i32, left: i32) {
|
||
if up != 0 {
|
||
send(Cmd::Wheel(Axis::Vertical, -up));
|
||
}
|
||
if left != 0 {
|
||
send(Cmd::Wheel(Axis::Horizontal, -left));
|
||
}
|
||
}
|
||
|
||
pub fn key(sym: u32, press: bool) {
|
||
send(Cmd::Keysym(sym, press));
|
||
}
|
||
|
||
/// Typed directly: plain keyboard characters. The rest go through the clipboard.
|
||
pub fn typable(c: char) -> bool {
|
||
c == '\n' || c == '\t' || (' '..='~').contains(&c)
|
||
}
|
||
|
||
fn tap(sym: u32) {
|
||
key(sym, true);
|
||
std::thread::sleep(Duration::from_millis(8));
|
||
key(sym, false);
|
||
std::thread::sleep(Duration::from_millis(8));
|
||
}
|
||
|
||
pub fn type_text(text: &str) {
|
||
let mut rest = String::new();
|
||
let flush_paste = |s: &mut String| {
|
||
if s.is_empty() {
|
||
return;
|
||
}
|
||
paste(s);
|
||
s.clear();
|
||
};
|
||
for c in text.chars() {
|
||
if typable(c) {
|
||
flush_paste(&mut rest);
|
||
tap(crate::x11::keysym_for_char(c));
|
||
} else {
|
||
rest.push(c);
|
||
}
|
||
}
|
||
flush_paste(&mut rest);
|
||
}
|
||
|
||
/// Put text on the clipboard, press Ctrl+V, then put back what was there.
|
||
fn paste(text: &str) {
|
||
use std::io::Write;
|
||
use std::process::{Command, Stdio};
|
||
let old = Command::new("wl-paste").args(["-n", "-t", "text"]).stderr(Stdio::null()).output().ok().filter(|o| o.status.success()).map(|o| o.stdout);
|
||
let set = |bytes: &[u8]| -> bool {
|
||
let Ok(mut child) = Command::new("wl-copy").stdin(Stdio::piped()).stdout(Stdio::null()).stderr(Stdio::null()).spawn() else { return false };
|
||
let ok = child.stdin.take().map(|mut s| s.write_all(bytes).is_ok()).unwrap_or(false);
|
||
ok && child.wait().map(|s| s.success()).unwrap_or(false)
|
||
};
|
||
if !set(text.as_bytes()) {
|
||
*LAST_ERROR.lock().unwrap() = "要打中文需要 wl-clipboard(sudo apt install wl-clipboard)".into();
|
||
return;
|
||
}
|
||
std::thread::sleep(Duration::from_millis(60));
|
||
key(0xffe3, true);
|
||
tap(0x76);
|
||
key(0xffe3, false);
|
||
std::thread::sleep(Duration::from_millis(250));
|
||
if let Some(old) = old {
|
||
set(&old);
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn uri_paths_and_typing() {
|
||
assert_eq!(file_path_of("file:///home/a/Pictures/Screenshot%20From%202026.png").unwrap(), std::path::PathBuf::from("/home/a/Pictures/Screenshot From 2026.png"));
|
||
assert_eq!(file_path_of("file:///tmp/%E5%9C%96.png").unwrap(), std::path::PathBuf::from("/tmp/圖.png"));
|
||
assert!(file_path_of("https://x").is_none());
|
||
assert!(typable('a') && typable('\n') && !typable('你') && !typable('é'));
|
||
}
|
||
}
|