333 lines
13 KiB
Rust
333 lines
13 KiB
Rust
|
|
// Linux Wayland on GNOME and KDE: the desktop portals, the way these desktops let an app see the
|
|||
|
|
// screen and use the mouse and keyboard.
|
|||
|
|
// - RemoteDesktop (+ ScreenCast for the screens): the desktop asks the person once
|
|||
|
|
// ("AnyBot wants to control your screen"); the answer is remembered with a restore token
|
|||
|
|
// kept in AnyBot's folder, so later sessions start without asking (portal version 2+;
|
|||
|
|
// older desktops such as Ubuntu 22.04 ask once each time AnyBot starts).
|
|||
|
|
// - Screenshot: a still picture of the screen, without a dialog.
|
|||
|
|
// Characters the keyboard layout has are sent as keysyms; others (Chinese…) go through the
|
|||
|
|
// clipboard and Ctrl+V, because these desktops only press keys their layout has.
|
|||
|
|
|
|||
|
|
use std::sync::mpsc::{channel, Sender};
|
|||
|
|
use std::sync::Mutex;
|
|||
|
|
use std::time::Duration;
|
|||
|
|
|
|||
|
|
use ashpd::desktop::remote_desktop::{Axis, DeviceType, KeyState, RemoteDesktop, SelectDevicesOptions};
|
|||
|
|
use ashpd::desktop::screencast::{Screencast, SelectSourcesOptions, SourceType};
|
|||
|
|
use ashpd::desktop::PersistMode;
|
|||
|
|
|
|||
|
|
#[derive(Debug)]
|
|||
|
|
enum Cmd {
|
|||
|
|
Motion(f64, f64),
|
|||
|
|
Button(i32, bool),
|
|||
|
|
Keysym(u32, bool),
|
|||
|
|
Wheel(Axis, i32),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// A screen the session can point at: its PipeWire node and logical place.
|
|||
|
|
#[derive(Clone, Copy, Debug, Default)]
|
|||
|
|
pub struct Screen {
|
|||
|
|
node: u32,
|
|||
|
|
x: i32,
|
|||
|
|
y: i32,
|
|||
|
|
w: i32,
|
|||
|
|
h: i32,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
struct Live {
|
|||
|
|
tx: Sender<Cmd>,
|
|||
|
|
screens: Vec<Screen>,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
static LIVE: Mutex<Option<Live>> = Mutex::new(None);
|
|||
|
|
static LAST_ERROR: Mutex<String> = Mutex::new(String::new());
|
|||
|
|
|
|||
|
|
fn token_file() -> std::path::PathBuf {
|
|||
|
|
crate::model::State::path().parent().map(|d| d.join("wayland-remote-token.txt")).unwrap_or_default()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// The portal offers remote control here (doesn't ask the person anything).
|
|||
|
|
pub fn available() -> bool {
|
|||
|
|
futures_lite::future::block_on(async {
|
|||
|
|
match RemoteDesktop::new().await {
|
|||
|
|
Ok(rd) => rd.version() >= 1,
|
|||
|
|
Err(_) => false,
|
|||
|
|
}
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// Why the last start failed (for the bot's message).
|
|||
|
|
pub fn last_error() -> String {
|
|||
|
|
LAST_ERROR.lock().unwrap().clone()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// Start the session if it isn't running (the first time, the desktop asks the person).
|
|||
|
|
fn ensure() -> Option<Vec<Screen>> {
|
|||
|
|
if let Some(l) = LIVE.lock().unwrap().as_ref() {
|
|||
|
|
return Some(l.screens.clone());
|
|||
|
|
}
|
|||
|
|
let (ready_tx, ready_rx) = channel::<Result<Vec<Screen>, String>>();
|
|||
|
|
let (tx, rx) = channel::<Cmd>();
|
|||
|
|
std::thread::spawn(move || {
|
|||
|
|
futures_lite::future::block_on(async move {
|
|||
|
|
let started = async {
|
|||
|
|
let rd = RemoteDesktop::new().await.map_err(|e| e.to_string())?;
|
|||
|
|
let session = rd.create_session(Default::default()).await.map_err(|e| e.to_string())?;
|
|||
|
|
let token = std::fs::read_to_string(token_file()).ok().map(|s| s.trim().to_string()).filter(|s| !s.is_empty());
|
|||
|
|
// Version 2 can remember the person's answer (older desktops ask every time AnyBot starts).
|
|||
|
|
let mut devices = SelectDevicesOptions::default().set_devices(DeviceType::Keyboard | DeviceType::Pointer);
|
|||
|
|
if rd.version() >= 2 {
|
|||
|
|
devices = devices.set_persist_mode(PersistMode::ExplicitlyRevoked).set_restore_token(token.as_deref());
|
|||
|
|
}
|
|||
|
|
rd.select_devices(&session, devices)
|
|||
|
|
.await
|
|||
|
|
.map_err(|e| e.to_string())?;
|
|||
|
|
let sc = Screencast::new().await.map_err(|e| e.to_string())?;
|
|||
|
|
sc.select_sources(&session, SelectSourcesOptions::default().set_sources(ashpd::enumflags2::BitFlags::from(SourceType::Monitor)).set_multiple(true))
|
|||
|
|
.await
|
|||
|
|
.map_err(|e| e.to_string())?;
|
|||
|
|
let res = rd.start(&session, None, Default::default()).await.map_err(|e| e.to_string())?.response().map_err(|e| match e {
|
|||
|
|
ashpd::Error::Response(_) => "你在系統視窗裡沒有允許 AnyBot 操作畫面".to_string(),
|
|||
|
|
e => e.to_string(),
|
|||
|
|
})?;
|
|||
|
|
// GNOME's dialog has an 「允許遠端操作」 switch that starts off: sharing alone lets
|
|||
|
|
// AnyBot see but not use the screen.
|
|||
|
|
if !res.devices().contains(DeviceType::Pointer) || !res.devices().contains(DeviceType::Keyboard) {
|
|||
|
|
let _ = session.close().await;
|
|||
|
|
return Err("系統視窗裡的「允許遠端操作」沒有打開,所以 AnyBot 只能看、不能操作。請再試一次,這次先打開「允許遠端操作」再按「分享」".to_string());
|
|||
|
|
}
|
|||
|
|
if let Some(t) = res.restore_token() {
|
|||
|
|
let _ = std::fs::write(token_file(), t);
|
|||
|
|
}
|
|||
|
|
let screens: Vec<Screen> = res
|
|||
|
|
.streams()
|
|||
|
|
.iter()
|
|||
|
|
.map(|s| {
|
|||
|
|
let (x, y) = s.position().unwrap_or((0, 0));
|
|||
|
|
let (w, h) = s.size().unwrap_or((0, 0));
|
|||
|
|
Screen { node: s.pipe_wire_node_id(), x, y, w, h }
|
|||
|
|
})
|
|||
|
|
.collect();
|
|||
|
|
if screens.is_empty() {
|
|||
|
|
return Err("系統沒有給 AnyBot 任何螢幕".to_string());
|
|||
|
|
}
|
|||
|
|
Ok::<_, String>((rd, session, screens))
|
|||
|
|
};
|
|||
|
|
let (rd, session, screens) = match started.await {
|
|||
|
|
Ok(v) => v,
|
|||
|
|
Err(e) => {
|
|||
|
|
let _ = ready_tx.send(Err(e));
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
// GNOME drops the first event of a new virtual keyboard: wake it with a Shift tap
|
|||
|
|
// (no visible effect) before AnyBot types anything.
|
|||
|
|
for p in [KeyState::Pressed, KeyState::Released] {
|
|||
|
|
let _ = rd.notify_keyboard_keysym(&session, 0xffe1, p, Default::default()).await;
|
|||
|
|
}
|
|||
|
|
async_io::Timer::after(Duration::from_millis(150)).await;
|
|||
|
|
let _ = ready_tx.send(Ok(screens.clone()));
|
|||
|
|
// Serve commands until AnyBot stops using the session (or the desktop ends it).
|
|||
|
|
while let Ok(cmd) = rx.recv() {
|
|||
|
|
let r = match cmd {
|
|||
|
|
Cmd::Motion(x, y) => {
|
|||
|
|
let s = screens.iter().find(|s| x >= s.x as f64 && y >= s.y as f64 && x < (s.x + s.w) as f64 && y < (s.y + s.h) as f64).or(screens.first()).copied().unwrap_or_default();
|
|||
|
|
// Twice: a new virtual pointer can miss its first move, and a click must
|
|||
|
|
// never land where the mouse was before.
|
|||
|
|
let (lx, ly) = ((x - s.x as f64).max(0.0), (y - s.y as f64).max(0.0));
|
|||
|
|
let _ = rd.notify_pointer_motion_absolute(&session, s.node, lx, ly, Default::default()).await;
|
|||
|
|
rd.notify_pointer_motion_absolute(&session, s.node, lx, ly, Default::default()).await
|
|||
|
|
}
|
|||
|
|
Cmd::Button(b, p) => rd.notify_pointer_button(&session, b, if p { KeyState::Pressed } else { KeyState::Released }, Default::default()).await,
|
|||
|
|
Cmd::Keysym(k, p) => rd.notify_keyboard_keysym(&session, k as i32, if p { KeyState::Pressed } else { KeyState::Released }, Default::default()).await,
|
|||
|
|
Cmd::Wheel(axis, steps) => rd.notify_pointer_axis_discrete(&session, axis, steps, Default::default()).await,
|
|||
|
|
};
|
|||
|
|
if r.is_err() {
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
let _ = session.close().await;
|
|||
|
|
*LIVE.lock().unwrap() = None;
|
|||
|
|
});
|
|||
|
|
});
|
|||
|
|
match ready_rx.recv_timeout(Duration::from_secs(120)) {
|
|||
|
|
Ok(Ok(screens)) => {
|
|||
|
|
*LIVE.lock().unwrap() = Some(Live { tx, screens: screens.clone() });
|
|||
|
|
Some(screens)
|
|||
|
|
}
|
|||
|
|
Ok(Err(e)) => {
|
|||
|
|
*LAST_ERROR.lock().unwrap() = e;
|
|||
|
|
None
|
|||
|
|
}
|
|||
|
|
Err(_) => {
|
|||
|
|
*LAST_ERROR.lock().unwrap() = "等太久了,系統的允許視窗沒有回應".into();
|
|||
|
|
None
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
fn send(cmd: Cmd) {
|
|||
|
|
if ensure().is_none() {
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
let ok = LIVE.lock().unwrap().as_ref().map(|l| l.tx.send(cmd).is_ok()).unwrap_or(false);
|
|||
|
|
if !ok {
|
|||
|
|
*LIVE.lock().unwrap() = None;
|
|||
|
|
}
|
|||
|
|
// Commands are asynchronous; a short gap keeps press/release order sane for the desktop.
|
|||
|
|
std::thread::sleep(Duration::from_millis(4));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// The screens' bounding box in logical points.
|
|||
|
|
pub fn screen_points() -> (f64, f64) {
|
|||
|
|
let Some(screens) = ensure() else { return (0.0, 0.0) };
|
|||
|
|
let w = screens.iter().map(|s| s.x + s.w).max().unwrap_or(0);
|
|||
|
|
let h = screens.iter().map(|s| s.y + s.h).max().unwrap_or(0);
|
|||
|
|
(w as f64, h as f64)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
static PIXELS: Mutex<(u32, u32)> = Mutex::new((0, 0));
|
|||
|
|
|
|||
|
|
pub fn screen_pixels() -> (u32, u32) {
|
|||
|
|
let p = *PIXELS.lock().unwrap();
|
|||
|
|
if p.0 > 0 {
|
|||
|
|
return p;
|
|||
|
|
}
|
|||
|
|
let (w, h) = screen_points();
|
|||
|
|
(w as u32, h as u32)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// A `file://` URI's path.
|
|||
|
|
pub fn file_path_of(uri: &str) -> Option<std::path::PathBuf> {
|
|||
|
|
let rest = uri.strip_prefix("file://")?;
|
|||
|
|
let bytes = rest.as_bytes();
|
|||
|
|
let mut out = Vec::with_capacity(bytes.len());
|
|||
|
|
let mut i = 0;
|
|||
|
|
while i < bytes.len() {
|
|||
|
|
if bytes[i] == b'%' && i + 2 < bytes.len() {
|
|||
|
|
if let Ok(b) = u8::from_str_radix(std::str::from_utf8(&bytes[i + 1..i + 3]).ok()?, 16) {
|
|||
|
|
out.push(b);
|
|||
|
|
i += 3;
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
out.push(bytes[i]);
|
|||
|
|
i += 1;
|
|||
|
|
}
|
|||
|
|
Some(std::path::PathBuf::from(String::from_utf8(out).ok()?))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
pub fn capture() -> Result<image::RgbImage, String> {
|
|||
|
|
if ensure().is_none() {
|
|||
|
|
return Err(format!("AnyBot 還不能看這台電腦的畫面:{}。", last_error()));
|
|||
|
|
}
|
|||
|
|
let uri = futures_lite::future::block_on(async {
|
|||
|
|
let req = ashpd::desktop::screenshot::Screenshot::request().interactive(false).modal(false).send().await.map_err(|e| e.to_string())?;
|
|||
|
|
req.response().map(|s| s.uri().as_str().to_string()).map_err(|e| e.to_string())
|
|||
|
|
})
|
|||
|
|
.map_err(|e| format!("Wayland 截圖失敗:{e}"))?;
|
|||
|
|
let path = file_path_of(&uri).ok_or("截圖的位置看不懂")?;
|
|||
|
|
let img = image::open(&path).map_err(|e| e.to_string());
|
|||
|
|
// The desktop saved it (e.g. in 圖片/Screenshots); it was only for the bot.
|
|||
|
|
let _ = std::fs::remove_file(&path);
|
|||
|
|
let img = img?.to_rgb8();
|
|||
|
|
*PIXELS.lock().unwrap() = img.dimensions();
|
|||
|
|
Ok(img)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
pub fn motion(x: f64, y: f64) {
|
|||
|
|
send(Cmd::Motion(x, y));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// 1 left, 2 middle, 3 right → Linux button codes.
|
|||
|
|
pub fn button(b: u8, press: bool) {
|
|||
|
|
let code = match b {
|
|||
|
|
3 => 0x111,
|
|||
|
|
2 => 0x112,
|
|||
|
|
_ => 0x110,
|
|||
|
|
};
|
|||
|
|
send(Cmd::Button(code, press));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
pub fn wheel(up: i32, left: i32) {
|
|||
|
|
if up != 0 {
|
|||
|
|
send(Cmd::Wheel(Axis::Vertical, -up));
|
|||
|
|
}
|
|||
|
|
if left != 0 {
|
|||
|
|
send(Cmd::Wheel(Axis::Horizontal, -left));
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
pub fn key(sym: u32, press: bool) {
|
|||
|
|
send(Cmd::Keysym(sym, press));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// Typed directly: plain keyboard characters. The rest go through the clipboard.
|
|||
|
|
pub fn typable(c: char) -> bool {
|
|||
|
|
c == '\n' || c == '\t' || (' '..='~').contains(&c)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
fn tap(sym: u32) {
|
|||
|
|
key(sym, true);
|
|||
|
|
std::thread::sleep(Duration::from_millis(8));
|
|||
|
|
key(sym, false);
|
|||
|
|
std::thread::sleep(Duration::from_millis(8));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
pub fn type_text(text: &str) {
|
|||
|
|
let mut rest = String::new();
|
|||
|
|
let flush_paste = |s: &mut String| {
|
|||
|
|
if s.is_empty() {
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
paste(s);
|
|||
|
|
s.clear();
|
|||
|
|
};
|
|||
|
|
for c in text.chars() {
|
|||
|
|
if typable(c) {
|
|||
|
|
flush_paste(&mut rest);
|
|||
|
|
tap(crate::x11::keysym_for_char(c));
|
|||
|
|
} else {
|
|||
|
|
rest.push(c);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
flush_paste(&mut rest);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// Put text on the clipboard, press Ctrl+V, then put back what was there.
|
|||
|
|
fn paste(text: &str) {
|
|||
|
|
use std::io::Write;
|
|||
|
|
use std::process::{Command, Stdio};
|
|||
|
|
let old = Command::new("wl-paste").args(["-n", "-t", "text"]).stderr(Stdio::null()).output().ok().filter(|o| o.status.success()).map(|o| o.stdout);
|
|||
|
|
let set = |bytes: &[u8]| -> bool {
|
|||
|
|
let Ok(mut child) = Command::new("wl-copy").stdin(Stdio::piped()).stdout(Stdio::null()).stderr(Stdio::null()).spawn() else { return false };
|
|||
|
|
let ok = child.stdin.take().map(|mut s| s.write_all(bytes).is_ok()).unwrap_or(false);
|
|||
|
|
ok && child.wait().map(|s| s.success()).unwrap_or(false)
|
|||
|
|
};
|
|||
|
|
if !set(text.as_bytes()) {
|
|||
|
|
*LAST_ERROR.lock().unwrap() = "要打中文需要 wl-clipboard(sudo apt install wl-clipboard)".into();
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
std::thread::sleep(Duration::from_millis(60));
|
|||
|
|
key(0xffe3, true);
|
|||
|
|
tap(0x76);
|
|||
|
|
key(0xffe3, false);
|
|||
|
|
std::thread::sleep(Duration::from_millis(250));
|
|||
|
|
if let Some(old) = old {
|
|||
|
|
set(&old);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
#[cfg(test)]
|
|||
|
|
mod tests {
|
|||
|
|
use super::*;
|
|||
|
|
|
|||
|
|
#[test]
|
|||
|
|
fn uri_paths_and_typing() {
|
|||
|
|
assert_eq!(file_path_of("file:///home/a/Pictures/Screenshot%20From%202026.png").unwrap(), std::path::PathBuf::from("/home/a/Pictures/Screenshot From 2026.png"));
|
|||
|
|
assert_eq!(file_path_of("file:///tmp/%E5%9C%96.png").unwrap(), std::path::PathBuf::from("/tmp/圖.png"));
|
|||
|
|
assert!(file_path_of("https://x").is_none());
|
|||
|
|
assert!(typable('a') && typable('\n') && !typable('你') && !typable('é'));
|
|||
|
|
}
|
|||
|
|
}
|