AnyBot/src/portal.rs

333 lines
13 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// Linux Wayland on GNOME and KDE: the desktop portals, the way these desktops let an app see the
// screen and use the mouse and keyboard.
// - RemoteDesktop (+ ScreenCast for the screens): the desktop asks the person once
// ("AnyBot wants to control your screen"); the answer is remembered with a restore token
// kept in AnyBot's folder, so later sessions start without asking (portal version 2+;
// older desktops such as Ubuntu 22.04 ask once each time AnyBot starts).
// - Screenshot: a still picture of the screen, without a dialog.
// Characters the keyboard layout has are sent as keysyms; others (Chinese…) go through the
// clipboard and Ctrl+V, because these desktops only press keys their layout has.
use std::sync::mpsc::{channel, Sender};
use std::sync::Mutex;
use std::time::Duration;
use ashpd::desktop::remote_desktop::{Axis, DeviceType, KeyState, RemoteDesktop, SelectDevicesOptions};
use ashpd::desktop::screencast::{Screencast, SelectSourcesOptions, SourceType};
use ashpd::desktop::PersistMode;
#[derive(Debug)]
enum Cmd {
Motion(f64, f64),
Button(i32, bool),
Keysym(u32, bool),
Wheel(Axis, i32),
}
/// A screen the session can point at: its PipeWire node and logical place.
#[derive(Clone, Copy, Debug, Default)]
pub struct Screen {
node: u32,
x: i32,
y: i32,
w: i32,
h: i32,
}
struct Live {
tx: Sender<Cmd>,
screens: Vec<Screen>,
}
static LIVE: Mutex<Option<Live>> = Mutex::new(None);
static LAST_ERROR: Mutex<String> = Mutex::new(String::new());
fn token_file() -> std::path::PathBuf {
crate::model::State::path().parent().map(|d| d.join("wayland-remote-token.txt")).unwrap_or_default()
}
/// The portal offers remote control here (doesn't ask the person anything).
pub fn available() -> bool {
futures_lite::future::block_on(async {
match RemoteDesktop::new().await {
Ok(rd) => rd.version() >= 1,
Err(_) => false,
}
})
}
/// Why the last start failed (for the bot's message).
pub fn last_error() -> String {
LAST_ERROR.lock().unwrap().clone()
}
/// Start the session if it isn't running (the first time, the desktop asks the person).
fn ensure() -> Option<Vec<Screen>> {
if let Some(l) = LIVE.lock().unwrap().as_ref() {
return Some(l.screens.clone());
}
let (ready_tx, ready_rx) = channel::<Result<Vec<Screen>, String>>();
let (tx, rx) = channel::<Cmd>();
std::thread::spawn(move || {
futures_lite::future::block_on(async move {
let started = async {
let rd = RemoteDesktop::new().await.map_err(|e| e.to_string())?;
let session = rd.create_session(Default::default()).await.map_err(|e| e.to_string())?;
let token = std::fs::read_to_string(token_file()).ok().map(|s| s.trim().to_string()).filter(|s| !s.is_empty());
// Version 2 can remember the person's answer (older desktops ask every time AnyBot starts).
let mut devices = SelectDevicesOptions::default().set_devices(DeviceType::Keyboard | DeviceType::Pointer);
if rd.version() >= 2 {
devices = devices.set_persist_mode(PersistMode::ExplicitlyRevoked).set_restore_token(token.as_deref());
}
rd.select_devices(&session, devices)
.await
.map_err(|e| e.to_string())?;
let sc = Screencast::new().await.map_err(|e| e.to_string())?;
sc.select_sources(&session, SelectSourcesOptions::default().set_sources(ashpd::enumflags2::BitFlags::from(SourceType::Monitor)).set_multiple(true))
.await
.map_err(|e| e.to_string())?;
let res = rd.start(&session, None, Default::default()).await.map_err(|e| e.to_string())?.response().map_err(|e| match e {
ashpd::Error::Response(_) => "你在系統視窗裡沒有允許 AnyBot 操作畫面".to_string(),
e => e.to_string(),
})?;
// GNOME's dialog has an 「允許遠端操作」 switch that starts off: sharing alone lets
// AnyBot see but not use the screen.
if !res.devices().contains(DeviceType::Pointer) || !res.devices().contains(DeviceType::Keyboard) {
let _ = session.close().await;
return Err("系統視窗裡的「允許遠端操作」沒有打開,所以 AnyBot 只能看、不能操作。請再試一次,這次先打開「允許遠端操作」再按「分享」".to_string());
}
if let Some(t) = res.restore_token() {
let _ = std::fs::write(token_file(), t);
}
let screens: Vec<Screen> = res
.streams()
.iter()
.map(|s| {
let (x, y) = s.position().unwrap_or((0, 0));
let (w, h) = s.size().unwrap_or((0, 0));
Screen { node: s.pipe_wire_node_id(), x, y, w, h }
})
.collect();
if screens.is_empty() {
return Err("系統沒有給 AnyBot 任何螢幕".to_string());
}
Ok::<_, String>((rd, session, screens))
};
let (rd, session, screens) = match started.await {
Ok(v) => v,
Err(e) => {
let _ = ready_tx.send(Err(e));
return;
}
};
// GNOME drops the first event of a new virtual keyboard: wake it with a Shift tap
// (no visible effect) before AnyBot types anything.
for p in [KeyState::Pressed, KeyState::Released] {
let _ = rd.notify_keyboard_keysym(&session, 0xffe1, p, Default::default()).await;
}
async_io::Timer::after(Duration::from_millis(150)).await;
let _ = ready_tx.send(Ok(screens.clone()));
// Serve commands until AnyBot stops using the session (or the desktop ends it).
while let Ok(cmd) = rx.recv() {
let r = match cmd {
Cmd::Motion(x, y) => {
let s = screens.iter().find(|s| x >= s.x as f64 && y >= s.y as f64 && x < (s.x + s.w) as f64 && y < (s.y + s.h) as f64).or(screens.first()).copied().unwrap_or_default();
// Twice: a new virtual pointer can miss its first move, and a click must
// never land where the mouse was before.
let (lx, ly) = ((x - s.x as f64).max(0.0), (y - s.y as f64).max(0.0));
let _ = rd.notify_pointer_motion_absolute(&session, s.node, lx, ly, Default::default()).await;
rd.notify_pointer_motion_absolute(&session, s.node, lx, ly, Default::default()).await
}
Cmd::Button(b, p) => rd.notify_pointer_button(&session, b, if p { KeyState::Pressed } else { KeyState::Released }, Default::default()).await,
Cmd::Keysym(k, p) => rd.notify_keyboard_keysym(&session, k as i32, if p { KeyState::Pressed } else { KeyState::Released }, Default::default()).await,
Cmd::Wheel(axis, steps) => rd.notify_pointer_axis_discrete(&session, axis, steps, Default::default()).await,
};
if r.is_err() {
break;
}
}
let _ = session.close().await;
*LIVE.lock().unwrap() = None;
});
});
match ready_rx.recv_timeout(Duration::from_secs(120)) {
Ok(Ok(screens)) => {
*LIVE.lock().unwrap() = Some(Live { tx, screens: screens.clone() });
Some(screens)
}
Ok(Err(e)) => {
*LAST_ERROR.lock().unwrap() = e;
None
}
Err(_) => {
*LAST_ERROR.lock().unwrap() = "等太久了,系統的允許視窗沒有回應".into();
None
}
}
}
fn send(cmd: Cmd) {
if ensure().is_none() {
return;
}
let ok = LIVE.lock().unwrap().as_ref().map(|l| l.tx.send(cmd).is_ok()).unwrap_or(false);
if !ok {
*LIVE.lock().unwrap() = None;
}
// Commands are asynchronous; a short gap keeps press/release order sane for the desktop.
std::thread::sleep(Duration::from_millis(4));
}
/// The screens' bounding box in logical points.
pub fn screen_points() -> (f64, f64) {
let Some(screens) = ensure() else { return (0.0, 0.0) };
let w = screens.iter().map(|s| s.x + s.w).max().unwrap_or(0);
let h = screens.iter().map(|s| s.y + s.h).max().unwrap_or(0);
(w as f64, h as f64)
}
static PIXELS: Mutex<(u32, u32)> = Mutex::new((0, 0));
pub fn screen_pixels() -> (u32, u32) {
let p = *PIXELS.lock().unwrap();
if p.0 > 0 {
return p;
}
let (w, h) = screen_points();
(w as u32, h as u32)
}
/// A `file://` URI's path.
pub fn file_path_of(uri: &str) -> Option<std::path::PathBuf> {
let rest = uri.strip_prefix("file://")?;
let bytes = rest.as_bytes();
let mut out = Vec::with_capacity(bytes.len());
let mut i = 0;
while i < bytes.len() {
if bytes[i] == b'%' && i + 2 < bytes.len() {
if let Ok(b) = u8::from_str_radix(std::str::from_utf8(&bytes[i + 1..i + 3]).ok()?, 16) {
out.push(b);
i += 3;
continue;
}
}
out.push(bytes[i]);
i += 1;
}
Some(std::path::PathBuf::from(String::from_utf8(out).ok()?))
}
pub fn capture() -> Result<image::RgbImage, String> {
if ensure().is_none() {
return Err(format!("AnyBot 還不能看這台電腦的畫面:{}。", last_error()));
}
let uri = futures_lite::future::block_on(async {
let req = ashpd::desktop::screenshot::Screenshot::request().interactive(false).modal(false).send().await.map_err(|e| e.to_string())?;
req.response().map(|s| s.uri().as_str().to_string()).map_err(|e| e.to_string())
})
.map_err(|e| format!("Wayland 截圖失敗:{e}"))?;
let path = file_path_of(&uri).ok_or("截圖的位置看不懂")?;
let img = image::open(&path).map_err(|e| e.to_string());
// The desktop saved it (e.g. in 圖片/Screenshots); it was only for the bot.
let _ = std::fs::remove_file(&path);
let img = img?.to_rgb8();
*PIXELS.lock().unwrap() = img.dimensions();
Ok(img)
}
pub fn motion(x: f64, y: f64) {
send(Cmd::Motion(x, y));
}
/// 1 left, 2 middle, 3 right → Linux button codes.
pub fn button(b: u8, press: bool) {
let code = match b {
3 => 0x111,
2 => 0x112,
_ => 0x110,
};
send(Cmd::Button(code, press));
}
pub fn wheel(up: i32, left: i32) {
if up != 0 {
send(Cmd::Wheel(Axis::Vertical, -up));
}
if left != 0 {
send(Cmd::Wheel(Axis::Horizontal, -left));
}
}
pub fn key(sym: u32, press: bool) {
send(Cmd::Keysym(sym, press));
}
/// Typed directly: plain keyboard characters. The rest go through the clipboard.
pub fn typable(c: char) -> bool {
c == '\n' || c == '\t' || (' '..='~').contains(&c)
}
fn tap(sym: u32) {
key(sym, true);
std::thread::sleep(Duration::from_millis(8));
key(sym, false);
std::thread::sleep(Duration::from_millis(8));
}
pub fn type_text(text: &str) {
let mut rest = String::new();
let flush_paste = |s: &mut String| {
if s.is_empty() {
return;
}
paste(s);
s.clear();
};
for c in text.chars() {
if typable(c) {
flush_paste(&mut rest);
tap(crate::x11::keysym_for_char(c));
} else {
rest.push(c);
}
}
flush_paste(&mut rest);
}
/// Put text on the clipboard, press Ctrl+V, then put back what was there.
fn paste(text: &str) {
use std::io::Write;
use std::process::{Command, Stdio};
let old = Command::new("wl-paste").args(["-n", "-t", "text"]).stderr(Stdio::null()).output().ok().filter(|o| o.status.success()).map(|o| o.stdout);
let set = |bytes: &[u8]| -> bool {
let Ok(mut child) = Command::new("wl-copy").stdin(Stdio::piped()).stdout(Stdio::null()).stderr(Stdio::null()).spawn() else { return false };
let ok = child.stdin.take().map(|mut s| s.write_all(bytes).is_ok()).unwrap_or(false);
ok && child.wait().map(|s| s.success()).unwrap_or(false)
};
if !set(text.as_bytes()) {
*LAST_ERROR.lock().unwrap() = "要打中文需要 wl-clipboard(sudo apt install wl-clipboard)".into();
return;
}
std::thread::sleep(Duration::from_millis(60));
key(0xffe3, true);
tap(0x76);
key(0xffe3, false);
std::thread::sleep(Duration::from_millis(250));
if let Some(old) = old {
set(&old);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn uri_paths_and_typing() {
assert_eq!(file_path_of("file:///home/a/Pictures/Screenshot%20From%202026.png").unwrap(), std::path::PathBuf::from("/home/a/Pictures/Screenshot From 2026.png"));
assert_eq!(file_path_of("file:///tmp/%E5%9C%96.png").unwrap(), std::path::PathBuf::from("/tmp/圖.png"));
assert!(file_path_of("https://x").is_none());
assert!(typable('a') && typable('\n') && !typable('你') && !typable('é'));
}
}