324 lines
9.9 KiB
Rust
324 lines
9.9 KiB
Rust
// Linux: one front for seeing and using the screen, whatever the desktop is.
|
||
// X11 (Xorg sessions) x11.rs
|
||
// Wayland, wlroots (sway, Hyprland…) wl.rs – the compositor's own protocols, no dialog
|
||
// Wayland, GNOME / KDE portal.rs – the desktop asks the person once
|
||
// The functions match computer.rs's macOS part, so the computer_use tools work the same way.
|
||
|
||
use std::sync::OnceLock;
|
||
use std::time::Duration;
|
||
|
||
use crate::x11;
|
||
|
||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||
pub enum Kind {
|
||
X11,
|
||
Wlr,
|
||
Portal,
|
||
None,
|
||
}
|
||
|
||
pub fn wayland() -> bool {
|
||
std::env::var("WAYLAND_DISPLAY").map(|v| !v.is_empty()).unwrap_or(false)
|
||
}
|
||
|
||
/// Which way this session works (decided once).
|
||
pub fn kind() -> Kind {
|
||
static K: OnceLock<Kind> = OnceLock::new();
|
||
*K.get_or_init(|| {
|
||
if std::env::var("ANYBOT_DESK").as_deref() == Ok("x11") {
|
||
return if x11::available() { Kind::X11 } else { Kind::None };
|
||
}
|
||
if wayland() {
|
||
if crate::wl::available() {
|
||
Kind::Wlr
|
||
} else if crate::portal::available() {
|
||
Kind::Portal
|
||
} else {
|
||
Kind::None
|
||
}
|
||
} else if x11::available() {
|
||
Kind::X11
|
||
} else {
|
||
Kind::None
|
||
}
|
||
})
|
||
}
|
||
|
||
pub fn available() -> bool {
|
||
kind() != Kind::None
|
||
}
|
||
|
||
/// A sentence for the bot and the settings page.
|
||
pub fn describe() -> &'static str {
|
||
match kind() {
|
||
Kind::X11 => "Linux（X11）：看畫面和操作滑鼠鍵盤都可以用，不需要另外允許。",
|
||
Kind::Wlr => "Linux（Wayland）：看畫面和操作滑鼠鍵盤都可以用，不需要另外允許。",
|
||
Kind::Portal => "Linux（Wayland）：第一次看畫面時，系統會問要不要讓 App 截圖，選「允許」；第一次操作時會跳出「遠端桌面」視窗，打開「允許遠端操作」再按「分享」。",
|
||
Kind::None if wayland() => "這個 Wayland 桌面沒有提供讓 App 操作畫面的方式（找不到 xdg-desktop-portal 的遠端桌面功能）。請安裝 xdg-desktop-portal-gnome 或 xdg-desktop-portal-kde，或在登入畫面改選 Xorg 工作階段。",
|
||
Kind::None => "這台 Linux 沒有桌面畫面（找不到顯示器），所以沒辦法看畫面和操作。",
|
||
}
|
||
}
|
||
|
||
pub fn capture() -> Result<image::RgbImage, String> {
|
||
match kind() {
|
||
Kind::X11 => x11::capture(),
|
||
Kind::Wlr => crate::wl::capture(),
|
||
Kind::Portal => crate::portal::capture(),
|
||
Kind::None => Err(describe().into()),
|
||
}
|
||
}
|
||
|
||
pub fn screen_points() -> (f64, f64) {
|
||
match kind() {
|
||
Kind::X11 => x11::screen_points(),
|
||
Kind::Wlr => crate::wl::screen_points(),
|
||
Kind::Portal => crate::portal::screen_points(),
|
||
Kind::None => (0.0, 0.0),
|
||
}
|
||
}
|
||
|
||
pub fn screen_pixels() -> (u32, u32) {
|
||
match kind() {
|
||
Kind::X11 => x11::screen_pixels(),
|
||
Kind::Wlr => crate::wl::screen_pixels(),
|
||
Kind::Portal => crate::portal::screen_pixels(),
|
||
Kind::None => (0, 0),
|
||
}
|
||
}
|
||
|
||
pub fn screen_capture_preflight() -> bool {
|
||
available()
|
||
}
|
||
|
||
// ---------- the Wayland primitives, and actions built from them ----------
|
||
|
||
fn motion(x: f64, y: f64) {
|
||
match kind() {
|
||
Kind::Wlr => crate::wl::motion(x, y),
|
||
Kind::Portal => crate::portal::motion(x, y),
|
||
_ => {}
|
||
}
|
||
}
|
||
|
||
fn button(b: u8, press: bool) {
|
||
match kind() {
|
||
Kind::Wlr => crate::wl::button(b, press),
|
||
Kind::Portal => crate::portal::button(b, press),
|
||
_ => {}
|
||
}
|
||
}
|
||
|
||
fn key(sym: u32, press: bool) {
|
||
match kind() {
|
||
Kind::Wlr => crate::wl::key(sym, press),
|
||
Kind::Portal => crate::portal::key(sym, press),
|
||
_ => {}
|
||
}
|
||
}
|
||
|
||
fn wheel(up: i32, left: i32) {
|
||
match kind() {
|
||
Kind::Wlr => crate::wl::wheel(up, left),
|
||
Kind::Portal => crate::portal::wheel(up, left),
|
||
_ => {}
|
||
}
|
||
}
|
||
|
||
fn sleep_secs(s: f64) {
|
||
std::thread::sleep(Duration::from_secs_f64(s));
|
||
}
|
||
|
||
fn gap() {
|
||
std::thread::sleep(Duration::from_millis(8));
|
||
}
|
||
|
||
fn button_no(button: &str) -> u8 {
|
||
match button {
|
||
"right" => 3,
|
||
"middle" => 2,
|
||
_ => 1,
|
||
}
|
||
}
|
||
|
||
fn with_mods(flags: u64, f: impl FnOnce()) {
|
||
let mods = x11::modifier_keysyms(flags);
|
||
for m in &mods {
|
||
key(*m, true);
|
||
}
|
||
if !mods.is_empty() {
|
||
gap();
|
||
}
|
||
f();
|
||
for m in mods.iter().rev() {
|
||
key(*m, false);
|
||
}
|
||
}
|
||
|
||
pub fn move_to(px: f64, py: f64) {
|
||
if kind() == Kind::X11 {
|
||
return x11::move_to(px, py);
|
||
}
|
||
motion(px, py);
|
||
}
|
||
|
||
pub fn click(px: f64, py: f64, which: &str, count: i64, flags: u64) {
|
||
if kind() == Kind::X11 {
|
||
return x11::click(px, py, which, count, flags);
|
||
}
|
||
motion(px, py);
|
||
gap();
|
||
let b = button_no(which);
|
||
with_mods(flags, || {
|
||
for n in 1..=count {
|
||
if n > 1 {
|
||
sleep_secs(crate::computer::CLICK_REPEAT_GAP_SECS);
|
||
}
|
||
button(b, true);
|
||
sleep_secs(crate::computer::CLICK_HOLD_SECS);
|
||
button(b, false);
|
||
}
|
||
});
|
||
}
|
||
|
||
pub fn drag(path: &[(f64, f64)], which: &str, flags: u64) {
|
||
if kind() == Kind::X11 {
|
||
return x11::drag(path, which, flags);
|
||
}
|
||
let (x0, y0) = path[0];
|
||
motion(x0, y0);
|
||
gap();
|
||
let b = button_no(which);
|
||
with_mods(flags, || {
|
||
button(b, true);
|
||
sleep_secs(crate::computer::DRAG_HOLD_SECS);
|
||
for (x, y) in crate::computer::drag_points(path) {
|
||
gap();
|
||
motion(x, y);
|
||
}
|
||
button(b, false);
|
||
});
|
||
}
|
||
|
||
pub fn scroll(at: Option<(f64, f64)>, dy: i32, dx: i32, flags: u64) {
|
||
if kind() == Kind::X11 {
|
||
return x11::scroll(at, dy, dx, flags);
|
||
}
|
||
if let Some((x, y)) = at {
|
||
motion(x, y);
|
||
gap();
|
||
}
|
||
let step = crate::computer::SCROLL_PIXELS_PER_STEP;
|
||
with_mods(flags, || {
|
||
// One wheel click at a time, like X11's buttons 4–7.
|
||
let (vy, vx) = (dy / step, dx / step);
|
||
for _ in 0..vy.abs() {
|
||
wheel(vy.signum(), 0);
|
||
std::thread::sleep(Duration::from_millis(12));
|
||
}
|
||
for _ in 0..vx.abs() {
|
||
wheel(0, vx.signum());
|
||
std::thread::sleep(Duration::from_millis(12));
|
||
}
|
||
});
|
||
}
|
||
|
||
pub fn press(code: u16, flags: u64, hold_secs: f64) {
|
||
if kind() == Kind::X11 {
|
||
return x11::press(code, flags, hold_secs);
|
||
}
|
||
let Some(sym) = x11::keysym_of_mac_code(code) else { return };
|
||
with_mods(flags, || {
|
||
key(sym, true);
|
||
sleep_secs(hold_secs);
|
||
key(sym, false);
|
||
});
|
||
}
|
||
|
||
pub fn type_text(text: &str) {
|
||
match kind() {
|
||
Kind::X11 => x11::type_text(text),
|
||
Kind::Wlr => crate::wl::type_text(text),
|
||
Kind::Portal => crate::portal::type_text(text),
|
||
Kind::None => {}
|
||
}
|
||
}
|
||
|
||
// ---------- windows ----------
|
||
|
||
/// The app under a point. Wayland doesn't tell apps where other windows are, so only X11 (and
|
||
/// XWayland windows) can answer.
|
||
pub fn window_at(px: f64, py: f64) -> Option<(String, i32)> {
|
||
match kind() {
|
||
Kind::X11 => x11::window_at(px, py),
|
||
_ => None,
|
||
}
|
||
}
|
||
|
||
pub fn front_app() -> Option<(String, i32)> {
|
||
match kind() {
|
||
Kind::X11 => x11::front_app(),
|
||
Kind::Wlr => crate::wl::front_app(),
|
||
_ => None,
|
||
}
|
||
}
|
||
|
||
// ---------- accessibility (the numbered controls) ----------
|
||
|
||
/// The program a window belongs to, the way agent-desktop names apps (the executable's name): a
|
||
/// Snap Firefox's window says "firefox_firefox", its program is "firefox".
|
||
pub fn program_of(pid: i32) -> Option<String> {
|
||
if pid <= 0 {
|
||
return None;
|
||
}
|
||
let exe = std::fs::read_link(format!("/proc/{pid}/exe")).ok()?;
|
||
let name = exe.file_name()?.to_str()?.trim_end_matches(" (deleted)").to_string();
|
||
(!name.is_empty()).then_some(name)
|
||
}
|
||
|
||
/// Turn the desktop's accessibility switch on when it is off (what a screen reader does):
|
||
/// without it most apps never show their controls, so the bot gets no numbers. True when it was
|
||
/// off and is on now; apps already open show theirs once restarted.
|
||
pub fn accessibility_switched_on() -> bool {
|
||
let Ok(conn) = zbus::blocking::Connection::session() else { return false };
|
||
let Ok(p) = zbus::blocking::Proxy::new(&conn, "org.a11y.Bus", "/org/a11y/bus", "org.a11y.Status") else { return false };
|
||
if p.get_property::<bool>("IsEnabled").unwrap_or(true) {
|
||
return false;
|
||
}
|
||
p.set_property("IsEnabled", true).is_ok()
|
||
}
|
||
|
||
// ---------- idle and screensaver ----------
|
||
|
||
fn dbus_call<R: serde::de::DeserializeOwned + zbus::zvariant::Type>(dest: &str, path: &str, iface: &str, method: &str) -> Option<R> {
|
||
let conn = zbus::blocking::Connection::session().ok()?;
|
||
let reply = conn.call_method(Some(dest), path, Some(iface), method, &()).ok()?;
|
||
reply.body().deserialize::<R>().ok()
|
||
}
|
||
|
||
/// Seconds since the last keyboard or mouse input.
|
||
pub fn idle_secs() -> Option<f64> {
|
||
if !wayland() {
|
||
return x11::idle_secs();
|
||
}
|
||
// GNOME: Mutter's idle monitor (milliseconds).
|
||
if let Some(ms) = dbus_call::<u64>("org.gnome.Mutter.IdleMonitor", "/org/gnome/Mutter/IdleMonitor/Core", "org.gnome.Mutter.IdleMonitor", "GetIdletime") {
|
||
return Some(ms as f64 / 1000.0);
|
||
}
|
||
// wlroots, KDE and others: ext-idle-notify.
|
||
crate::wl::idle_secs()
|
||
}
|
||
|
||
/// The screensaver (or lock screen) is on.
|
||
pub fn screensaver_on() -> bool {
|
||
if !wayland() {
|
||
return x11::screensaver_on();
|
||
}
|
||
// GNOME speaks org.gnome.ScreenSaver.GetActive at /org/gnome/ScreenSaver; the plain
|
||
// freedesktop one (KDE & co.) is org.freedesktop.ScreenSaver.IsActive at /ScreenSaver —
|
||
// the old fallback asked for a path and method that never exist, so on KDE it always
|
||
// answered "screen is free".
|
||
dbus_call::<bool>("org.gnome.ScreenSaver", "/org/gnome/ScreenSaver", "org.gnome.ScreenSaver", "GetActive")
|
||
.or_else(|| dbus_call::<bool>("org.freedesktop.ScreenSaver", "/ScreenSaver", "org.freedesktop.ScreenSaver", "IsActive"))
|
||
.unwrap_or(false)
|
||
}
|