AnyBot/src/desk.rs

// Linux: one front for seeing and using the screen, whatever the desktop is.
//   X11 (Xorg sessions)                  x11.rs
//   Wayland, wlroots (sway, Hyprland…)   wl.rs   – the compositor's own protocols, no dialog
//   Wayland, GNOME / KDE                 portal.rs – the desktop asks the person once
// The functions match computer.rs's macOS part, so the computer_use tools work the same way.

use std::sync::OnceLock;
use std::time::Duration;

use crate::x11;

#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Kind {
    X11,
    Wlr,
    Portal,
    None,
}

pub fn wayland() -> bool {
    std::env::var("WAYLAND_DISPLAY").map(|v| !v.is_empty()).unwrap_or(false)
}

/// Which way this session works (decided once).
pub fn kind() -> Kind {
    static K: OnceLock<Kind> = OnceLock::new();
    *K.get_or_init(|| {
        if std::env::var("ANYBOT_DESK").as_deref() == Ok("x11") {
            return if x11::available() { Kind::X11 } else { Kind::None };
        }
        if wayland() {
            if crate::wl::available() {
                Kind::Wlr
            } else if crate::portal::available() {
                Kind::Portal
            } else {
                Kind::None
            }
        } else if x11::available() {
            Kind::X11
        } else {
            Kind::None
        }
    })
}

pub fn available() -> bool {
    kind() != Kind::None
}

/// A sentence for the bot and the settings page.
pub fn describe() -> &'static str {
    match kind() {
        Kind::X11 => "Linux（X11）：看畫面和操作滑鼠鍵盤都可以用，不需要另外允許。",
        Kind::Wlr => "Linux（Wayland）：看畫面和操作滑鼠鍵盤都可以用，不需要另外允許。",
        Kind::Portal => "Linux（Wayland）：第一次看畫面時，系統會問要不要讓 App 截圖，選「允許」；第一次操作時會跳出「遠端桌面」視窗，打開「允許遠端操作」再按「分享」。",
        Kind::None if wayland() => "這個 Wayland 桌面沒有提供讓 App 操作畫面的方式（找不到 xdg-desktop-portal 的遠端桌面功能）。請安裝 xdg-desktop-portal-gnome 或 xdg-desktop-portal-kde，或在登入畫面改選 Xorg 工作階段。",
        Kind::None => "這台 Linux 沒有桌面畫面（找不到顯示器），所以沒辦法看畫面和操作。",
    }
}

pub fn capture() -> Result<image::RgbImage, String> {
    match kind() {
        Kind::X11 => x11::capture(),
        Kind::Wlr => crate::wl::capture(),
        Kind::Portal => crate::portal::capture(),
        Kind::None => Err(describe().into()),
    }
}

pub fn screen_points() -> (f64, f64) {
    match kind() {
        Kind::X11 => x11::screen_points(),
        Kind::Wlr => crate::wl::screen_points(),
        Kind::Portal => crate::portal::screen_points(),
        Kind::None => (0.0, 0.0),
    }
}

pub fn screen_pixels() -> (u32, u32) {
    match kind() {
        Kind::X11 => x11::screen_pixels(),
        Kind::Wlr => crate::wl::screen_pixels(),
        Kind::Portal => crate::portal::screen_pixels(),
        Kind::None => (0, 0),
    }
}

pub fn screen_capture_preflight() -> bool {
    available()
}

// ---------- the Wayland primitives, and actions built from them ----------

fn motion(x: f64, y: f64) {
    match kind() {
        Kind::Wlr => crate::wl::motion(x, y),
        Kind::Portal => crate::portal::motion(x, y),
        _ => {}
    }
}

fn button(b: u8, press: bool) {
    match kind() {
        Kind::Wlr => crate::wl::button(b, press),
        Kind::Portal => crate::portal::button(b, press),
        _ => {}
    }
}

fn key(sym: u32, press: bool) {
    match kind() {
        Kind::Wlr => crate::wl::key(sym, press),
        Kind::Portal => crate::portal::key(sym, press),
        _ => {}
    }
}

fn wheel(up: i32, left: i32) {
    match kind() {
        Kind::Wlr => crate::wl::wheel(up, left),
        Kind::Portal => crate::portal::wheel(up, left),
        _ => {}
    }
}

fn sleep_secs(s: f64) {
    std::thread::sleep(Duration::from_secs_f64(s));
}

fn gap() {
    std::thread::sleep(Duration::from_millis(8));
}

fn button_no(button: &str) -> u8 {
    match button {
        "right" => 3,
        "middle" => 2,
        _ => 1,
    }
}

fn with_mods(flags: u64, f: impl FnOnce()) {
    let mods = x11::modifier_keysyms(flags);
    for m in &mods {
        key(*m, true);
    }
    if !mods.is_empty() {
        gap();
    }
    f();
    for m in mods.iter().rev() {
        key(*m, false);
    }
}

pub fn move_to(px: f64, py: f64) {
    if kind() == Kind::X11 {
        return x11::move_to(px, py);
    }
    motion(px, py);
}

pub fn click(px: f64, py: f64, which: &str, count: i64, flags: u64) {
    if kind() == Kind::X11 {
        return x11::click(px, py, which, count, flags);
    }
    motion(px, py);
    gap();
    let b = button_no(which);
    with_mods(flags, || {
        for n in 1..=count {
            if n > 1 {
                sleep_secs(crate::computer::CLICK_REPEAT_GAP_SECS);
            }
            button(b, true);
            sleep_secs(crate::computer::CLICK_HOLD_SECS);
            button(b, false);
        }
    });
}

pub fn drag(path: &[(f64, f64)], which: &str, flags: u64) {
    if kind() == Kind::X11 {
        return x11::drag(path, which, flags);
    }
    let (x0, y0) = path[0];
    motion(x0, y0);
    gap();
    let b = button_no(which);
    with_mods(flags, || {
        button(b, true);
        sleep_secs(crate::computer::DRAG_HOLD_SECS);
        for (x, y) in crate::computer::drag_points(path) {
            gap();
            motion(x, y);
        }
        button(b, false);
    });
}

pub fn scroll(at: Option<(f64, f64)>, dy: i32, dx: i32, flags: u64) {
    if kind() == Kind::X11 {
        return x11::scroll(at, dy, dx, flags);
    }
    if let Some((x, y)) = at {
        motion(x, y);
        gap();
    }
    let step = crate::computer::SCROLL_PIXELS_PER_STEP;
    with_mods(flags, || {
        // One wheel click at a time, like X11's buttons 4–7.
        let (vy, vx) = (dy / step, dx / step);
        for _ in 0..vy.abs() {
            wheel(vy.signum(), 0);
            std::thread::sleep(Duration::from_millis(12));
        }
        for _ in 0..vx.abs() {
            wheel(0, vx.signum());
            std::thread::sleep(Duration::from_millis(12));
        }
    });
}

pub fn press(code: u16, flags: u64, hold_secs: f64) {
    if kind() == Kind::X11 {
        return x11::press(code, flags, hold_secs);
    }
    let Some(sym) = x11::keysym_of_mac_code(code) else { return };
    with_mods(flags, || {
        key(sym, true);
        sleep_secs(hold_secs);
        key(sym, false);
    });
}

pub fn type_text(text: &str) {
    match kind() {
        Kind::X11 => x11::type_text(text),
        Kind::Wlr => crate::wl::type_text(text),
        Kind::Portal => crate::portal::type_text(text),
        Kind::None => {}
    }
}

// ---------- windows ----------

/// The app under a point. Wayland doesn't tell apps where other windows are, so only X11 (and
/// XWayland windows) can answer.
pub fn window_at(px: f64, py: f64) -> Option<(String, i32)> {
    match kind() {
        Kind::X11 => x11::window_at(px, py),
        _ => None,
    }
}

pub fn front_app() -> Option<(String, i32)> {
    match kind() {
        Kind::X11 => x11::front_app(),
        Kind::Wlr => crate::wl::front_app(),
        _ => None,
    }
}

// ---------- accessibility (the numbered controls) ----------

/// The program a window belongs to, the way agent-desktop names apps (the executable's name): a
/// Snap Firefox's window says "firefox_firefox", its program is "firefox".
pub fn program_of(pid: i32) -> Option<String> {
    if pid <= 0 {
        return None;
    }
    let exe = std::fs::read_link(format!("/proc/{pid}/exe")).ok()?;
    let name = exe.file_name()?.to_str()?.trim_end_matches(" (deleted)").to_string();
    (!name.is_empty()).then_some(name)
}

/// Turn the desktop's accessibility switch on when it is off (what a screen reader does):
/// without it most apps never show their controls, so the bot gets no numbers. True when it was
/// off and is on now; apps already open show theirs once restarted.
pub fn accessibility_switched_on() -> bool {
    let Ok(conn) = zbus::blocking::Connection::session() else { return false };
    let Ok(p) = zbus::blocking::Proxy::new(&conn, "org.a11y.Bus", "/org/a11y/bus", "org.a11y.Status") else { return false };
    if p.get_property::<bool>("IsEnabled").unwrap_or(true) {
        return false;
    }
    p.set_property("IsEnabled", true).is_ok()
}

// ---------- idle and screensaver ----------

fn dbus_call<R: serde::de::DeserializeOwned + zbus::zvariant::Type>(dest: &str, path: &str, iface: &str, method: &str) -> Option<R> {
    let conn = zbus::blocking::Connection::session().ok()?;
    let reply = conn.call_method(Some(dest), path, Some(iface), method, &()).ok()?;
    reply.body().deserialize::<R>().ok()
}

/// Seconds since the last keyboard or mouse input.
pub fn idle_secs() -> Option<f64> {
    if !wayland() {
        return x11::idle_secs();
    }
    // GNOME: Mutter's idle monitor (milliseconds).
    if let Some(ms) = dbus_call::<u64>("org.gnome.Mutter.IdleMonitor", "/org/gnome/Mutter/IdleMonitor/Core", "org.gnome.Mutter.IdleMonitor", "GetIdletime") {
        return Some(ms as f64 / 1000.0);
    }
    // wlroots, KDE and others: ext-idle-notify.
    crate::wl::idle_secs()
}

/// The screensaver (or lock screen) is on.
pub fn screensaver_on() -> bool {
    if !wayland() {
        return x11::screensaver_on();
    }
    // GNOME speaks org.gnome.ScreenSaver.GetActive at /org/gnome/ScreenSaver; the plain
    // freedesktop one (KDE & co.) is org.freedesktop.ScreenSaver.IsActive at /ScreenSaver —
    // the old fallback asked for a path and method that never exist, so on KDE it always
    // answered "screen is free".
    dbus_call::<bool>("org.gnome.ScreenSaver", "/org/gnome/ScreenSaver", "org.gnome.ScreenSaver", "GetActive")
        .or_else(|| dbus_call::<bool>("org.freedesktop.ScreenSaver", "/ScreenSaver", "org.freedesktop.ScreenSaver", "IsActive"))
        .unwrap_or(false)
}