use lazyboy_contracts::{ComputerAction, PointerButton, PointerType, ScrollDirection}; pub fn is_browser_title(title: &str) -> bool { let title = title.to_lowercase(); title.contains("chromium") || title.contains("chrome") } use crate::screen::{PRIMARY_DISPLAY, normalize_display}; pub const DISPLAY: &str = PRIMARY_DISPLAY; pub const HOME: &str = "/home/lazyboy"; fn display_env(display: &str) -> String { format!("DISPLAY={}", normalize_display(display)) } pub fn xdotool_argv(action: &ComputerAction) -> Option> { xdotool_argv_on(PRIMARY_DISPLAY, action) } pub fn xdotool_argv_on(display: &str, action: &ComputerAction) -> Option> { let mut argv = vec!["env".into(), display_env(display), "xdotool".into()]; match action { ComputerAction::Pointer { x, y, pointer_type, button, } => { let button_n = match button.unwrap_or(PointerButton::Left) { PointerButton::Left => "1", PointerButton::Middle => "2", PointerButton::Right => "3", }; match pointer_type { PointerType::Move => { argv.extend([ "mousemove".into(), "--sync".into(), "--".into(), x.to_string(), y.to_string(), ]); } PointerType::Click => { argv.extend([ "mousemove".into(), "--sync".into(), "--".into(), x.to_string(), y.to_string(), "click".into(), "--delay".into(), "40".into(), button_n.into(), ]); } PointerType::Down => { argv.extend([ "mousemove".into(), "--".into(), x.to_string(), y.to_string(), "mousedown".into(), button_n.into(), ]); } PointerType::Up => { argv.extend([ "mousemove".into(), "--".into(), x.to_string(), y.to_string(), "mouseup".into(), button_n.into(), ]); } } } ComputerAction::Key { key, modifiers } => { let combo = match modifiers { Some(items) if !items.is_empty() => format!("{}+{key}", items.join("+")), _ => key.clone(), }; argv.extend(["key".into(), "--clearmodifiers".into(), combo]); } ComputerAction::Clipboard { text } => { let quoted = shell_single_quote(text); if looks_like_typed_ascii(text) { argv.extend([ "type".into(), "--delay".into(), "16".into(), "--".into(), text.clone(), ]); } else { return Some(vec![ "env".into(), display_env(display), "bash".into(), "-lc".into(), format!( "printf %s {quoted} | xclip -selection clipboard && xdotool key --clearmodifiers ctrl+v" ), ]); } } ComputerAction::Scroll { direction, amount } => { let button = match direction { ScrollDirection::Up => "4", ScrollDirection::Down => "5", }; argv.extend([ "click".into(), "--repeat".into(), amount.unwrap_or(12).to_string(), "--delay".into(), "15".into(), button.into(), ]); } ComputerAction::Focus { title } => { let quoted = shell_single_quote(title); return Some(vec![ "env".into(), display_env(display), "bash".into(), "-lc".into(), format!( "wmctrl -a {quoted} || xdotool search --name {quoted} windowactivate --sync windowfocus" ), ]); } ComputerAction::Wait { .. } | ComputerAction::Open { .. } | ComputerAction::Launch { .. } | ComputerAction::Ref { .. } => { return None; } } Some(argv) } fn shell_single_quote(value: &str) -> String { format!("'{}'", value.replace('\'', "'\\''")) } fn looks_like_typed_ascii(text: &str) -> bool { text.len() <= 48 && text .chars() .all(|ch| ch.is_ascii() && (!ch.is_control() || ch == '\n' || ch == '\t')) } pub fn action_pause_ms(action: &ComputerAction) -> u64 { match action { ComputerAction::Pointer { pointer_type, .. } => match pointer_type { PointerType::Move => 18, PointerType::Click => 55, PointerType::Down | PointerType::Up => 30, }, ComputerAction::Scroll { .. } => 45, ComputerAction::Clipboard { text } if looks_like_typed_ascii(text) => 40, ComputerAction::Clipboard { .. } | ComputerAction::Key { .. } => 35, ComputerAction::Focus { .. } => 90, ComputerAction::Open { .. } | ComputerAction::Launch { .. } => 220, ComputerAction::Wait { .. } => 0, ComputerAction::Ref { .. } => 55, } } pub fn pointer_state_command() -> Vec { pointer_state_command_on(PRIMARY_DISPLAY) } pub fn pointer_state_command_on(display: &str) -> Vec { vec![ "env".into(), display_env(display), "python3".into(), "-c".into(), r#" import json, subprocess def out(args): try: return subprocess.check_output(args, stderr=subprocess.DEVNULL, text=True).strip() except Exception: return "" vals = {} for line in out(["xdotool", "getmouselocation", "--shell"]).splitlines(): if "=" in line: key, value = line.split("=", 1) vals[key] = value wid = out(["xdotool", "getactivewindow"]) title = out(["xdotool", "getwindowname", wid]) if wid else "" print(json.dumps({"x": int(vals.get("X") or 0), "y": int(vals.get("Y") or 0), "id": wid, "title": title})) "# .into(), ] } pub fn window_list_command() -> Vec { window_list_command_on(PRIMARY_DISPLAY) } pub fn window_list_command_on(display: &str) -> Vec { vec![ "env".into(), display_env(display), "python3".into(), "-c".into(), r#" import json, subprocess def out(args): try: return subprocess.check_output(args, stderr=subprocess.DEVNULL, text=True) except Exception: return "" els = [] n = 1 for line in out(["wmctrl", "-lG"]).splitlines(): parts = line.split(None, 7) if len(parts) < 7: continue try: x, y, w, h = int(parts[2]), int(parts[3]), int(parts[4]), int(parts[5]) except ValueError: continue if w < 32 or h < 16: continue title = parts[7].strip() if len(parts) > 7 else parts[6] if not title or title in ("Desktop", "xfce4-panel"): continue els.append({"id": n, "title": title[:80], "kind": "window", "x": max(0, x), "y": max(0, y), "w": w, "h": h}) n += 1 print(json.dumps(els)) "# .into(), ] } pub fn parse_ui_elements(raw: &str) -> Vec { let value: serde_json::Value = serde_json::from_str(raw.trim()).unwrap_or(serde_json::Value::Null); let Some(items) = value.as_array() else { return Vec::new(); }; items .iter() .filter_map(|item| { Some(lazyboy_contracts::UiElement { id: item.get("id")?.as_u64()? as u32, title: item.get("title")?.as_str()?.to_string(), x: item.get("x")?.as_u64()? as u32, y: item.get("y")?.as_u64()? as u32, w: item.get("w")?.as_u64()? as u32, h: item.get("h")?.as_u64()? as u32, selector: item .get("selector") .and_then(serde_json::Value::as_str) .filter(|value| !value.is_empty()) .map(str::to_string), kind: item .get("kind") .and_then(serde_json::Value::as_str) .filter(|value| !value.is_empty()) .map(str::to_string), role: item .get("role") .and_then(serde_json::Value::as_str) .filter(|value| !value.is_empty()) .map(str::to_string), }) }) .collect() } pub fn parse_pointer_state( raw: &str, ) -> ( Option, Option, ) { let value: serde_json::Value = serde_json::from_str(raw.trim()).unwrap_or(serde_json::Value::Null); let cursor = match ( value.get("x").and_then(serde_json::Value::as_i64), value.get("y").and_then(serde_json::Value::as_i64), ) { (Some(x), Some(y)) => Some(lazyboy_contracts::CursorPosition { x: x as i32, y: y as i32, }), _ => None, }; let window = value .get("id") .and_then(serde_json::Value::as_str) .filter(|id| !id.is_empty()) .map(|id| lazyboy_contracts::ActiveWindow { id: id.to_string(), title: value .get("title") .and_then(serde_json::Value::as_str) .filter(|title| !title.is_empty()) .map(str::to_string), }); (cursor, window) } pub fn open_argv(path: &str) -> Vec { open_argv_on(PRIMARY_DISPLAY, None, path) } pub fn open_argv_on(display: &str, profile: Option<&str>, path: &str) -> Vec { if path.starts_with("http://") || path.starts_with("https://") { return browser_argv(display, profile, Some(path)); } vec![ "env".into(), display_env(display), "xdg-open".into(), path.into(), ] } pub fn launch_argv(application: &str, uri: Option<&str>) -> Option> { launch_argv_on(PRIMARY_DISPLAY, None, application, uri) } pub fn launch_argv_on( display: &str, profile: Option<&str>, application: &str, uri: Option<&str>, ) -> Option> { match application { "browser" | "chrome" | "chromium" | "lazyboy-browser" => { Some(browser_argv(display, profile, uri)) } "xterm" | "terminal" | "xfce4-terminal" | "lazyboy-terminal" => { let mut argv = vec![ "env".into(), display_env(display), "lazyboy-terminal".into(), ]; if let Some(uri) = uri { argv.push(uri.into()); } Some(argv) } _ => None, } } fn browser_argv(display: &str, profile: Option<&str>, uri: Option<&str>) -> Vec { let mut argv = vec!["env".into(), display_env(display)]; if let Some(profile) = profile.filter(|value| !value.is_empty()) { argv.push(format!("LAZYBOY_BROWSER_PROFILE={profile}")); } argv.push("lazyboy-browser".into()); argv.push(format!( "--remote-debugging-port={}", crate::devtools_port(display) )); argv.push("--remote-allow-origins=*".into()); if let Some(uri) = uri { argv.push(uri.into()); } argv } pub fn screenshot_command() -> Vec { screenshot_command_on(PRIMARY_DISPLAY) } pub fn screenshot_command_on(display: &str) -> Vec { vec![ "bash".into(), "-lc".into(), format!( "DISPLAY={} xwd -root -silent | convert xwd:- -quality 60 jpeg:-", normalize_display(display) ), ] } #[cfg(test)] mod tests { use super::*; #[test] fn click_maps_to_xdotool() { let argv = xdotool_argv(&ComputerAction::Pointer { x: 12, y: 40, pointer_type: PointerType::Click, button: Some(PointerButton::Left), }) .unwrap(); assert!(argv.contains(&"click".into())); assert!(argv.contains(&"12".into())); } #[test] fn pointer_up_moves_before_release() { let argv = xdotool_argv(&ComputerAction::Pointer { x: 80, y: 90, pointer_type: PointerType::Up, button: Some(PointerButton::Left), }) .unwrap(); assert!(argv.contains(&"mousemove".into())); assert!(argv.contains(&"80".into())); assert!(argv.contains(&"mouseup".into())); } #[test] fn extra_display_is_injected_into_input_commands() { let argv = xdotool_argv_on( ":2", &ComputerAction::Pointer { x: 4, y: 8, pointer_type: PointerType::Move, button: None, }, ) .unwrap(); assert!(argv.contains(&"DISPLAY=:2".into())); let shot = screenshot_command_on(":3"); assert!(shot.last().unwrap().contains("DISPLAY=:3")); assert!(shot.last().unwrap().contains("jpeg:-")); let browser = launch_argv_on( ":2", Some("/home/lazyboy/.browser-profiles/bots/a"), "browser", None, ) .unwrap(); assert!(browser.contains(&"DISPLAY=:2".into())); assert!(browser.iter().any(|item| item.contains("bots/a"))); assert!(browser.contains(&"--remote-debugging-port=9223".into())); } #[test] fn parses_window_list_elements() { let elements = parse_ui_elements(r#"[{"id":1,"title":"Chromium","x":0,"y":24,"w":1280,"h":776}]"#); assert_eq!(elements.len(), 1); assert_eq!(elements[0].title, "Chromium"); assert_eq!(elements[0].center(), (640, 412)); } #[test] fn parses_dom_element_selector() { let elements = parse_ui_elements( r#"[{"id":1,"title":"Submit","selector":"[data-lazyboy=\"1\"]","kind":"dom","x":10,"y":20,"w":80,"h":24}]"#, ); assert_eq!(elements[0].kind.as_deref(), Some("dom")); assert_eq!( elements[0].selector.as_deref(), Some("[data-lazyboy=\"1\"]") ); } #[test] fn parses_a11y_role_and_path() { let elements = parse_ui_elements( r#"[{"id":2,"title":"push button Open","selector":"0/3/1","kind":"a11y","role":"push button","x":8,"y":9,"w":40,"h":16}]"#, ); assert_eq!(elements[0].kind.as_deref(), Some("a11y")); assert_eq!(elements[0].role.as_deref(), Some("push button")); assert_eq!(elements[0].selector.as_deref(), Some("0/3/1")); assert!(elements[0].has_ref()); } #[test] fn browser_titles_match_chromium() { assert!(is_browser_title("Example - Chromium")); assert!(is_browser_title("chrome")); assert!(!is_browser_title("Thunar")); } #[test] fn empty_or_junk_window_list_is_empty() { assert!(parse_ui_elements("").is_empty()); assert!(parse_ui_elements("not-json").is_empty()); } } /// Native clipboard path; text is supplied on stdin, never process arguments. pub fn paste_command_on(display: &str) -> Vec { vec![ "env".into(), display_env(display), "python3".into(), "-c".into(), include_str!("clipboard.py").into(), ] }