use std::sync::Mutex; use lazyboy_contracts::{ComputerMode, ComputerObservation}; use lazyboy_control::{ frames_match, parse_computer_actions, resolve_bot_workspace_cwd, resolve_bot_workspace_path, ActionRequest, AdapterContext, CommandRequest, ComputerRef, SandboxProvider, }; use rig_core::completion::ToolDefinition; use serde_json::{json, Value}; pub struct ToolCtx { pub sandbox: std::sync::Arc, pub computer: ComputerRef, pub context: AdapterContext, pub mode: ComputerMode, pub bot_id: String, pub vision: bool, pub gui_block: Option, pub previous_frame: Mutex>, pub takeover_requested: Mutex, } pub fn tool_definitions() -> Vec { vec![ ToolDefinition { name: "computer_observe".into(), description: "Capture a fresh desktop screenshot. Frame metadata comes back as text; the image is attached to the next model turn.".into(), parameters: json!({"type":"object","properties":{}}), }, ToolDefinition { name: "computer_act".into(), description: "Drive the desktop like a person: move to the target, click, type, drag, and scroll. Coordinates are 1280x800 from the top-left. Always include x,y for click/scroll/drag. Batch a whole gesture in one call. The resulting screenshot is attached to the next turn.".into(), parameters: json!({ "type":"object", "properties":{ "actions":{ "type":"array", "items":{ "type":"object", "properties":{ "kind":{"type":"string","enum":["click","move","down","up","hover","drag","type","key","scroll","wait","focus"]}, "title":{"type":"string"}, "x2":{"type":"number"}, "y2":{"type":"number"}, "x":{"type":"number"}, "y":{"type":"number"}, "text":{"type":"string"}, "key":{"type":"string"}, "modifiers":{"type":"array","items":{"type":"string"}}, "button":{"type":"string","enum":["left","right"]}, "double":{"type":"boolean"}, "direction":{"type":"string","enum":["up","down"]}, "amount":{"type":"number"}, "ms":{"type":"number"} }, "required":["kind"] } }, "observe":{"type":"boolean"}, "settle_ms":{"type":"number"} }, "required":["actions"] }), }, ToolDefinition { name: "shell".into(), description: "Run a command inside this bot's computer.".into(), parameters: json!({ "type":"object", "properties":{ "command":{"type":"string"}, "cwd":{"type":"string"} }, "required":["command"] }), }, ToolDefinition { name: "list_files".into(), description: "List files in this bot's home.".into(), parameters: json!({"type":"object","properties":{"path":{"type":"string"}}}), }, ToolDefinition { name: "read_file".into(), description: "Read a UTF-8 text file from this bot's home.".into(), parameters: json!({"type":"object","properties":{"path":{"type":"string"}},"required":["path"]}), }, ToolDefinition { name: "write_file".into(), description: "Write a UTF-8 file into this bot's home.".into(), parameters: json!({ "type":"object", "properties":{"path":{"type":"string"},"content":{"type":"string"}}, "required":["path","content"] }), }, ToolDefinition { name: "open_path".into(), description: "Open a workspace file or http(s) URL on the desktop.".into(), parameters: json!({"type":"object","properties":{"path":{"type":"string"}},"required":["path"]}), }, ToolDefinition { name: "launch_app".into(), description: "Launch browser or terminal on the desktop.".into(), parameters: json!({ "type":"object", "properties":{"application":{"type":"string"},"uri":{"type":"string"}}, "required":["application"] }), }, ToolDefinition { name: "request_takeover".into(), description: "Ask the user to take over for passwords, 2FA, CAPTCHA, or protected input. Never ask them to paste secrets in chat.".into(), parameters: json!({"type":"object","properties":{"reason":{"type":"string"}},"required":["reason"]}), }, ] } pub struct ToolOutcome { pub text: String, pub image: Option>, pub pause: bool, } pub async fn dispatch(ctx: &ToolCtx, name: &str, args: &Value) -> ToolOutcome { match name { "computer_observe" => observe(ctx).await, "computer_act" => act(ctx, args).await, "shell" => shell(ctx, args).await, "list_files" => list_files(ctx, args).await, "read_file" => read_file(ctx, args).await, "write_file" => write_file(ctx, args).await, "open_path" => open_path(ctx, args).await, "launch_app" => launch_app(ctx, args).await, "request_takeover" => { *ctx.takeover_requested.lock().unwrap() = true; ToolOutcome { text: args .get("reason") .and_then(Value::as_str) .unwrap_or("The bot asked you to take control.") .to_string(), image: None, pause: true, } } other => ToolOutcome { text: format!("unknown tool {other}"), image: None, pause: false, }, } } fn vision_guard(ctx: &ToolCtx) -> Option { if let Some(message) = &ctx.gui_block { return Some(ToolOutcome { text: message.clone(), image: None, pause: false, }); } if ctx.vision { None } else { Some(ToolOutcome { text: "This model cannot see the screen. Use shell and file tools, or pick a vision model.".into(), image: None, pause: false, }) } } fn observation_text(note: &str, observation: &ComputerObservation, unchanged: bool) -> String { format!( "{note}{}\n{}", if unchanged { " (screen unchanged)" } else { "" }, json!({ "frameId": observation.frame_id, "width": observation.width, "height": observation.height, "capturedAt": observation.captured_at, "cursor": observation.cursor, "activeWindow": observation.active_window, }) ) } async fn observe(ctx: &ToolCtx) -> ToolOutcome { if let Some(blocked) = vision_guard(ctx) { return blocked; } match ctx.sandbox.observe(&ctx.computer, &ctx.context).await { Ok(observation) => pack_observation(ctx, "computer observed", observation), Err(error) => ToolOutcome { text: error.to_string(), image: None, pause: false, }, } } async fn act(ctx: &ToolCtx, args: &Value) -> ToolOutcome { if let Some(blocked) = vision_guard(ctx) { return blocked; } let actions = match parse_computer_actions(args.get("actions").unwrap_or(&Value::Null)) { Ok(actions) => actions, Err(error) => { return ToolOutcome { text: error.to_string(), image: None, pause: false, }; } }; match ctx .sandbox .act( &ctx.computer, ActionRequest { actions, observe: args.get("observe").and_then(Value::as_bool) != Some(false), settle_ms: args.get("settle_ms").and_then(Value::as_u64).unwrap_or(120) as u32, display: ctx.context.display.clone(), profile_path: ctx.context.profile_path.clone(), }, &ctx.context, ) .await { Ok(result) => { if let Some(observation) = result.observation { pack_observation( ctx, &format!("completed {} computer action(s)", result.completed), observation, ) } else { ToolOutcome { text: json!({"ok": true, "completed": result.completed}).to_string(), image: None, pause: false, } } } Err(error) => ToolOutcome { text: error.to_string(), image: None, pause: false, }, } } fn pack_observation(ctx: &ToolCtx, note: &str, observation: ComputerObservation) -> ToolOutcome { let unchanged = frames_match(ctx.previous_frame.lock().unwrap().as_deref(), &observation); *ctx.previous_frame.lock().unwrap() = Some(observation.frame_id.clone()); ToolOutcome { text: observation_text(note, &observation, unchanged), image: if unchanged { None } else { Some(observation.image) }, pause: false, } } async fn shell(ctx: &ToolCtx, args: &Value) -> ToolOutcome { let command = args.get("command").and_then(Value::as_str).unwrap_or(""); let cwd = args.get("cwd").and_then(Value::as_str); let cwd = resolve_bot_workspace_cwd(ctx.mode, &ctx.bot_id, cwd) .ok() .flatten(); match ctx .sandbox .execute( &ctx.computer, CommandRequest { argv: vec!["bash".into(), "-lc".into(), command.into()], cwd, timeout_ms: Some(60_000), }, &ctx.context, ) .await { Ok(result) => ToolOutcome { text: format!("exit {}\n{}\n{}", result.code, result.stdout, result.stderr), image: None, pause: false, }, Err(error) => ToolOutcome { text: error.to_string(), image: None, pause: false, }, } } async fn list_files(ctx: &ToolCtx, args: &Value) -> ToolOutcome { let requested = args.get("path").and_then(Value::as_str).unwrap_or(""); let stored = match resolve_bot_workspace_path(ctx.mode, &ctx.bot_id, requested) { Ok(path) => path, Err(error) => { return ToolOutcome { text: error.to_string(), image: None, pause: false, }; } }; match ctx.sandbox.list_files(&ctx.computer, &stored, &ctx.context).await { Ok(entries) => ToolOutcome { text: serde_json::to_string(&entries).unwrap_or_else(|_| "[]".into()), image: None, pause: false, }, Err(error) => ToolOutcome { text: error.to_string(), image: None, pause: false, }, } } async fn read_file(ctx: &ToolCtx, args: &Value) -> ToolOutcome { let requested = args.get("path").and_then(Value::as_str).unwrap_or(""); let stored = match resolve_bot_workspace_path(ctx.mode, &ctx.bot_id, requested) { Ok(path) => path, Err(error) => { return ToolOutcome { text: error.to_string(), image: None, pause: false, }; } }; match ctx.sandbox.read_file(&ctx.computer, &stored, &ctx.context).await { Ok(bytes) => ToolOutcome { text: String::from_utf8_lossy(&bytes).into_owned(), image: None, pause: false, }, Err(error) => ToolOutcome { text: error.to_string(), image: None, pause: false, }, } } async fn write_file(ctx: &ToolCtx, args: &Value) -> ToolOutcome { let requested = args.get("path").and_then(Value::as_str).unwrap_or("notes.txt"); let content = args.get("content").and_then(Value::as_str).unwrap_or(""); let stored = match resolve_bot_workspace_path(ctx.mode, &ctx.bot_id, requested) { Ok(path) => path, Err(error) => { return ToolOutcome { text: error.to_string(), image: None, pause: false, }; } }; match ctx .sandbox .write_file(&ctx.computer, &stored, content.as_bytes(), &ctx.context) .await { Ok(()) => ToolOutcome { text: json!({"ok": true, "path": requested}).to_string(), image: None, pause: false, }, Err(error) => ToolOutcome { text: error.to_string(), image: None, pause: false, }, } } async fn open_path(ctx: &ToolCtx, args: &Value) -> ToolOutcome { if let Some(blocked) = vision_guard(ctx) { return blocked; } let path = args.get("path").and_then(Value::as_str).unwrap_or(""); match ctx .sandbox .act( &ctx.computer, ActionRequest { actions: vec![lazyboy_contracts::ComputerAction::Open { path: path.into() }], observe: true, settle_ms: 400, display: ctx.context.display.clone(), profile_path: ctx.context.profile_path.clone(), }, &ctx.context, ) .await { Ok(result) => { if let Some(observation) = result.observation { pack_observation(ctx, "opened path", observation) } else { ToolOutcome { text: "opened".into(), image: None, pause: false, } } } Err(error) => ToolOutcome { text: error.to_string(), image: None, pause: false, }, } } async fn launch_app(ctx: &ToolCtx, args: &Value) -> ToolOutcome { if let Some(blocked) = vision_guard(ctx) { return blocked; } let application = args.get("application").and_then(Value::as_str).unwrap_or("browser"); let uri = args.get("uri").and_then(Value::as_str).map(str::to_string); match ctx .sandbox .act( &ctx.computer, ActionRequest { actions: vec![lazyboy_contracts::ComputerAction::Launch { application: application.into(), uri, }], observe: true, settle_ms: 500, display: ctx.context.display.clone(), profile_path: ctx.context.profile_path.clone(), }, &ctx.context, ) .await { Ok(result) => { if let Some(observation) = result.observation { pack_observation(ctx, "launched app", observation) } else { ToolOutcome { text: "launched".into(), image: None, pause: false, } } } Err(error) => ToolOutcome { text: error.to_string(), image: None, pause: false, }, } }