1438 lines
51 KiB
Rust
1438 lines
51 KiB
Rust
use std::sync::Mutex;
|
|
|
|
use lazyboy_contracts::{
|
|
ComputerAction, ComputerMode, ComputerObservation, PointerType, UiElement,
|
|
};
|
|
use lazyboy_control::{
|
|
ActionError, ActionRequest, AdapterContext, CdpPage, CommandRequest, ComputerRef,
|
|
SandboxProvider, a11y_command_on, apply_element_targets, browser_gui_block, cdp_command_on,
|
|
click_fingerprint, element_id, format_ui_elements, frames_match, merge_ui_elements,
|
|
cdp_stdin_command_on, overlay_elements, parse_a11y_page, parse_cdp_page, parse_computer_actions,
|
|
resolve_bot_workspace_cwd, resolve_bot_workspace_path, should_block_stale_click,
|
|
};
|
|
use rig_core::completion::ToolDefinition;
|
|
use serde_json::{Value, json};
|
|
use sqlx::PgPool;
|
|
use uuid::Uuid;
|
|
|
|
use crate::db::Actor;
|
|
use crate::mcp::McpHub;
|
|
use crate::memory::{CreateMemoryInput, MemoryService};
|
|
|
|
pub struct ToolCtx {
|
|
pub sandbox: std::sync::Arc<dyn SandboxProvider>,
|
|
pub computer: Mutex<Option<ComputerRef>>,
|
|
pub context: Mutex<AdapterContext>,
|
|
pub mode: ComputerMode,
|
|
pub bot_id: String,
|
|
pub vision: bool,
|
|
pub gui_block: Mutex<Option<String>>,
|
|
pub previous_frame: Mutex<Option<String>>,
|
|
pub elements: Mutex<Vec<UiElement>>,
|
|
pub miss_streak: Mutex<u32>,
|
|
pub last_click_key: Mutex<Option<String>>,
|
|
pub click_misses: Mutex<u32>,
|
|
pub takeover_requested: Mutex<bool>,
|
|
pub pool: PgPool,
|
|
pub memory: MemoryService,
|
|
pub actor: Actor,
|
|
pub session_id: String,
|
|
pub run_id: String,
|
|
pub memory_enabled: bool,
|
|
pub mcp: McpHub,
|
|
}
|
|
|
|
impl ToolCtx {
|
|
pub fn computer_ref(&self) -> ComputerRef {
|
|
self.computer
|
|
.lock()
|
|
.unwrap()
|
|
.clone()
|
|
.expect("computer sandbox is not ready")
|
|
}
|
|
|
|
pub fn adapter(&self) -> AdapterContext {
|
|
self.context.lock().unwrap().clone()
|
|
}
|
|
}
|
|
|
|
pub fn tool_definitions(memory_enabled: bool) -> Vec<ToolDefinition> {
|
|
let mut definitions = vec![
|
|
ToolDefinition {
|
|
name: "computer_observe".into(),
|
|
description: "Capture a fresh desktop screenshot plus numbered targets (page DOM when Chromium is open, otherwise AT-SPI buttons/fields, otherwise windows). Only when the user asked you to do something on the computer. Not for greetings, chat, or listing files — those need no screenshot. The image attaches only if the screen changed.".into(),
|
|
parameters: json!({"type":"object","properties":{}}),
|
|
},
|
|
ToolDefinition {
|
|
name: "computer_act".into(),
|
|
description: "Drive native desktop GUI (dialogs, file manager, XFCE) when the user asked you to operate the computer. Prefer element id from the latest observation (AT-SPI, not pixels). For Chromium pages use the browser tool — computer_act clicks on the browser window are rejected. x,y are 1280x800 fallback for canvas / no-tree widgets.".into(),
|
|
parameters: json!({
|
|
"type":"object",
|
|
"properties":{
|
|
"actions":{
|
|
"type":"array",
|
|
"items":{
|
|
"type":"object",
|
|
"properties":{
|
|
"kind":{"type":"string","enum":["click","move","down","up","hover","drag","type","key","scroll","wait","focus"]},
|
|
"element":{"type":"number"},
|
|
"title":{"type":"string"},
|
|
"x2":{"type":"number"},
|
|
"y2":{"type":"number"},
|
|
"x":{"type":"number"},
|
|
"y":{"type":"number"},
|
|
"text":{"type":"string"},
|
|
"key":{"type":"string"},
|
|
"modifiers":{"type":"array","items":{"type":"string"}},
|
|
"button":{"type":"string","enum":["left","right"]},
|
|
"double":{"type":"boolean"},
|
|
"direction":{"type":"string","enum":["up","down"]},
|
|
"amount":{"type":"number"},
|
|
"ms":{"type":"number"}
|
|
},
|
|
"required":["kind"]
|
|
}
|
|
},
|
|
"observe":{"type":"boolean"},
|
|
"settle_ms":{"type":"number"}
|
|
},
|
|
"required":["actions"]
|
|
}),
|
|
},
|
|
ToolDefinition {
|
|
name: "shell".into(),
|
|
description: "Run a command inside this bot's computer when the user asked you to do work there. Not for greetings, small talk, or questions you can answer in text.".into(),
|
|
parameters: json!({
|
|
"type":"object",
|
|
"properties":{
|
|
"command":{"type":"string"},
|
|
"cwd":{"type":"string"}
|
|
},
|
|
"required":["command"]
|
|
}),
|
|
},
|
|
ToolDefinition {
|
|
name: "list_files".into(),
|
|
description: "List files in this bot's home when the user asked about workspace files. Not for greetings or chat — do not ls the desktop to start a conversation.".into(),
|
|
parameters: json!({"type":"object","properties":{"path":{"type":"string"}}}),
|
|
},
|
|
ToolDefinition {
|
|
name: "read_file".into(),
|
|
description: "Read a UTF-8 text file from this bot's home when the user asked about that file. Not for greetings or chat.".into(),
|
|
parameters: json!({"type":"object","properties":{"path":{"type":"string"}},"required":["path"]}),
|
|
},
|
|
ToolDefinition {
|
|
name: "write_file".into(),
|
|
description: "Write a UTF-8 file into this bot's home when the user asked you to save something there. Not for greetings or chat.".into(),
|
|
parameters: json!({
|
|
"type":"object",
|
|
"properties":{"path":{"type":"string"},"content":{"type":"string"}},
|
|
"required":["path","content"]
|
|
}),
|
|
},
|
|
ToolDefinition {
|
|
name: "open_path".into(),
|
|
description: "Open a workspace file or http(s) URL on the desktop when the user asked you to open it. Not for greetings or chat.".into(),
|
|
parameters: json!({"type":"object","properties":{"path":{"type":"string"}},"required":["path"]}),
|
|
},
|
|
ToolDefinition {
|
|
name: "browser".into(),
|
|
description: "Control Chromium through the page DOM when the user asked you to use the browser. Prefer this over computer_act for anything in the page. snapshot returns numbered elements and visible text (no screenshot). click/type/navigate by element id or CSS selector. click scrolls off-screen elements into view and, if the control is [disabled], waits up to 45s (waitMs to change) for it to enable before clicking. Ids are renumbered after every page change. The human still sees the live window.".into(),
|
|
parameters: json!({
|
|
"type":"object",
|
|
"properties":{
|
|
"action":{"type":"string","enum":["snapshot","click","type","press","navigate","wait"]},
|
|
"element":{"type":"number"},
|
|
"selector":{"type":"string"},
|
|
"text":{"type":"string"},
|
|
"key":{"type":"string"},
|
|
"url":{"type":"string"},
|
|
"ms":{"type":"number"},
|
|
"waitMs":{"type":"number","description":"click only: how long to wait for a disabled control to enable (default 45000, max 120000)"}
|
|
},
|
|
"required":["action"]
|
|
}),
|
|
},
|
|
ToolDefinition {
|
|
name: "launch_app".into(),
|
|
description: "Launch browser or terminal on the desktop when the user asked you to open an app.".into(),
|
|
parameters: json!({
|
|
"type":"object",
|
|
"properties":{"application":{"type":"string"},"uri":{"type":"string"}},
|
|
"required":["application"]
|
|
}),
|
|
},
|
|
ToolDefinition {
|
|
name: "wait".into(),
|
|
description: "Wait for the screen to change on its own (video playing, page loading, a button that enables later), then return a fresh observation. seconds: 1-60.".into(),
|
|
parameters: json!({
|
|
"type":"object",
|
|
"properties":{"seconds":{"type":"number","minimum":1,"maximum":60},"reason":{"type":"string"}},
|
|
"required":["seconds"]
|
|
}),
|
|
},
|
|
ToolDefinition {
|
|
name: "request_takeover".into(),
|
|
description: "Ask the user to take over the live screen for passwords, 2FA, CAPTCHA, or a login wall when no saved account fits. Never ask them to paste secrets in chat. Prefer use_saved_login when list_accounts has a matching site.".into(),
|
|
parameters: json!({
|
|
"type":"object",
|
|
"properties":{
|
|
"reason":{"type":"string"},
|
|
"site":{"type":"string","description":"Site or app name, e.g. Gmail"},
|
|
"why":{"type":"string","description":"What you will do once they have signed in"}
|
|
},
|
|
"required":["reason"]
|
|
}),
|
|
},
|
|
ToolDefinition {
|
|
name: "list_accounts".into(),
|
|
description: "List saved logins for this bot (site + username only, never passwords). Use before filling a login wall. If the site is missing, tell the human to add it under 帳號 — do not ask them to paste a password.".into(),
|
|
parameters: json!({"type":"object","properties":{}}),
|
|
},
|
|
ToolDefinition {
|
|
name: "use_saved_login".into(),
|
|
description: "Fill the current Chromium login form with a saved account. Pass only accountId from list_accounts. The password never appears in chat. If 2FA or CAPTCHA appears afterwards, call request_takeover.".into(),
|
|
parameters: json!({
|
|
"type":"object",
|
|
"properties":{"accountId":{"type":"string"}},
|
|
"required":["accountId"]
|
|
}),
|
|
},
|
|
ToolDefinition {
|
|
name: "create_schedule".into(),
|
|
description: "Schedule recurring work for this bot. Use when the user says every day / weekdays / every Monday / from now on at 9. cron is five fields in the bot timezone (default Asia/Taipei), e.g. \"0 9 * * 1-5\".".into(),
|
|
parameters: json!({
|
|
"type":"object",
|
|
"properties":{
|
|
"name":{"type":"string"},
|
|
"cron":{"type":"string"},
|
|
"instructions":{"type":"string","description":"What to do each time it fires, written to yourself."},
|
|
"timezone":{"type":"string"}
|
|
},
|
|
"required":["name","cron","instructions"]
|
|
}),
|
|
},
|
|
ToolDefinition {
|
|
name: "list_schedules".into(),
|
|
description: "List this bot's scheduled jobs.".into(),
|
|
parameters: json!({"type":"object","properties":{}}),
|
|
},
|
|
ToolDefinition {
|
|
name: "cancel_schedule".into(),
|
|
description: "Disable or delete a scheduled job by id from list_schedules.".into(),
|
|
parameters: json!({
|
|
"type":"object",
|
|
"properties":{
|
|
"scheduleId":{"type":"string"},
|
|
"delete":{"type":"boolean"}
|
|
},
|
|
"required":["scheduleId"]
|
|
}),
|
|
},
|
|
ToolDefinition {
|
|
name: "use_skill".into(),
|
|
description: "Load the playbook of a skill the human taught this bot by demonstration (see 'Taught skills' in your instructions). Returns intent, inputs and semantic steps to follow with the normal tools.".into(),
|
|
parameters: json!({"type":"object","properties":{"name":{"type":"string"}},"required":["name"]}),
|
|
},
|
|
];
|
|
if memory_enabled {
|
|
definitions.extend([
|
|
ToolDefinition {
|
|
name: "remember".into(),
|
|
description: "Explicitly save a durable user preference or fact for this agent only. Never store passwords, tokens, private keys, or other secrets.".into(),
|
|
parameters: json!({
|
|
"type":"object",
|
|
"properties":{
|
|
"content":{"type":"string"},
|
|
"importance":{"type":"number","minimum":0,"maximum":1}
|
|
},
|
|
"required":["content"]
|
|
}),
|
|
},
|
|
ToolDefinition {
|
|
name: "recall_memory".into(),
|
|
description: "Search durable memories belonging only to this agent.".into(),
|
|
parameters: json!({
|
|
"type":"object",
|
|
"properties":{"query":{"type":"string"},"limit":{"type":"integer","minimum":1,"maximum":20}},
|
|
"required":["query"]
|
|
}),
|
|
},
|
|
ToolDefinition {
|
|
name: "forget_memory".into(),
|
|
description: "Soft-delete one durable memory belonging to this agent by its ID.".into(),
|
|
parameters: json!({
|
|
"type":"object",
|
|
"properties":{"memory_id":{"type":"string","format":"uuid"}},
|
|
"required":["memory_id"]
|
|
}),
|
|
},
|
|
]);
|
|
}
|
|
definitions
|
|
}
|
|
|
|
pub struct ToolOutcome {
|
|
pub text: String,
|
|
pub image: Option<Vec<u8>>,
|
|
pub pause: bool,
|
|
pub blocks: Vec<Value>,
|
|
}
|
|
|
|
pub async fn dispatch(ctx: &ToolCtx, name: &str, args: &Value) -> ToolOutcome {
|
|
match name {
|
|
"computer_observe" => observe(ctx).await,
|
|
"computer_act" => act(ctx, args).await,
|
|
"wait" => wait_then_observe(ctx, args).await,
|
|
"browser" => browser(ctx, args).await,
|
|
"shell" => shell(ctx, args).await,
|
|
"list_files" => list_files(ctx, args).await,
|
|
"read_file" => read_file(ctx, args).await,
|
|
"write_file" => write_file(ctx, args).await,
|
|
"open_path" => open_path(ctx, args).await,
|
|
"launch_app" => launch_app(ctx, args).await,
|
|
"remember" => remember(ctx, args).await,
|
|
"recall_memory" => recall_memory(ctx, args).await,
|
|
"forget_memory" => forget_memory(ctx, args).await,
|
|
"request_takeover" => {
|
|
*ctx.takeover_requested.lock().unwrap() = true;
|
|
ToolOutcome {
|
|
text: args
|
|
.get("reason")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("The bot asked you to take control.")
|
|
.to_string(),
|
|
image: None,
|
|
pause: true,
|
|
blocks: login_blocks(args),
|
|
}
|
|
}
|
|
"list_accounts" => list_saved_accounts(ctx).await,
|
|
"use_saved_login" => use_saved_login(ctx, args).await,
|
|
"create_schedule" => create_schedule_tool(ctx, args).await,
|
|
"list_schedules" => list_schedules_tool(ctx).await,
|
|
"cancel_schedule" => cancel_schedule_tool(ctx, args).await,
|
|
"use_skill" => {
|
|
let name = args.get("name").and_then(Value::as_str).unwrap_or("");
|
|
let skills = crate::skills::saved_skills(&ctx.pool, &ctx.bot_id).await;
|
|
match crate::skills::find_skill(&skills, name) {
|
|
Some(skill) => text_outcome(crate::skills::format_playbook_for_run(skill)),
|
|
None => text_outcome(format!(
|
|
"no taught skill named {name:?}. Available: {}",
|
|
if skills.is_empty() {
|
|
"none".to_string()
|
|
} else {
|
|
skills
|
|
.iter()
|
|
.map(|skill| skill.name.clone())
|
|
.collect::<Vec<_>>()
|
|
.join(", ")
|
|
}
|
|
)),
|
|
}
|
|
}
|
|
other if other.starts_with("mcp_") => match ctx.mcp.call(other, args).await {
|
|
Ok(text) => text_outcome(text),
|
|
Err(error) => text_outcome(format!("MCP 工具失敗：{error}")),
|
|
},
|
|
other => ToolOutcome {
|
|
text: format!("unknown tool {other}"),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
},
|
|
}
|
|
}
|
|
|
|
async fn remember(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
if !ctx.memory_enabled {
|
|
return text_outcome("memory is disabled for this agent");
|
|
}
|
|
let input = CreateMemoryInput {
|
|
content: args
|
|
.get("content")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("")
|
|
.to_string(),
|
|
importance: args
|
|
.get("importance")
|
|
.and_then(Value::as_f64)
|
|
.unwrap_or(0.5) as f32,
|
|
session_id: Some(ctx.session_id.clone()),
|
|
source_run_id: Some(ctx.run_id.clone()),
|
|
source_message_id: None,
|
|
};
|
|
match ctx
|
|
.memory
|
|
.remember(&ctx.pool, &ctx.actor, &ctx.bot_id, input)
|
|
.await
|
|
{
|
|
Ok(item) => text_outcome(json!({"ok":true,"memory":item}).to_string()),
|
|
Err(error) => text_outcome(error),
|
|
}
|
|
}
|
|
|
|
async fn recall_memory(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
if !ctx.memory_enabled {
|
|
return text_outcome("memory is disabled for this agent");
|
|
}
|
|
let query = args.get("query").and_then(Value::as_str).unwrap_or("");
|
|
let limit = args.get("limit").and_then(Value::as_i64);
|
|
match ctx
|
|
.memory
|
|
.recall(&ctx.pool, &ctx.actor, &ctx.bot_id, query, limit)
|
|
.await
|
|
{
|
|
Ok(items) => text_outcome(serde_json::to_string(&items).unwrap_or_else(|_| "[]".into())),
|
|
Err(error) => text_outcome(error),
|
|
}
|
|
}
|
|
|
|
async fn forget_memory(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
if !ctx.memory_enabled {
|
|
return text_outcome("memory is disabled for this agent");
|
|
}
|
|
let Some(id) = args
|
|
.get("memory_id")
|
|
.and_then(Value::as_str)
|
|
.and_then(|id| Uuid::parse_str(id).ok())
|
|
else {
|
|
return text_outcome("memory_id must be a UUID");
|
|
};
|
|
match ctx
|
|
.memory
|
|
.forget(&ctx.pool, &ctx.actor, &ctx.bot_id, id)
|
|
.await
|
|
{
|
|
Ok(deleted) => text_outcome(json!({"ok":deleted}).to_string()),
|
|
Err(error) => text_outcome(error),
|
|
}
|
|
}
|
|
|
|
fn text_outcome(text: impl Into<String>) -> ToolOutcome {
|
|
ToolOutcome {
|
|
text: text.into(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
}
|
|
}
|
|
|
|
fn vision_guard(ctx: &ToolCtx) -> Option<ToolOutcome> {
|
|
if let Some(message) = ctx.gui_block.lock().unwrap().clone() {
|
|
return Some(ToolOutcome {
|
|
text: message,
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
});
|
|
}
|
|
if ctx.vision {
|
|
None
|
|
} else {
|
|
Some(ToolOutcome {
|
|
text: "This model cannot see the screen. Use shell and file tools, or pick a vision model.".into(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
})
|
|
}
|
|
}
|
|
|
|
fn observation_text(note: &str, observation: &ComputerObservation, unchanged: bool) -> String {
|
|
let label = if observation.elements.iter().any(|element| {
|
|
matches!(element.kind.as_deref(), Some("dom") | Some("a11y"))
|
|
}) {
|
|
"Clickable controls"
|
|
} else {
|
|
"Clickable windows"
|
|
};
|
|
format!(
|
|
"{note}{}\n{label}: {}\n{}",
|
|
if unchanged { " (screen unchanged)" } else { "" },
|
|
format_ui_elements(&observation.elements),
|
|
json!({
|
|
"frameId": observation.frame_id,
|
|
"width": observation.width,
|
|
"height": observation.height,
|
|
"capturedAt": observation.captured_at,
|
|
"cursor": observation.cursor,
|
|
"activeWindow": observation.active_window,
|
|
"elements": observation.elements,
|
|
})
|
|
)
|
|
}
|
|
|
|
async fn observe(ctx: &ToolCtx) -> ToolOutcome {
|
|
if let Some(blocked) = vision_guard(ctx) {
|
|
return blocked;
|
|
}
|
|
match ctx.sandbox.observe(&ctx.computer_ref(), &ctx.adapter()).await {
|
|
Ok(observation) => {
|
|
let (observation, note) =
|
|
attach_ui_elements(ctx, observation, "computer observed").await;
|
|
pack_observation(ctx, ¬e, observation)
|
|
}
|
|
Err(error) => ToolOutcome {
|
|
text: error.to_string(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Bounded so it always fits inside the 90s per-tool budget with an observe.
|
|
const MAX_WAIT_SECS: f64 = 60.0;
|
|
|
|
async fn wait_then_observe(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
let seconds = args
|
|
.get("seconds")
|
|
.and_then(Value::as_f64)
|
|
.unwrap_or(5.0)
|
|
.clamp(1.0, MAX_WAIT_SECS);
|
|
tokio::time::sleep(std::time::Duration::from_secs_f64(seconds)).await;
|
|
if vision_guard(ctx).is_some() {
|
|
return text_outcome(format!("waited {seconds:.0}s"));
|
|
}
|
|
match ctx.sandbox.observe(&ctx.computer_ref(), &ctx.adapter()).await {
|
|
Ok(observation) => {
|
|
let note = format!("waited {seconds:.0}s");
|
|
let (observation, note) = attach_ui_elements(ctx, observation, ¬e).await;
|
|
pack_observation(ctx, ¬e, observation)
|
|
}
|
|
Err(error) => text_outcome(format!("waited {seconds:.0}s; observe failed: {error}")),
|
|
}
|
|
}
|
|
|
|
async fn attach_ui_elements(
|
|
ctx: &ToolCtx,
|
|
mut observation: ComputerObservation,
|
|
note: &str,
|
|
) -> (ComputerObservation, String) {
|
|
let page = cdp_snapshot(ctx, false).await;
|
|
let include_browser = page
|
|
.as_ref()
|
|
.map(|page| !page.ok || page.elements.is_empty())
|
|
.unwrap_or(true);
|
|
let a11y = a11y_snapshot(ctx, include_browser).await;
|
|
let page_elements = page
|
|
.as_ref()
|
|
.map(|page| page.elements.as_slice())
|
|
.unwrap_or(&[]);
|
|
let a11y_elements = a11y
|
|
.as_ref()
|
|
.filter(|page| page.ok)
|
|
.map(|page| page.elements.as_slice())
|
|
.unwrap_or(&[]);
|
|
observation.elements = merge_ui_elements(observation.elements, page_elements, a11y_elements);
|
|
let mut note = note.to_string();
|
|
if let Some(page) = page.as_ref().filter(|page| page.ok) {
|
|
if !page.url.is_empty() || !page.title.is_empty() {
|
|
note.push_str(&format!("\nPage: {} {}", page.title, page.url));
|
|
}
|
|
if !page.text.is_empty() {
|
|
note.push_str("\nVisible text:\n");
|
|
note.push_str(&page.text);
|
|
}
|
|
}
|
|
(observation, note)
|
|
}
|
|
|
|
async fn a11y_snapshot(ctx: &ToolCtx, include_browser: bool) -> Option<lazyboy_control::A11yPage> {
|
|
let page = a11y_call(
|
|
ctx,
|
|
json!({"action": "snapshot", "includeBrowser": include_browser}),
|
|
)
|
|
.await;
|
|
if page.ok {
|
|
Some(page)
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
async fn a11y_call(ctx: &ToolCtx, request: serde_json::Value) -> lazyboy_control::A11yPage {
|
|
let adapter = ctx.adapter();
|
|
let display = adapter.display.as_deref().unwrap_or(":1");
|
|
let argv = a11y_command_on(display, &request);
|
|
match ctx
|
|
.sandbox
|
|
.execute(
|
|
&ctx.computer_ref(),
|
|
CommandRequest {
|
|
argv,
|
|
cwd: None,
|
|
timeout_ms: Some(8_000),
|
|
stdin: None,
|
|
},
|
|
&ctx.adapter(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(result) => {
|
|
let raw = if result.stdout.trim().is_empty() {
|
|
result.stderr
|
|
} else {
|
|
result.stdout
|
|
};
|
|
parse_a11y_page(&raw)
|
|
}
|
|
Err(error) => lazyboy_control::A11yPage {
|
|
ok: false,
|
|
error: Some(error.to_string()),
|
|
..lazyboy_control::A11yPage::default()
|
|
},
|
|
}
|
|
}
|
|
|
|
async fn cdp_snapshot(ctx: &ToolCtx, ensure: bool) -> Option<CdpPage> {
|
|
let page = cdp_call(ctx, json!({"action": "snapshot", "ensure": ensure})).await;
|
|
if page.ok { Some(page) } else { None }
|
|
}
|
|
|
|
async fn cdp_call(ctx: &ToolCtx, request: Value) -> CdpPage {
|
|
let adapter = ctx.adapter();
|
|
let display = adapter.display.as_deref().unwrap_or(":1");
|
|
// Clicks may sit through a page's stay timer (CLICK_WAIT_MS in cdp.py).
|
|
let timeout_ms = if request.get("action").and_then(Value::as_str) == Some("click") {
|
|
let wait = request
|
|
.get("waitMs")
|
|
.and_then(Value::as_u64)
|
|
.unwrap_or(45_000)
|
|
.min(120_000);
|
|
wait + 25_000
|
|
} else {
|
|
20_000
|
|
};
|
|
let argv = cdp_command_on(display, adapter.profile_path.as_deref(), &request);
|
|
match ctx
|
|
.sandbox
|
|
.execute(
|
|
&ctx.computer_ref(),
|
|
CommandRequest {
|
|
argv,
|
|
cwd: None,
|
|
timeout_ms: Some(timeout_ms),
|
|
stdin: None,
|
|
},
|
|
&ctx.adapter(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(result) => {
|
|
let raw = if result.stdout.trim().is_empty() {
|
|
result.stderr
|
|
} else {
|
|
result.stdout
|
|
};
|
|
parse_cdp_page(&raw)
|
|
}
|
|
Err(error) => CdpPage {
|
|
ok: false,
|
|
error: Some(error.to_string()),
|
|
..CdpPage::default()
|
|
},
|
|
}
|
|
}
|
|
|
|
async fn browser(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
if let Some(blocked) = vision_guard(ctx) {
|
|
return blocked;
|
|
}
|
|
let action = args
|
|
.get("action")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("snapshot");
|
|
let mut request = json!({
|
|
"action": action,
|
|
"ensure": true,
|
|
});
|
|
for key in ["url", "text", "key", "ms", "selector", "waitMs"] {
|
|
if let Some(value) = args.get(key) {
|
|
request[key] = value.clone();
|
|
}
|
|
}
|
|
if request.get("selector").and_then(Value::as_str).is_none() {
|
|
if let Some(id) = element_id(args.get("element").or_else(|| args.get("id"))) {
|
|
let elements = ctx.elements.lock().unwrap().clone();
|
|
match elements
|
|
.iter()
|
|
.find(|element| u64::from(element.id) == id)
|
|
.and_then(|element| element.selector.clone())
|
|
{
|
|
Some(selector) => request["selector"] = json!(selector),
|
|
None if matches!(action, "click" | "type") => {
|
|
return pause_unknown_element(ctx, id as u32, &elements);
|
|
}
|
|
None => {}
|
|
}
|
|
}
|
|
}
|
|
if matches!(action, "click") && request.get("selector").and_then(Value::as_str).is_none() {
|
|
let known: Vec<String> = ctx
|
|
.elements
|
|
.lock()
|
|
.unwrap()
|
|
.iter()
|
|
.filter(|element| element.selector.is_some())
|
|
.map(|element| format!("[{}] {}", element.id, element.title))
|
|
.take(12)
|
|
.collect();
|
|
return text_outcome(format!(
|
|
"browser click needs {{\"action\":\"click\",\"element\":N}} with a number from the last snapshot, or a CSS selector. You sent: {}. Known elements: {}",
|
|
serde_json::to_string(args).unwrap_or_default(),
|
|
if known.is_empty() {
|
|
"none yet, call browser snapshot first".to_string()
|
|
} else {
|
|
known.join(", ")
|
|
}
|
|
));
|
|
}
|
|
let page = cdp_call(ctx, request).await;
|
|
if !page.elements.is_empty() {
|
|
*ctx.elements.lock().unwrap() = page.elements.clone();
|
|
}
|
|
if !page.ok {
|
|
let mut text = page.error.unwrap_or_else(|| "browser failed".into());
|
|
if !page.elements.is_empty() {
|
|
text.push_str(&format!(
|
|
"\nPage: {} {}\nClickable page elements: {}",
|
|
page.title,
|
|
page.url,
|
|
format_ui_elements(&page.elements)
|
|
));
|
|
}
|
|
return text_outcome(text);
|
|
}
|
|
let mut text = browser_result_text(action, &page);
|
|
if let Some(seconds) = page.waited_seconds {
|
|
text.push_str(&format!(
|
|
" (the control was disabled; waited {seconds:.0}s for it to enable before clicking)"
|
|
));
|
|
}
|
|
if action == "snapshot" {
|
|
return ToolOutcome {
|
|
text,
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
};
|
|
}
|
|
match ctx.sandbox.observe(&ctx.computer_ref(), &ctx.adapter()).await {
|
|
Ok(observation) => {
|
|
let (observation, note) = attach_ui_elements(ctx, observation, &text).await;
|
|
pack_observation(ctx, ¬e, observation)
|
|
}
|
|
Err(_) => ToolOutcome {
|
|
text,
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
},
|
|
}
|
|
}
|
|
|
|
fn browser_result_text(action: &str, page: &CdpPage) -> String {
|
|
if matches!(action, "snapshot" | "navigate") {
|
|
format!(
|
|
"browser {action}\nPage: {} {}\nClickable page elements: {}\nVisible text:\n{}",
|
|
page.title,
|
|
page.url,
|
|
format_ui_elements(&page.elements),
|
|
page.text
|
|
)
|
|
} else {
|
|
format!("browser {action} ok")
|
|
}
|
|
}
|
|
|
|
async fn act(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
if let Some(blocked) = vision_guard(ctx) {
|
|
return blocked;
|
|
}
|
|
let mut args = args.clone();
|
|
let elements = ctx.elements.lock().unwrap().clone();
|
|
let actions_value = args.get("actions").cloned().unwrap_or(Value::Null);
|
|
if let Some(message) = browser_gui_block(&actions_value, &elements) {
|
|
return text_outcome(message);
|
|
}
|
|
let click_key = click_fingerprint(&actions_value);
|
|
if should_block_stale_click(
|
|
*ctx.miss_streak.lock().unwrap(),
|
|
ctx.last_click_key.lock().unwrap().as_deref(),
|
|
click_key.as_deref(),
|
|
) {
|
|
return text_outcome(
|
|
"The last clicks changed nothing. Do not repeat the same click. Options: the control may be disabled until a video/loading finishes (use wait, then re-observe); the target may be off (use browser snapshot and click by element id, or pick a different native control); if the page needs login, CAPTCHA or a human decision, call request_takeover.",
|
|
);
|
|
}
|
|
if let Some(actions) = args.get_mut("actions") {
|
|
if let Err(error) = apply_element_targets(actions, &elements) {
|
|
if let ActionError::UnknownElement(id) = error {
|
|
return pause_unknown_element(ctx, id, &elements);
|
|
}
|
|
return text_outcome(error.to_string());
|
|
}
|
|
if let Some(items) = actions.as_array_mut() {
|
|
apply_semantic_actions(ctx, items, &elements).await;
|
|
}
|
|
}
|
|
let actions = match parse_computer_actions(args.get("actions").unwrap_or(&Value::Null)) {
|
|
Ok(actions) => actions,
|
|
Err(error) => {
|
|
return ToolOutcome {
|
|
text: error.to_string(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
};
|
|
}
|
|
};
|
|
let had_click = actions.iter().any(action_is_click) || click_key.is_some();
|
|
if let Some(key) = click_key {
|
|
*ctx.last_click_key.lock().unwrap() = Some(key);
|
|
}
|
|
match ctx
|
|
.sandbox
|
|
.act(
|
|
&ctx.computer_ref(),
|
|
ActionRequest {
|
|
actions,
|
|
observe: args.get("observe").and_then(Value::as_bool) != Some(false),
|
|
settle_ms: args.get("settle_ms").and_then(Value::as_u64).unwrap_or(350) as u32,
|
|
display: ctx.adapter().display.clone(),
|
|
profile_path: ctx.adapter().profile_path.clone(),
|
|
},
|
|
&ctx.adapter(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(result) => {
|
|
if let Some(observation) = result.observation {
|
|
let (observation, note) = attach_ui_elements(
|
|
ctx,
|
|
observation,
|
|
&format!("completed {} computer action(s)", result.completed),
|
|
)
|
|
.await;
|
|
let unchanged =
|
|
frames_match(ctx.previous_frame.lock().unwrap().as_deref(), &observation);
|
|
let mut outcome = pack_observation(ctx, ¬e, observation);
|
|
note_click_result(ctx, had_click, unchanged, &mut outcome);
|
|
outcome
|
|
} else {
|
|
ToolOutcome {
|
|
text: json!({"ok": true, "completed": result.completed}).to_string(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
}
|
|
}
|
|
}
|
|
Err(error) => ToolOutcome {
|
|
text: error.to_string(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
},
|
|
}
|
|
}
|
|
|
|
fn action_is_click(action: &ComputerAction) -> bool {
|
|
matches!(
|
|
action,
|
|
ComputerAction::Pointer {
|
|
pointer_type: PointerType::Click | PointerType::Down,
|
|
..
|
|
} | ComputerAction::Ref {
|
|
verb: lazyboy_contracts::RefVerb::Click,
|
|
..
|
|
}
|
|
)
|
|
}
|
|
|
|
async fn apply_semantic_actions(ctx: &ToolCtx, items: &mut [Value], elements: &[UiElement]) {
|
|
for item in items {
|
|
let Some(kind) = item.get("kind").and_then(Value::as_str) else {
|
|
continue;
|
|
};
|
|
if !matches!(kind, "click" | "type") {
|
|
continue;
|
|
}
|
|
let Some(target) = item
|
|
.get("target")
|
|
.and_then(Value::as_str)
|
|
.filter(|value| !value.is_empty())
|
|
.map(str::to_string)
|
|
else {
|
|
continue;
|
|
};
|
|
let ref_kind = item
|
|
.get("refKind")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("a11y")
|
|
.to_string();
|
|
let doubled = item.get("double").and_then(Value::as_bool) == Some(true) && kind == "click";
|
|
let text = item
|
|
.get("text")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("")
|
|
.to_string();
|
|
let mut request = json!({"action": kind, "selector": target, "ensure": false});
|
|
if kind == "type" {
|
|
request["text"] = json!(text);
|
|
}
|
|
let ok = if ref_kind == "dom" {
|
|
let page = cdp_call(ctx, request.clone()).await;
|
|
if doubled && page.ok {
|
|
cdp_call(ctx, request).await.ok
|
|
} else {
|
|
page.ok
|
|
}
|
|
} else {
|
|
let page = a11y_call(ctx, request.clone()).await;
|
|
if doubled && page.ok {
|
|
a11y_call(ctx, request).await.ok
|
|
} else {
|
|
page.ok
|
|
}
|
|
};
|
|
if ok {
|
|
if let Some(object) = item.as_object_mut() {
|
|
object.insert("kind".into(), json!("wait"));
|
|
object.insert("ms".into(), json!(80));
|
|
object.remove("target");
|
|
object.remove("x");
|
|
object.remove("y");
|
|
}
|
|
continue;
|
|
}
|
|
let Some(id) = element_id(item.get("element")) else {
|
|
continue;
|
|
};
|
|
let Some(element) = elements.iter().find(|element| u64::from(element.id) == id) else {
|
|
continue;
|
|
};
|
|
if element.is_offscreen() {
|
|
continue;
|
|
}
|
|
if let Some(object) = item.as_object_mut() {
|
|
let (x, y) = element.center();
|
|
object.insert("x".into(), json!(x));
|
|
object.insert("y".into(), json!(y));
|
|
object.remove("target");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A stale element id is the model's mistake, not a reason to park the run
|
|
/// on the human: tell it what is visible now and let it re-observe.
|
|
fn pause_unknown_element(_ctx: &ToolCtx, id: u32, elements: &[UiElement]) -> ToolOutcome {
|
|
ToolOutcome {
|
|
text: format!(
|
|
"element {id} is not on screen any more. Visible now: {}. Call browser snapshot or computer_observe to get fresh ids, then retry.",
|
|
format_ui_elements(elements)
|
|
),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
}
|
|
}
|
|
|
|
/// Clicks that change nothing are common and usually recoverable (disabled
|
|
/// button, video still playing, slightly off target). Coach the model instead
|
|
/// of pausing; it can still call request_takeover when it is truly stuck.
|
|
fn note_click_result(ctx: &ToolCtx, had_click: bool, unchanged: bool, outcome: &mut ToolOutcome) {
|
|
if !had_click {
|
|
return;
|
|
}
|
|
let mut streak = ctx.miss_streak.lock().unwrap();
|
|
if unchanged {
|
|
*streak += 1;
|
|
*ctx.click_misses.lock().unwrap() += 1;
|
|
if *streak >= 2 {
|
|
outcome.text.push_str(
|
|
"\nThe last clicks changed nothing. Do not repeat the same click. Options: the control may be disabled until a video/loading finishes (use wait, then re-observe); the target may be off (use browser snapshot and click by element id, or pick coordinates from a fresh computer_observe); if the page needs login, CAPTCHA or a human decision, call request_takeover.",
|
|
);
|
|
}
|
|
} else {
|
|
*streak = 0;
|
|
}
|
|
}
|
|
|
|
fn pack_observation(ctx: &ToolCtx, note: &str, observation: ComputerObservation) -> ToolOutcome {
|
|
let unchanged = frames_match(ctx.previous_frame.lock().unwrap().as_deref(), &observation);
|
|
*ctx.previous_frame.lock().unwrap() = Some(observation.frame_id.clone());
|
|
*ctx.elements.lock().unwrap() = observation.elements.clone();
|
|
ToolOutcome {
|
|
text: observation_text(note, &observation, unchanged),
|
|
image: if unchanged {
|
|
None
|
|
} else {
|
|
Some(overlay_elements(&observation.image, &observation.elements))
|
|
},
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
}
|
|
}
|
|
|
|
async fn shell(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
let command = args.get("command").and_then(Value::as_str).unwrap_or("");
|
|
let cwd = args.get("cwd").and_then(Value::as_str);
|
|
let cwd = resolve_bot_workspace_cwd(ctx.mode, &ctx.bot_id, cwd)
|
|
.ok()
|
|
.flatten();
|
|
match ctx
|
|
.sandbox
|
|
.execute(
|
|
&ctx.computer_ref(),
|
|
CommandRequest {
|
|
argv: vec!["bash".into(), "-lc".into(), command.into()],
|
|
cwd,
|
|
timeout_ms: Some(60_000),
|
|
stdin: None,
|
|
},
|
|
&ctx.adapter(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(result) => ToolOutcome {
|
|
text: format!("exit {}\n{}\n{}", result.code, result.stdout, result.stderr),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
},
|
|
Err(error) => ToolOutcome {
|
|
text: error.to_string(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
},
|
|
}
|
|
}
|
|
|
|
async fn list_files(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
let requested = args.get("path").and_then(Value::as_str).unwrap_or("");
|
|
let stored = match resolve_bot_workspace_path(ctx.mode, &ctx.bot_id, requested) {
|
|
Ok(path) => path,
|
|
Err(error) => {
|
|
return ToolOutcome {
|
|
text: error.to_string(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
};
|
|
}
|
|
};
|
|
match ctx
|
|
.sandbox
|
|
.list_files(&ctx.computer_ref(), &stored, &ctx.adapter())
|
|
.await
|
|
{
|
|
Ok(entries) => ToolOutcome {
|
|
text: serde_json::to_string(&entries).unwrap_or_else(|_| "[]".into()),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
},
|
|
Err(error) => ToolOutcome {
|
|
text: error.to_string(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
},
|
|
}
|
|
}
|
|
|
|
async fn read_file(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
let requested = args.get("path").and_then(Value::as_str).unwrap_or("");
|
|
let stored = match resolve_bot_workspace_path(ctx.mode, &ctx.bot_id, requested) {
|
|
Ok(path) => path,
|
|
Err(error) => {
|
|
return ToolOutcome {
|
|
text: error.to_string(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
};
|
|
}
|
|
};
|
|
match ctx
|
|
.sandbox
|
|
.read_file(&ctx.computer_ref(), &stored, &ctx.adapter())
|
|
.await
|
|
{
|
|
Ok(bytes) => ToolOutcome {
|
|
text: String::from_utf8_lossy(&bytes).into_owned(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
},
|
|
Err(error) => ToolOutcome {
|
|
text: error.to_string(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
},
|
|
}
|
|
}
|
|
|
|
async fn write_file(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
let requested = args
|
|
.get("path")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("notes.txt");
|
|
let content = args.get("content").and_then(Value::as_str).unwrap_or("");
|
|
let stored = match resolve_bot_workspace_path(ctx.mode, &ctx.bot_id, requested) {
|
|
Ok(path) => path,
|
|
Err(error) => {
|
|
return ToolOutcome {
|
|
text: error.to_string(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
};
|
|
}
|
|
};
|
|
match ctx
|
|
.sandbox
|
|
.write_file(&ctx.computer_ref(), &stored, content.as_bytes(), &ctx.adapter())
|
|
.await
|
|
{
|
|
Ok(()) => ToolOutcome {
|
|
text: json!({"ok": true, "path": requested}).to_string(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
},
|
|
Err(error) => ToolOutcome {
|
|
text: error.to_string(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
},
|
|
}
|
|
}
|
|
|
|
async fn open_path(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
if let Some(blocked) = vision_guard(ctx) {
|
|
return blocked;
|
|
}
|
|
let path = args.get("path").and_then(Value::as_str).unwrap_or("");
|
|
match ctx
|
|
.sandbox
|
|
.act(
|
|
&ctx.computer_ref(),
|
|
ActionRequest {
|
|
actions: vec![lazyboy_contracts::ComputerAction::Open { path: path.into() }],
|
|
observe: true,
|
|
settle_ms: 400,
|
|
display: ctx.adapter().display.clone(),
|
|
profile_path: ctx.adapter().profile_path.clone(),
|
|
},
|
|
&ctx.adapter(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(result) => {
|
|
if let Some(observation) = result.observation {
|
|
pack_observation(ctx, "opened path", observation)
|
|
} else {
|
|
ToolOutcome {
|
|
text: "opened".into(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
}
|
|
}
|
|
}
|
|
Err(error) => ToolOutcome {
|
|
text: error.to_string(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
},
|
|
}
|
|
}
|
|
|
|
async fn launch_app(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
if let Some(blocked) = vision_guard(ctx) {
|
|
return blocked;
|
|
}
|
|
let application = args
|
|
.get("application")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("browser");
|
|
let uri = args.get("uri").and_then(Value::as_str).map(str::to_string);
|
|
match ctx
|
|
.sandbox
|
|
.act(
|
|
&ctx.computer_ref(),
|
|
ActionRequest {
|
|
actions: vec![lazyboy_contracts::ComputerAction::Launch {
|
|
application: application.into(),
|
|
uri,
|
|
}],
|
|
observe: true,
|
|
settle_ms: 500,
|
|
display: ctx.adapter().display.clone(),
|
|
profile_path: ctx.adapter().profile_path.clone(),
|
|
},
|
|
&ctx.adapter(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(result) => {
|
|
if let Some(observation) = result.observation {
|
|
pack_observation(ctx, "launched app", observation)
|
|
} else {
|
|
ToolOutcome {
|
|
text: "launched".into(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
}
|
|
}
|
|
}
|
|
Err(error) => ToolOutcome {
|
|
text: error.to_string(),
|
|
image: None,
|
|
pause: false,
|
|
blocks: Vec::new(),
|
|
},
|
|
}
|
|
}
|
|
|
|
fn login_blocks(args: &Value) -> Vec<Value> {
|
|
let reason = args
|
|
.get("reason")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("")
|
|
.trim();
|
|
let site = args
|
|
.get("site")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("")
|
|
.trim();
|
|
let why = args.get("why").and_then(Value::as_str).unwrap_or("").trim();
|
|
let site = if site.is_empty() { reason } else { site };
|
|
let why = if why.is_empty() { reason } else { why };
|
|
vec![json!({
|
|
"kind": "login",
|
|
"site": site,
|
|
"why": why,
|
|
})]
|
|
}
|
|
|
|
async fn list_saved_accounts(ctx: &ToolCtx) -> ToolOutcome {
|
|
match crate::vault::list_on(&ctx.pool, &ctx.actor, &ctx.bot_id).await {
|
|
Ok(items) => {
|
|
let slim: Vec<Value> = items
|
|
.iter()
|
|
.map(|item| {
|
|
json!({
|
|
"accountId": item.id,
|
|
"site": item.site,
|
|
"host": item.host,
|
|
"username": item.username,
|
|
})
|
|
})
|
|
.collect();
|
|
if slim.is_empty() {
|
|
text_outcome("No saved logins. Ask the human to add one under 帳號, or call request_takeover so they can sign in on the screen.")
|
|
} else {
|
|
text_outcome(json!({"accounts": slim}).to_string())
|
|
}
|
|
}
|
|
Err(error) => text_outcome(error),
|
|
}
|
|
}
|
|
|
|
async fn use_saved_login(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
if let Some(blocked) = vision_guard(ctx) {
|
|
return blocked;
|
|
}
|
|
let Some(account_id) = args.get("accountId").and_then(Value::as_str) else {
|
|
return text_outcome("accountId is required");
|
|
};
|
|
let secret =
|
|
match crate::vault::get_secret_on(&ctx.pool, &ctx.actor, &ctx.bot_id, account_id).await {
|
|
Ok(Some(row)) => row,
|
|
Ok(None) => return text_outcome("that saved account was not found"),
|
|
Err(error) => return text_outcome(error),
|
|
};
|
|
let (account, username, password) = secret;
|
|
let adapter = ctx.adapter();
|
|
let display = adapter.display.as_deref().unwrap_or(":1");
|
|
let request = json!({
|
|
"action": "fill_login",
|
|
"expectedHost": account.host,
|
|
"ensure": false,
|
|
"username": username,
|
|
"password": password,
|
|
"display": display,
|
|
"port": lazyboy_control::devtools_port(display),
|
|
"profile": adapter.profile_path.clone().unwrap_or_default(),
|
|
});
|
|
let argv = cdp_stdin_command_on(display, adapter.profile_path.as_deref());
|
|
let raw = match ctx
|
|
.sandbox
|
|
.execute(
|
|
&ctx.computer_ref(),
|
|
CommandRequest {
|
|
argv,
|
|
cwd: None,
|
|
timeout_ms: Some(20_000),
|
|
stdin: Some(request.to_string()),
|
|
},
|
|
&ctx.adapter(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(result) => {
|
|
if result.stdout.trim().is_empty() {
|
|
result.stderr
|
|
} else {
|
|
result.stdout
|
|
}
|
|
}
|
|
Err(error) => {
|
|
return text_outcome(format!("could not fill login: {error}"));
|
|
}
|
|
};
|
|
let page = parse_cdp_page(&raw);
|
|
if !page.ok {
|
|
return text_outcome(format!(
|
|
"could not fill {} login: {}",
|
|
account.site,
|
|
page.error.unwrap_or_else(|| "unknown error".into())
|
|
));
|
|
}
|
|
let submitted = serde_json::from_str::<Value>(raw.trim())
|
|
.ok()
|
|
.and_then(|value| value.get("submitted").and_then(Value::as_bool))
|
|
.unwrap_or(false);
|
|
text_outcome(format!(
|
|
"Filled {} as {}. {} If a 2FA/CAPTCHA wall is next, call request_takeover.",
|
|
account.site,
|
|
account.username,
|
|
if submitted {
|
|
"Submitted the form."
|
|
} else {
|
|
"Username and password are in the fields; click Sign in if needed."
|
|
}
|
|
))
|
|
}
|
|
|
|
async fn create_schedule_tool(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
let name = args.get("name").and_then(Value::as_str).unwrap_or("");
|
|
let cron = args.get("cron").and_then(Value::as_str).unwrap_or("");
|
|
let instructions = args
|
|
.get("instructions")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("");
|
|
let timezone = args
|
|
.get("timezone")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("Asia/Taipei");
|
|
let state = schedule_state(ctx);
|
|
match crate::schedules::create(
|
|
&state,
|
|
&ctx.actor,
|
|
&ctx.bot_id,
|
|
crate::schedules::CreateSchedule {
|
|
name: name.to_string(),
|
|
cron: cron.to_string(),
|
|
timezone: timezone.to_string(),
|
|
instructions: instructions.to_string(),
|
|
thread_id: Some(ctx.session_id.clone()),
|
|
enabled: true,
|
|
},
|
|
)
|
|
.await
|
|
{
|
|
Ok(row) => {
|
|
let human = crate::schedules::describe_cron(&row.cron);
|
|
ToolOutcome {
|
|
text: format!("Scheduled 「{}」 ({human}).", row.name),
|
|
image: None,
|
|
pause: false,
|
|
blocks: vec![json!({
|
|
"kind": "schedule",
|
|
"scheduleId": row.id,
|
|
"name": row.name,
|
|
"cron": row.cron,
|
|
"human": human,
|
|
})],
|
|
}
|
|
}
|
|
Err(error) => text_outcome(error),
|
|
}
|
|
}
|
|
|
|
async fn list_schedules_tool(ctx: &ToolCtx) -> ToolOutcome {
|
|
let state = schedule_state(ctx);
|
|
match crate::schedules::list(&state, &ctx.actor, &ctx.bot_id).await {
|
|
Ok(rows) => text_outcome(
|
|
json!({
|
|
"schedules": rows.into_iter().map(crate::schedules::public_json).collect::<Vec<_>>()
|
|
})
|
|
.to_string(),
|
|
),
|
|
Err(error) => text_outcome(error),
|
|
}
|
|
}
|
|
|
|
async fn cancel_schedule_tool(ctx: &ToolCtx, args: &Value) -> ToolOutcome {
|
|
let Some(id) = args.get("scheduleId").and_then(Value::as_str) else {
|
|
return text_outcome("scheduleId is required");
|
|
};
|
|
let delete = args.get("delete").and_then(Value::as_bool).unwrap_or(false);
|
|
if delete {
|
|
let result = sqlx::query(
|
|
"DELETE FROM schedules WHERE id=$1 AND bot_id=$2 AND space_id=$3 AND user_id=$4",
|
|
)
|
|
.bind(id)
|
|
.bind(&ctx.bot_id)
|
|
.bind(&ctx.actor.space_id)
|
|
.bind(&ctx.actor.user_id)
|
|
.execute(&ctx.pool)
|
|
.await;
|
|
return match result {
|
|
Ok(done) if done.rows_affected() == 1 => text_outcome("schedule deleted"),
|
|
Ok(_) => text_outcome("schedule not found"),
|
|
Err(error) => text_outcome(error.to_string()),
|
|
};
|
|
}
|
|
let result = sqlx::query(
|
|
"UPDATE schedules SET enabled=false, next_run_at=NULL, updated_at=now()
|
|
WHERE id=$1 AND bot_id=$2 AND space_id=$3 AND user_id=$4",
|
|
)
|
|
.bind(id)
|
|
.bind(&ctx.bot_id)
|
|
.bind(&ctx.actor.space_id)
|
|
.bind(&ctx.actor.user_id)
|
|
.execute(&ctx.pool)
|
|
.await;
|
|
match result {
|
|
Ok(done) if done.rows_affected() == 1 => text_outcome("schedule paused"),
|
|
Ok(_) => text_outcome("schedule not found"),
|
|
Err(error) => text_outcome(error.to_string()),
|
|
}
|
|
}
|
|
|
|
fn schedule_state(ctx: &ToolCtx) -> crate::state::AppState {
|
|
crate::state::AppState {
|
|
db: crate::db::Db {
|
|
pool: ctx.pool.clone(),
|
|
},
|
|
sandbox: ctx.sandbox.clone(),
|
|
data_dir: String::new(),
|
|
auth: crate::auth::AuthConfig::from_env(),
|
|
memory: ctx.memory.clone(),
|
|
mcp: ctx.mcp.clone(),
|
|
calls: crate::state::CallRegistry::default(),
|
|
}
|
|
}
|