4599 lines
170 KiB
Rust
4599 lines
170 KiB
Rust
use lazyboy_harness::execution::{
|
||
AUTONOMOUS_RECOVERY, ExecutionMode, GOAL_CONTINUE, GOAL_INSTRUCTIONS, GoalOutcome, MAX_NUDGES_GOAL,
|
||
MAX_NUDGES_PLAIN, NEEDS_INPUT_MARKER, PLAN_INSTRUCTIONS, StopReason, VERIFY_BEFORE_DONE,
|
||
asks_for_input, goal_request, reports_completion, stop_reason,
|
||
};
|
||
use lazyboy_harness::policy::{ActionObserved, LoopGuard, RunPolicy, Verdict};
|
||
use std::sync::Arc;
|
||
use std::time::Duration;
|
||
|
||
use base64::Engine;
|
||
use futures_util::StreamExt;
|
||
use lazyboy_contracts::{ModelProvider, SessionAttachment};
|
||
use lazyboy_harness::{
|
||
CredentialChain, DynModel, ResolveModelRequest, connect_model, resolve_backend,
|
||
};
|
||
use rig_core::completion::message::{
|
||
AssistantContent, ImageDetail, ImageMediaType, Message, ToolResultContent, UserContent,
|
||
};
|
||
use rig_core::completion::{CompletionModel, ToolDefinition};
|
||
use rig_core::streaming::StreamedAssistantContent;
|
||
use serde_json::{Value, json};
|
||
use uuid::Uuid;
|
||
|
||
use crate::computer::{self, adapter_context_for};
|
||
use crate::db::{Actor, parse_mode};
|
||
use crate::state::AppState;
|
||
use crate::tools::{ToolCtx, ToolOutcome, dispatch, tool_definitions};
|
||
|
||
const SCREENSHOT_CAPTION: &str = "Desktop screenshot (1280x800) with yellow numbered marks. Click by those element ids. The live VNC view has no marks.";
|
||
|
||
const TAKEOVER_RESUME_PROMPT: &str = "The user finished collaborating and released control. Continue the original task from the CURRENT screen. Do not restart from scratch. Element ids and page refs from before the handoff are invalid; use only the fresh observation below.";
|
||
|
||
const SYSTEM_CHAT: &str = "You are this bot's assistant. This message is conversation — a greeting, small talk, a question you can answer from knowledge, planning, or explaining.
|
||
|
||
Reply in text only. Do not try to use the desktop, browser, files, or shell, and do not narrate that you are checking a screen. You have a Linux desktop for later if the user asks you to operate it; this turn does not need it.
|
||
|
||
Be brief and friendly. If they ask who you are, say you can chat and also do work on a computer when they want that.";
|
||
|
||
const SYSTEM: &str = "You are this bot's assistant. You have a Linux desktop you can use, but most conversation does not need it.
|
||
|
||
Reply in text — no tools — for greetings, small talk, questions you can answer from knowledge, planning, or explaining. Do not call computer_observe, computer_act, browser, launch_app, open_path, wait, list_files, exec, or shell just to check the screen or because a desktop exists. A hello does not need a screenshot or a file listing.
|
||
|
||
Use tools only when the user wants something done on the computer: open a site, click through a UI, run a command, read/write workspace files, or follow a taught skill. Route directly by task:
|
||
1) Website, video, search, or anything in Chromium: use browser first. Navigate directly, then snapshot/click/type/press by element id. Do not use shell/curl to inspect a web page. For canvas or controls the browser tool cannot operate, call computer_observe and use Cua computer_act coordinates from that fresh screenshot. Gmail/Outlook with an authorized connector use that API — never fall back to the browser after POLICY_DENIED.
|
||
For an unknown file mutation result, use file_operation_status with targetOperationId to inspect its receipt; never automatically repeat the mutation.
|
||
2) Workspace files or commands: use list_files, read_file_bytes, search_files, stat_file, read_file, patch_file, move_file, write_file, or exec. Use shell only when the user asked to watch the visible GUI terminal.
|
||
3) Connected services: use an MCP tool when it directly matches the task. Packages installed on this Computer (computer_mcp) run inside the assigned container, not the API. Long web forms: prefer form_fill with locators from the latest snapshot; it stops on the first failed field.
|
||
4) Opening a local file or non-browser app: use open_path or launch_app.
|
||
5) Native GUI with no DOM (dialogs, file manager, XFCE): use computer_act by element id. Those ids are AT-SPI controls, not window boxes.
|
||
|
||
exec returns real stdout/stderr/exit from the bound Computer. Prefer argv. The visible `shell` tool types into a Cua-controlled terminal on the shared VNC screen and returns screenshots — use it for TUI collaboration, not ordinary commands. A timeout on either tool does not prove the process stopped. File tools are native (text, hash, no screenshot); read_file supports start_line and lines, and binary files are not decoded as UTF-8. Pixel clicks, the visible GUI terminal, and opening desktop apps need a vision model. A text-only model can still read computer_observe as an element tree, drive Chromium through browser snapshot/click/type by element id, and use exec/list_files/read_file/write_file.
|
||
|
||
When you ARE using the desktop: the human can interact with the same live screen while you work; this does not pause your task. Prefer browser/native element actions over moving the shared pointer. If the screen changes unexpectedly, observe again and continue from the current state; do not undo human changes or replay an uncertain click. Request human assistance only when the task needs it. Only the latest screenshot you received is current; they may have interacted since. Call computer_observe before coordinate clicks, after navigation, when the outcome is uncertain, and before describing what is on screen. Never guess the screen state from files, history or memory. Never kill or restart the browser, display, or desktop processes; if the browser tool reports it is unavailable, use computer_observe / computer_act on the existing window instead.
|
||
|
||
When you use the browser tool:
|
||
- snapshot first; click {\"action\":\"click\",\"element\":N}; type {\"action\":\"type\",\"element\":N,\"text\":\"...\"}; open a URL with navigate. CSS selectors are not supported (SELECTOR_UNSUPPORTED); use the numbered snapshot ref.
|
||
- Yellow numbered marks on the screenshot match the element list. Click the number, not guessed pixels.
|
||
- Elements tagged [below viewport ...] / [above viewport ...] are outside the visible area but still clickable by id; the click scrolls to them. Do not scroll manually just to reach them.
|
||
- A control is disabled only when its entry says [disabled]. Never claim a button is disabled, counting down or loading unless the element list or the screenshot shows that.
|
||
- Clicking a [disabled] control is fine: the click waits up to 45s for it to become enabled (stay timers, short videos) and then clicks. So when Next is disabled, just click Next. Only if the click reports it is still disabled, call wait with a longer time and click again.
|
||
- Element ids are renumbered whenever the page changes; after navigation use the ids from the newest result, not older ones.
|
||
- If the control is not in the element list at all, take a fresh snapshot; it may not be rendered yet.
|
||
|
||
When a click changes nothing: do not repeat it blindly. Take a fresh snapshot, read the [disabled] tags and the page text, then act. Pages that need patience (training videos, quizzes, slow forms) are normal: keep working through them step by step and report progress in one short sentence when done.
|
||
|
||
Start clear work immediately with one short sentence in the user's language and call tools in the same turn. Report briefly when completing a meaningful phase, changing approach, or encountering a blocker; do not narrate every tool call or repeat that you are thinking. A progress sentence never completes the task or asks the human to say continue. Never narrate element lists, screenshots, or English logs. When a tool fails, explain what failed and how you will recover in your next update. Recover from temporary errors by observing current state and choosing a different action; never replay an uncertain mutation blindly. Before any necessary stop, state what is completed, what remains, the specific blocker, and the next action needed. Never silently stop or claim a failed tool succeeded.
|
||
|
||
Waiting is a tool call, never a reply. Ending your turn with \"waiting for X\" stops the whole run; nobody resumes it. If something must finish first, call wait (or click, which waits) and continue.
|
||
|
||
Multi-step tasks and taught skills: you are done only when the playbook's check passes (for example the course shows completed, the form shows a confirmation). Do not stop with a status sentence in the middle; keep calling tools until the check passes or you are truly blocked, then say exactly why. Never repeat an earlier reply word for word; describe the current screen.
|
||
|
||
When the work is done, write one short summary in the user's language: what you did and where the result is. Do not take it back, do not add a second reply that says it is unfinished, and do not keep checking the screen after you already reported success. If you are blocked, say so once with [NEEDS_INPUT].
|
||
|
||
When you genuinely must stop and need the human — a decision only they can make, a credential you do not have, a file that is missing, or a result they must approve — first say what you already did and what the next step would be, then end that reply with a standalone [NEEDS_INPUT] line. The run pauses for their answer and resumes exactly where you stopped. Never use it when the task is simply finished.
|
||
|
||
You cannot: solve arbitrary CAPTCHA or 2FA (except one connection_check for a Cloudflare checkbox), use an account that is not in this bot's vault, act in the physical world, or install a GUI app that is not on the Computer. You can: Chromium via browser, workspace files via list_files/read_file/write_file/exec, and the native desktop when the task needs it. If you are blocked, say so on the first reply with [NEEDS_INPUT]. Do not retry the same failing click or login. Do not call computer_observe, browser, or wait for a question you can answer in text.
|
||
|
||
Never ask for passwords, codes, or tokens in chat. At a login wall: call list_accounts, then use_saved_login {accountId} when a saved account matches. For a simple Cloudflare connection-check checkbox, first take computer_observe and use connection_check once with coordinates from that screenshot. Check the returned page content before continuing; disappearance of the checkbox alone is not success. Never reload repeatedly or restart the browser to retry. For other CAPTCHA, 2FA, an unsuccessful connection check, or no matching saved login, call request_takeover with site and why so the human signs in on YOUR screen. Recurring work uses create_schedule (five-field cron, Asia/Taipei unless told otherwise).
|
||
|
||
computer_act examples (native windows only):
|
||
- {\"kind\":\"click\",\"element\":1}
|
||
- {\"kind\":\"type\",\"element\":1,\"text\":\"filename.pdf\"}
|
||
- {\"kind\":\"click\",\"x\":N,\"y\":N} (canvas / no numbered control)
|
||
- {\"kind\":\"type\",\"text\":\"...\"}
|
||
- {\"kind\":\"key\",\"key\":\"Return\"}
|
||
- {\"kind\":\"focus\",\"title\":\"Open File\"}
|
||
- wait: {\"seconds\":30,\"reason\":\"video playing\"}
|
||
|
||
On a Team Computer, relative files live in your bot folder; use shared/ for shared work. User-attached files appear in inbox/ for two hours only — chat history does not keep the bytes. Open them with open_path when you need the original file. Finish the user's task.";
|
||
|
||
pub async fn send(
|
||
state: &AppState,
|
||
actor: &Actor,
|
||
bot_id: &str,
|
||
thread_id: &str,
|
||
text: &str,
|
||
client_nonce: Option<&str>,
|
||
blocks: &[Value],
|
||
attachments: &[SessionAttachment],
|
||
) -> Result<Value, String> {
|
||
if crate::skills::recording_skill(state.pool(), bot_id)
|
||
.await
|
||
.is_some()
|
||
{
|
||
return Err("示範進行中:先按「完成示範」或「取消」,再送訊息。".into());
|
||
}
|
||
let decoded = crate::attachments::decode_incoming(attachments)?;
|
||
if text.trim().is_empty() && decoded.is_empty() {
|
||
return Err("empty message".into());
|
||
}
|
||
let stored_body = crate::attachments::caption_for_title(text, &decoded);
|
||
let mut stored_blocks = blocks.to_vec();
|
||
stored_blocks.extend(crate::attachments::stored_blocks(&decoded));
|
||
// Who answers is decided before the row lock: an `@name` settles it for
|
||
// free, and the one model call that sometimes stands in for it must never
|
||
// hold this thread's transaction open.
|
||
let room = crate::routing::room_for_thread(state, thread_id).await?;
|
||
// One run per chosen member — `@name` wins, then the picker's short list,
|
||
// then the host. A room message keeps this list with it so the chat can say
|
||
// who is about to talk; a direct message records nothing extra.
|
||
let mut member_ids: Vec<String> = match room.as_ref() {
|
||
Some(room) => {
|
||
crate::routing::audience_for(state, actor, room, thread_id, text)
|
||
.await
|
||
.targets
|
||
}
|
||
None => vec![bot_id.to_string()],
|
||
};
|
||
if member_ids.is_empty() {
|
||
member_ids.push(bot_id.to_string());
|
||
}
|
||
let reply_bot_ids: Option<Vec<String>> = room.map(|_| member_ids.clone());
|
||
let mut tx = state
|
||
.pool()
|
||
.begin()
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
let scoped: Option<(String, Option<String>)> = sqlx::query_as(
|
||
"SELECT t.title, t.room_id FROM threads t JOIN bots b ON b.id=t.bot_id
|
||
WHERE t.id=$1 AND t.bot_id=$2 AND t.space_id=$3 AND t.user_id=$4
|
||
AND b.space_id=$3 AND b.user_id=$4 AND t.status='active'
|
||
FOR UPDATE OF t",
|
||
)
|
||
.bind(thread_id)
|
||
.bind(bot_id)
|
||
.bind(&actor.space_id)
|
||
.bind(&actor.user_id)
|
||
.fetch_optional(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
let Some((current_title, room_id)) = scoped else {
|
||
return Err("session not found".into());
|
||
};
|
||
if let Some(nonce) = client_nonce {
|
||
let existing: Option<(String, Option<String>)> = sqlx::query_as(
|
||
"SELECT id, run_id FROM messages WHERE thread_id=$1 AND client_nonce=$2",
|
||
)
|
||
.bind(thread_id)
|
||
.bind(nonce)
|
||
.fetch_optional(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
if let Some((message_id, run_id)) = existing {
|
||
tx.rollback().await.map_err(|error| error.to_string())?;
|
||
return Ok(json!({
|
||
"messageId": message_id,
|
||
"runId": run_id,
|
||
"duplicate": true,
|
||
"queued": true
|
||
}));
|
||
}
|
||
}
|
||
// A message that continues an existing task belongs to that task: reuse its
|
||
// run id so the loop resumes from the saved harness state instead of
|
||
// starting over and fighting for the same desktop. That covers /goal
|
||
// steering and every run paused while waiting for an answer. A human who
|
||
// still holds the mouse is never interrupted by an incoming message.
|
||
let merged_run: Option<String> = if room_id.is_none() {
|
||
sqlx::query_scalar(
|
||
"SELECT r.id FROM runs r
|
||
WHERE r.bot_id=$1 AND r.thread_id=$2
|
||
AND (
|
||
r.status IN ('queued','leased','running')
|
||
OR r.status='waiting_input'
|
||
OR (r.status='waiting_takeover' AND NOT EXISTS (
|
||
SELECT 1 FROM computers c JOIN bots b ON b.computer_id=c.id
|
||
WHERE b.id=r.bot_id AND c.control_holder='user'))
|
||
)
|
||
ORDER BY r.created_at ASC LIMIT 1",
|
||
)
|
||
.bind(bot_id)
|
||
.bind(thread_id)
|
||
.fetch_optional(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?
|
||
} else {
|
||
None
|
||
};
|
||
let merged = merged_run.is_some();
|
||
let run_id = merged_run.unwrap_or_else(|| Uuid::new_v4().to_string());
|
||
if merged {
|
||
// The answer itself arrives through the steering read; waking the run
|
||
// only makes it claimable again and retires the pending question.
|
||
sqlx::query(
|
||
"UPDATE runs SET status='queued', retry_count=0,
|
||
checkpoint=checkpoint-'awaitResume'-'taskReport', updated_at=now()
|
||
WHERE id=$1 AND status IN ('waiting_input','waiting_takeover')",
|
||
)
|
||
.bind(&run_id)
|
||
.execute(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
}
|
||
let message_id = Uuid::new_v4().to_string();
|
||
let seq: i32 = sqlx::query_scalar(
|
||
"UPDATE threads
|
||
SET next_message_seq=next_message_seq+1, updated_at=now()
|
||
WHERE id=$1 RETURNING next_message_seq-1",
|
||
)
|
||
.bind(thread_id)
|
||
.fetch_one(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
sqlx::query(
|
||
"INSERT INTO messages (id,thread_id,seq,role,body,blocks,run_id,client_nonce,reply_bot_ids)
|
||
VALUES ($1,$2,$3,'user',$4,$5,$6,$7,$8)",
|
||
)
|
||
.bind(&message_id)
|
||
.bind(thread_id)
|
||
.bind(seq)
|
||
.bind(&stored_body)
|
||
.bind(json!(stored_blocks))
|
||
.bind(&run_id)
|
||
.bind(client_nonce)
|
||
.bind(&reply_bot_ids)
|
||
.execute(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
if crate::sessions::is_default_session_title(¤t_title) {
|
||
sqlx::query("UPDATE threads SET title=$2 WHERE id=$1")
|
||
.bind(thread_id)
|
||
.bind(crate::sessions::title_from_first_message(&stored_body))
|
||
.execute(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
}
|
||
for (index, member_id) in member_ids.iter().enumerate() {
|
||
if merged && index == 0 {
|
||
continue;
|
||
}
|
||
let member_run = if index == 0 {
|
||
run_id.clone()
|
||
} else {
|
||
Uuid::new_v4().to_string()
|
||
};
|
||
sqlx::query(
|
||
"INSERT INTO runs (id,space_id,bot_id,thread_id,user_id,status,prompt,checkpoint)
|
||
VALUES ($1,$2,$3,$4,$5,'queued',$6,$7)",
|
||
)
|
||
.bind(&member_run)
|
||
.bind(&actor.space_id)
|
||
.bind(member_id)
|
||
.bind(thread_id)
|
||
.bind(&actor.user_id)
|
||
.bind(&stored_body)
|
||
.bind(json!({"messageSeq":seq}))
|
||
.execute(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
}
|
||
let event_seq: i32 = sqlx::query_scalar(
|
||
"UPDATE threads SET next_event_seq=next_event_seq+1 WHERE id=$1 RETURNING next_event_seq",
|
||
)
|
||
.bind(thread_id)
|
||
.fetch_one(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
sqlx::query(
|
||
"INSERT INTO events (id,thread_id,seq,type,payload) VALUES ($1,$2,$3,'message.created',$4)",
|
||
)
|
||
.bind(Uuid::new_v4().to_string())
|
||
.bind(thread_id)
|
||
.bind(event_seq)
|
||
.bind(json!({"id":message_id,"seq":seq,"role":"user","body":stored_body,"runId":run_id}))
|
||
.execute(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
let queued_behind_active = if merged {
|
||
true
|
||
} else {
|
||
sqlx::query_scalar(
|
||
"SELECT EXISTS(SELECT 1 FROM runs WHERE bot_id=$1 AND id<>$2
|
||
AND status IN ('queued','leased','running','waiting_input','waiting_takeover'))",
|
||
)
|
||
// The question is about the bot that will actually answer, which in a
|
||
// room is whoever routing put first — not the host the message was
|
||
// addressed to.
|
||
.bind(
|
||
member_ids
|
||
.first()
|
||
.cloned()
|
||
.unwrap_or_else(|| bot_id.to_string()),
|
||
)
|
||
.bind(&run_id)
|
||
.fetch_one(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?
|
||
};
|
||
if let Some(room_id) = room_id.as_deref() {
|
||
sqlx::query("UPDATE rooms SET updated_at=now() WHERE id=$1")
|
||
.bind(room_id)
|
||
.execute(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
}
|
||
tx.commit().await.map_err(|error| error.to_string())?;
|
||
// The user message and its event were written in the transaction above
|
||
// rather than through `sessions::append_event`, so this is where the open
|
||
// chat windows get told to read it: every other tab sees the message without
|
||
// waiting for its next poll.
|
||
state.wakes.wake(thread_id);
|
||
if let Err(error) =
|
||
crate::attachments::stage_for_bots(state, actor, &member_ids, &decoded).await
|
||
{
|
||
tracing::warn!("stage attachments for {message_id}: {error}");
|
||
}
|
||
Ok(json!({
|
||
"messageId": message_id,
|
||
"runId": run_id,
|
||
"duplicate": false,
|
||
"queued": true,
|
||
"queuedBehindActive": queued_behind_active
|
||
}))
|
||
}
|
||
|
||
/// `worker_loop` claim projection: run id, bot id, thread id, prompt, user id, space id.
|
||
type RetryCandidateRow = (String, String, String, String, String, String);
|
||
|
||
pub async fn worker_loop(state: AppState) {
|
||
let inflight = Arc::new(tokio::sync::Semaphore::new(16));
|
||
let lease_owner = format!("api-{}", Uuid::new_v4());
|
||
// Jobs run on their Computer and survive an API restart. Reconcile with
|
||
// the supervisor, or invalidate on Computer generation change; starting
|
||
// an API worker must not invalidate other workers' live jobs.
|
||
// Without a knock the loop sits on its 200 ms timer before it can see a run
|
||
// that was queued a moment ago; going straight to the claim query is what
|
||
// makes the thinking indicator follow the message instead of the timer.
|
||
let mut wakes = state.wakes.subscribe();
|
||
loop {
|
||
tokio::select! {
|
||
_ = wakes.wait_any() => {}
|
||
_ = tokio::time::sleep(Duration::from_millis(200)) => {}
|
||
}
|
||
crate::operations::flush_outbox(state.pool()).await;
|
||
let interrupted: Vec<(String, String, String)> = sqlx::query_as(
|
||
"WITH doomed AS (
|
||
UPDATE runs SET status='failed',error=$1,completed_at=now(),
|
||
lease_owner=NULL, lease_expires_at=NULL
|
||
WHERE status IN ('leased','running') AND lease_expires_at<now()
|
||
AND COALESCE((checkpoint->>'toolsStarted')::boolean,false)
|
||
RETURNING id, thread_id, bot_id
|
||
)
|
||
SELECT id, thread_id, bot_id FROM doomed LIMIT 50",
|
||
)
|
||
.bind(INTERRUPTED_ERROR)
|
||
.fetch_all(state.pool())
|
||
.await
|
||
.unwrap_or_default();
|
||
for (orphan_run, orphan_thread, orphan_bot) in interrupted {
|
||
report_run_failure(
|
||
&state,
|
||
&orphan_run,
|
||
&orphan_thread,
|
||
&orphan_bot,
|
||
INTERRUPTED_ERROR,
|
||
)
|
||
.await;
|
||
}
|
||
let Ok(permit) = inflight.clone().try_acquire_owned() else {
|
||
continue;
|
||
};
|
||
let queued: Result<Option<RetryCandidateRow>, _> = sqlx::query_as(
|
||
"WITH candidate AS (
|
||
SELECT r.id
|
||
FROM runs r
|
||
WHERE r.retry_count < r.max_retries
|
||
AND (
|
||
r.status='queued'
|
||
OR (
|
||
r.status IN ('leased','running')
|
||
AND (r.lease_expires_at IS NULL OR r.lease_expires_at < now())
|
||
)
|
||
)
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM runs a
|
||
WHERE a.bot_id=r.bot_id AND a.id<>r.id
|
||
AND a.status IN ('leased','running','waiting_input','waiting_takeover')
|
||
AND (a.lease_expires_at IS NULL OR a.lease_expires_at >= now())
|
||
)
|
||
ORDER BY CASE WHEN r.status='queued' THEN 1 ELSE 0 END, r.created_at
|
||
FOR UPDATE SKIP LOCKED
|
||
LIMIT 1
|
||
)
|
||
UPDATE runs r
|
||
SET status='leased', lease_owner=$1,
|
||
lease_expires_at=now()+interval '5 minutes',
|
||
lease_fence=lease_fence+1, retry_count=retry_count+1, updated_at=now()
|
||
FROM candidate c WHERE r.id=c.id
|
||
RETURNING r.id,r.bot_id,r.thread_id,r.prompt,r.user_id,r.space_id",
|
||
)
|
||
.bind(&lease_owner)
|
||
.fetch_optional(state.pool())
|
||
.await;
|
||
let Ok(Some((run_id, bot_id, thread_id, prompt, user_id, space_id))) = queued else {
|
||
drop(permit);
|
||
continue;
|
||
};
|
||
let state = state.clone();
|
||
let owner = lease_owner.clone();
|
||
tokio::spawn(async move {
|
||
let _permit = permit;
|
||
let actor = Actor { user_id, space_id };
|
||
let result = tokio::select! {
|
||
result = execute_run(&state, &actor, &owner, &run_id, &bot_id, &thread_id, &prompt) => result,
|
||
error = keep_run_alive(&state, &run_id, &owner) => Err(error),
|
||
};
|
||
if let Err(error) = result
|
||
{
|
||
tracing::error!("run {run_id} failed: {error}");
|
||
let retryable = retryable_run_error(&error);
|
||
let next_status: Option<String> = sqlx::query_scalar(
|
||
"UPDATE runs
|
||
SET status=CASE WHEN $4 AND retry_count < max_retries AND NOT COALESCE((checkpoint->>'toolsStarted')::boolean,false) THEN 'queued' ELSE 'failed' END,
|
||
error=$2, completed_at=CASE WHEN $4 AND retry_count < max_retries AND NOT COALESCE((checkpoint->>'toolsStarted')::boolean,false) THEN NULL ELSE now() END,
|
||
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||
WHERE id=$1 AND lease_owner=$3 AND status IN ('leased','running') RETURNING status",
|
||
)
|
||
.bind(&run_id)
|
||
.bind(&error)
|
||
.bind(&owner)
|
||
.bind(retryable)
|
||
.fetch_optional(state.pool())
|
||
.await
|
||
.ok()
|
||
.flatten();
|
||
if next_status.as_deref() == Some("failed") {
|
||
report_run_failure(&state, &run_id, &thread_id, &bot_id, &error).await;
|
||
let _ = crate::sessions::append_event(
|
||
&state,
|
||
&thread_id,
|
||
"run.failed",
|
||
json!({"runId":run_id,"error":error}),
|
||
)
|
||
.await;
|
||
}
|
||
let _ = computer::release_screen_execution(&state, &run_id).await;
|
||
let _ = sqlx::query(
|
||
"UPDATE computers SET execution_bot_id = NULL, execution_run_id = NULL, execution_lease_expires_at = NULL, updated_at = now()
|
||
WHERE execution_run_id = $1",
|
||
)
|
||
.bind(&run_id)
|
||
.execute(state.pool())
|
||
.await;
|
||
}
|
||
});
|
||
}
|
||
}
|
||
|
||
async fn execute_run(
|
||
state: &AppState,
|
||
actor: &Actor,
|
||
lease_owner: &str,
|
||
run_id: &str,
|
||
bot_id: &str,
|
||
thread_id: &str,
|
||
prompt: &str,
|
||
) -> Result<(), String> {
|
||
let started = sqlx::query(
|
||
"UPDATE runs SET status='running', started_at=COALESCE(started_at,now()), updated_at=now()
|
||
WHERE id=$1 AND lease_owner=$2 AND lease_expires_at>now() AND status='leased'",
|
||
)
|
||
.bind(run_id)
|
||
.bind(lease_owner)
|
||
.execute(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
if started.rows_affected() != 1 {
|
||
let status: Option<String> = sqlx::query_scalar("SELECT status FROM runs WHERE id=$1")
|
||
.bind(run_id)
|
||
.fetch_optional(state.pool())
|
||
.await
|
||
.ok()
|
||
.flatten();
|
||
if halt_from_status(status.as_deref()).is_some() {
|
||
return Ok(());
|
||
}
|
||
return Err("run lease was lost before execution".into());
|
||
}
|
||
let _ = crate::sessions::append_event(
|
||
state,
|
||
thread_id,
|
||
"run.started",
|
||
json!({"runId":run_id,"botId":bot_id}),
|
||
)
|
||
.await;
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"run",
|
||
json!({"event": "started", "task": crate::monitor::snippet(prompt, 160)}),
|
||
)
|
||
.await;
|
||
|
||
let bot = state
|
||
.db
|
||
.get_bot(actor, bot_id)
|
||
.await
|
||
.map_err(|error| error.to_string())?
|
||
.ok_or_else(|| "bot not found".to_string())?;
|
||
let computer = state
|
||
.db
|
||
.get_computer(bot.computer_id.as_deref().unwrap_or(""))
|
||
.await
|
||
.map_err(|error| error.to_string())?
|
||
.ok_or_else(|| "computer not found".to_string())?;
|
||
|
||
let (model, vision, model_id) = bot_model(state, actor, &bot, thread_id).await?;
|
||
let skills = crate::skills::saved_skills(state.pool(), bot_id).await;
|
||
|
||
let ctx = Arc::new(ToolCtx {
|
||
sandbox: state.sandbox.clone(),
|
||
computer: std::sync::Mutex::new(None),
|
||
context: std::sync::Mutex::new(adapter_context_for(
|
||
actor,
|
||
bot_id,
|
||
"run",
|
||
None,
|
||
Some(run_id),
|
||
)),
|
||
mode: parse_mode(&computer.scope),
|
||
bot_id: bot_id.to_string(),
|
||
vision,
|
||
model_id,
|
||
gui_block: std::sync::Mutex::new(None),
|
||
previous_frame: std::sync::Mutex::new(None),
|
||
previous_signature: std::sync::Mutex::new(None),
|
||
delivered_frame: std::sync::Mutex::new(None),
|
||
delivered_signature: std::sync::Mutex::new(None),
|
||
delivered_model_id: std::sync::Mutex::new(None),
|
||
force_image: std::sync::Mutex::new(false),
|
||
pending_image_delivery: std::sync::Mutex::new(None),
|
||
elements: std::sync::Mutex::new(Vec::new()),
|
||
miss_streak: std::sync::Mutex::new(0),
|
||
last_click_key: std::sync::Mutex::new(None),
|
||
click_misses: std::sync::Mutex::new(0),
|
||
takeover_requested: std::sync::Mutex::new(false),
|
||
connection_check_attempted: std::sync::Mutex::new(false),
|
||
pool: state.pool().clone(),
|
||
memory: state.memory.clone(),
|
||
actor: actor.clone(),
|
||
session_id: thread_id.to_string(),
|
||
run_id: run_id.to_string(),
|
||
memory_enabled: bot.memory_enabled && state.memory.globally_enabled(),
|
||
mcp: state.mcp.clone(),
|
||
last_operation_id: std::sync::Mutex::new(None),
|
||
});
|
||
|
||
let mut defs = tool_definitions(ctx.memory_enabled);
|
||
let computer_mcp_bound = crate::installed_mcp::any_bound(state.pool(), actor, bot_id)
|
||
.await
|
||
.unwrap_or(false);
|
||
let mcp_defs = state.mcp.definitions_for(actor).await;
|
||
let granted: Vec<String> = mcp_defs.iter().map(|tool| tool.name.clone()).collect();
|
||
let allowed = lazyboy_control::filter_run_tool_names(
|
||
defs.iter().map(|tool| tool.name.as_str()),
|
||
computer_mcp_bound,
|
||
&granted,
|
||
);
|
||
defs.retain(|tool| allowed.iter().any(|name| name == &tool.name));
|
||
if !mcp_defs.is_empty() {
|
||
defs.extend(mcp_defs);
|
||
}
|
||
let (mut summary, mut summary_seq, plan_shown): (String, i32, bool) = sqlx::query_as(
|
||
"SELECT history_summary, history_summary_seq, plan_shown FROM threads
|
||
WHERE id=$1 AND space_id=$2 AND user_id=$3",
|
||
)
|
||
.bind(thread_id)
|
||
.bind(&actor.space_id)
|
||
.bind(&actor.user_id)
|
||
.fetch_one(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
let checkpoint: Value = sqlx::query_scalar("SELECT checkpoint FROM runs WHERE id=$1")
|
||
.bind(run_id)
|
||
.fetch_one(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
let current_seq: i32 = checkpoint
|
||
.get("messageSeq")
|
||
.and_then(Value::as_i64)
|
||
.map(|seq| seq as i32)
|
||
.unwrap_or(i32::MAX);
|
||
let resume_after_takeover = checkpoint
|
||
.get("resumeAfterTakeover")
|
||
.and_then(Value::as_bool)
|
||
.unwrap_or(false);
|
||
if resume_after_takeover {
|
||
let _ = sqlx::query(
|
||
"UPDATE runs SET checkpoint = jsonb_set(checkpoint - 'resumeAfterTakeover','{taskReport,state}',to_jsonb('progress'::text)) WHERE id=$1",
|
||
)
|
||
.bind(run_id)
|
||
.execute(state.pool())
|
||
.await;
|
||
}
|
||
// The human answered a task that had paused to ask. Their message is an
|
||
// answer, not a new request, so the loop feeds it in before the next model
|
||
// turn instead of letting the run restart from scratch.
|
||
let await_resume = checkpoint.get("awaitResume").is_some();
|
||
if await_resume {
|
||
let _ = sqlx::query("UPDATE runs SET checkpoint = checkpoint - 'awaitResume' WHERE id=$1")
|
||
.bind(run_id)
|
||
.execute(state.pool())
|
||
.await;
|
||
}
|
||
let history_end = if resume_after_takeover {
|
||
i32::MAX
|
||
} else {
|
||
current_seq
|
||
};
|
||
crate::conversation_context::schedule(state.pool().clone(), actor.clone(), thread_id.to_string(), history_end, &model);
|
||
if summary_seq >= history_end { summary.clear(); summary_seq=0; }
|
||
let recent: Vec<(String, String, Option<String>, Option<String>)> = sqlx::query_as(
|
||
"SELECT role, body, speaker_bot_id, speaker_name FROM (
|
||
SELECT m.role, m.body, m.seq, m.speaker_bot_id, b.name AS speaker_name
|
||
FROM messages m
|
||
LEFT JOIN bots b ON b.id=m.speaker_bot_id
|
||
WHERE m.thread_id=$1 AND m.seq>$2 AND m.seq<$3
|
||
ORDER BY m.seq DESC LIMIT $4
|
||
) history ORDER BY seq ASC",
|
||
)
|
||
.bind(thread_id)
|
||
.bind(history_window_start(summary_seq, current_seq))
|
||
.bind(history_end)
|
||
.bind(thread_history_limit(!summary.trim().is_empty()))
|
||
.fetch_all(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
let mut history: Vec<Message> = Vec::new();
|
||
for (role, body, speaker_id, speaker_name) in recent {
|
||
if role == "user" {
|
||
history.push(Message::User {
|
||
content: vec![UserContent::text(body)],
|
||
});
|
||
} else if speaker_id.as_deref() == Some(bot_id) || speaker_id.is_none() {
|
||
history.push(Message::Assistant {
|
||
id: None,
|
||
content: vec![AssistantContent::text(
|
||
model_facing_reply(&body).to_string(),
|
||
)],
|
||
});
|
||
} else {
|
||
let name = speaker_name.unwrap_or_else(|| "agent".into());
|
||
history.push(Message::User {
|
||
content: vec![UserContent::text(format!("[{name}]: {body}"))],
|
||
});
|
||
}
|
||
}
|
||
let goal_mode = goal_request(prompt).is_some();
|
||
let goal_text = prompt
|
||
.trim()
|
||
.strip_prefix("/goal")
|
||
.unwrap_or("")
|
||
.trim()
|
||
.to_string();
|
||
let file_skill = if !resume_after_takeover && !goal_mode {
|
||
crate::file_skills::slash(prompt, &state.data_dir)
|
||
} else {
|
||
None
|
||
};
|
||
let initial_prompt = if goal_mode {
|
||
format!(
|
||
"Execute this goal until it is verified complete:\n{}",
|
||
goal_text
|
||
)
|
||
} else if let Some((skill, args)) = &file_skill {
|
||
format!(
|
||
"Run the /{} skill with these arguments: {}",
|
||
skill.name, args
|
||
)
|
||
} else {
|
||
prompt.to_string()
|
||
};
|
||
let mut first = if resume_after_takeover {
|
||
vec![UserContent::text(TAKEOVER_RESUME_PROMPT)]
|
||
} else {
|
||
vec![UserContent::text(initial_prompt)]
|
||
};
|
||
let blocks: Vec<Value> = if resume_after_takeover {
|
||
Vec::new()
|
||
} else {
|
||
let raw: Value = sqlx::query_scalar(
|
||
"SELECT blocks FROM messages WHERE thread_id=$1 AND seq=$2 AND role='user'",
|
||
)
|
||
.bind(thread_id)
|
||
.bind(current_seq)
|
||
.fetch_optional(state.pool())
|
||
.await
|
||
.ok()
|
||
.flatten()
|
||
.unwrap_or(json!([]));
|
||
raw.as_array().cloned().unwrap_or_default()
|
||
};
|
||
if !resume_after_takeover {
|
||
first.extend(crate::attachments::llm_parts(state, actor, bot_id, &blocks, vision).await);
|
||
}
|
||
let mut skill_check: Option<String> = None;
|
||
if !resume_after_takeover {
|
||
// The user named a taught skill: hand the model the full playbook up
|
||
// front so it does not have to guess or call use_skill first.
|
||
if let Some(skill) = if goal_mode || file_skill.is_some() {
|
||
None
|
||
} else {
|
||
crate::skills::skill_for_prompt(state.pool(), bot_id, prompt).await
|
||
} {
|
||
first.push(UserContent::text(crate::skills::format_playbook_for_run(
|
||
&skill,
|
||
)));
|
||
skill_check = Some(crate::skills::skill_check_hint(&skill));
|
||
}
|
||
if let Some((skill, args)) = &file_skill {
|
||
first.push(UserContent::text(format!(
|
||
"File skill /{} (read-only source, follow its instructions):\n{}\nArguments: {}",
|
||
skill.name, skill.instructions, args
|
||
)));
|
||
skill_check = Some(format!("完成 /{} 的指令,並用工具驗證結果。", skill.name));
|
||
}
|
||
}
|
||
let mut earlier_replies: Vec<String> = assistant_texts(&history);
|
||
if skill_check.is_some() {
|
||
// A skill run is self-contained. Old chat turns about the same site
|
||
// ("Next is still counting down" x6) otherwise anchor the model into
|
||
// repeating its past conclusions instead of reading the screen.
|
||
history.clear();
|
||
first.push(UserContent::text(
|
||
"Earlier chat history is intentionally omitted for this skill run. Work only from the playbook above and the current screen.",
|
||
));
|
||
}
|
||
let workspace_file = blocks
|
||
.iter()
|
||
.any(|block| block.get("kind").and_then(Value::as_str) == Some("file"));
|
||
// Greetings and small talk must not even *see* desktop tools: models
|
||
// otherwise "check the screen" or `ls` the home on "hi" and boot Docker.
|
||
let chat_only = !resume_after_takeover
|
||
&& !goal_mode
|
||
&& file_skill.is_none()
|
||
&& skill_check.is_none()
|
||
&& !workspace_file
|
||
&& is_plain_chat(prompt);
|
||
let plan_gate = should_gate_on_plan(
|
||
chat_only,
|
||
resume_after_takeover,
|
||
await_resume,
|
||
plan_shown,
|
||
goal_mode || file_skill.is_some() || skill_check.is_some() || needs_plan_first(prompt),
|
||
);
|
||
if chat_only {
|
||
// Memory is recalled separately and injected into the preamble below.
|
||
// Do not expose even memory tools here: a plain greeting must be one
|
||
// model call with no chance of accidentally invoking any capability.
|
||
defs.clear();
|
||
if !summary.is_empty() || history.len() >= 8 { defs.push(crate::conversation_context::definition()); }
|
||
tracing::info!(run_id, "chat-only turn: all tools withheld");
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"notice",
|
||
json!({"text": "這一輪是對白,工具先收起來(不會動到電腦)。"}),
|
||
)
|
||
.await;
|
||
} else if plan_gate {
|
||
defs.clear();
|
||
tracing::info!(run_id, "planning turn: tools and desktop withheld");
|
||
} else {
|
||
mark_plan_shown(state, thread_id).await;
|
||
if prompt_needs_desktop(prompt) {
|
||
set_run_step(state, run_id, computer::STEP_BOOTING).await;
|
||
state.wakes.live(
|
||
thread_id,
|
||
"tool.started",
|
||
json!({"runId":run_id,"botId":bot_id,"name":"boot","step":computer::STEP_BOOTING}),
|
||
);
|
||
let boot_state = state.clone();
|
||
let boot_actor = actor.clone();
|
||
let boot_bot = bot_id.to_string();
|
||
tokio::spawn(async move {
|
||
let _ = computer::boot_for(&boot_state, &boot_actor, &boot_bot, false).await;
|
||
});
|
||
}
|
||
}
|
||
// Computer work keeps going until it verifies or explains a blocker.
|
||
// Plain chat stays short so a confused model cannot burn budget.
|
||
let goal_mode = goal_mode || !chat_only;
|
||
let execution_mode = if chat_only {
|
||
ExecutionMode::Bounded(4)
|
||
} else {
|
||
ExecutionMode::Goal
|
||
};
|
||
let turn_limit = match execution_mode {
|
||
ExecutionMode::Goal => None,
|
||
ExecutionMode::Bounded(limit) => Some(limit as i64),
|
||
};
|
||
let mut nudges: u32 = 0;
|
||
// Announce the first action once. Later updates are meaningful milestones
|
||
// chosen by the model, never generated by a clock or each tool call.
|
||
let mut work_announced = false;
|
||
let mut task_outcome = GoalOutcome::Continue;
|
||
let mut task_artifacts = json!([]);
|
||
let mut observed_files = std::collections::HashMap::<String,String>::new();
|
||
let mut screenshots: u32 = 0;
|
||
let mut screenshot_bytes: u64 = 0;
|
||
if resume_after_takeover {
|
||
if let Some(context)=checkpoint.get("handoffContext").and_then(Value::as_str) {
|
||
first.push(UserContent::text(format!("Previous work record (historical, not current page state; never reuse its element IDs):\n{context}")));
|
||
}
|
||
set_run_step(state, run_id, computer::STEP_HANDOFF).await;
|
||
prepare_run_computer(state, actor, bot_id, run_id, &ctx, true).await?;
|
||
}
|
||
if resume_after_takeover && ctx.gui_block.lock().unwrap().is_none() {
|
||
ctx.request_force_image();
|
||
let outcome = dispatch(&ctx, "computer_observe", &json!({})).await;
|
||
first.push(UserContent::text(outcome.text));
|
||
if let Some(image) = outcome.image {
|
||
screenshot_bytes += image.len() as u64;
|
||
screenshots += 1;
|
||
first.extend(screenshot_parts(image));
|
||
ctx.commit_image_delivery();
|
||
}
|
||
}
|
||
let mut pending = Message::User { content: first };
|
||
if !resume_after_takeover
|
||
&& let (Some(saved_history), Some(saved_pending)) = (
|
||
checkpoint.get("harnessHistory"),
|
||
checkpoint.get("harnessPending"),
|
||
)
|
||
&& let (Ok(restored), Ok(mut next)) = (
|
||
serde_json::from_value::<Vec<Message>>(saved_history.clone()),
|
||
serde_json::from_value::<Message>(saved_pending.clone()),
|
||
)
|
||
{
|
||
history = restored;
|
||
if let Message::User { content } = &mut next {
|
||
content.push(UserContent::text("Resumed after a completed tool batch. Do not repeat completed actions. Observe current browser/desktop before any new mutation; prior element references may be stale."));
|
||
}
|
||
pending = next;
|
||
// Checkpoints strip images, so the model no longer holds the last frame.
|
||
ctx.request_force_image();
|
||
}
|
||
|
||
let mut final_text = String::new();
|
||
let mut spoken: Vec<String> = Vec::new();
|
||
let mut turns: u32 = checkpoint
|
||
.get("harnessTurns")
|
||
.and_then(Value::as_u64)
|
||
.unwrap_or(0)
|
||
.min(u32::MAX as u64) as u32;
|
||
// Messages sent to this same thread while a /goal run is working are
|
||
// steering input. Keep the run alive and deliver each new message once
|
||
// before the next model turn instead of waiting for a second run to win
|
||
// the bot lease.
|
||
let mut steering_seq = checkpoint
|
||
.get("steeringSeq")
|
||
.and_then(Value::as_i64)
|
||
.map(|seq| seq as i32)
|
||
.unwrap_or(current_seq);
|
||
let mut used_gui = false;
|
||
let mut did_work = false;
|
||
// One verification demand per run: enough to catch "I'm done" that isn't,
|
||
// without trapping the model in an endless self-audit.
|
||
let mut verified = false;
|
||
// Round policy. A run gets no quota; it gets a watcher. `LoopGuard` looks
|
||
// for the shape of going in circles - the same action, the same failure, a
|
||
// long stretch with nothing new succeeding - and only the last-resort
|
||
// breakers use turns or wall clock. Wall clock is measured per attempt: a
|
||
// run that parked for three days waiting for a human starts fresh.
|
||
let run_policy = RunPolicy::from_env();
|
||
let mut guard = LoopGuard::with_watch(run_policy, turns);
|
||
let attempt_clock = std::time::Instant::now();
|
||
let memory_started = std::time::Instant::now();
|
||
let memory = if ctx.memory_enabled {
|
||
match state
|
||
.memory
|
||
.recall(state.pool(), actor, bot_id, prompt, None)
|
||
.await
|
||
{
|
||
Ok(items) => {
|
||
let context = state.memory.durable_context(&items);
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"memory",
|
||
json!({
|
||
"event": "recalled", "botId": bot_id,
|
||
"enabled": state.memory.globally_enabled(),
|
||
"elapsedMs": memory_started.elapsed().as_millis() as u64,
|
||
"candidateCount": items.len(), "memories": context.used,
|
||
}),
|
||
)
|
||
.await;
|
||
context.block
|
||
}
|
||
Err(error) => {
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"notice",
|
||
json!({"text": format!("記憶讀取失敗,這輪不用記憶:{error}")}),
|
||
)
|
||
.await;
|
||
tracing::warn!("memory retrieval failed for run {run_id}: {error}");
|
||
String::new()
|
||
}
|
||
}
|
||
} else {
|
||
String::new()
|
||
};
|
||
let system = if chat_only { SYSTEM_CHAT } else { SYSTEM };
|
||
let mut preamble = if bot.instructions.trim().is_empty() {
|
||
system.to_string()
|
||
} else {
|
||
format!(
|
||
"{system}\n\nBot-specific instructions:\n{}",
|
||
bot.instructions.trim()
|
||
)
|
||
};
|
||
|
||
let room_mates: Vec<String> = sqlx::query_scalar(
|
||
"SELECT b.name FROM threads t
|
||
JOIN room_members m ON m.room_id=t.room_id
|
||
JOIN bots b ON b.id=m.bot_id
|
||
WHERE t.id=$1 AND b.id<>$2
|
||
ORDER BY b.name",
|
||
)
|
||
.bind(thread_id)
|
||
.bind(bot_id)
|
||
.fetch_all(state.pool())
|
||
.await
|
||
.unwrap_or_default();
|
||
if !room_mates.is_empty() {
|
||
preamble.push_str(&format!(
|
||
"\n\nYou are {} in a group chat with: {}. Reply as yourself only. Other agents' lines are prefixed with [Name]. Do not speak for them. The human names who should answer: speak when your name is on the message, stay quiet when it is not. When the piece belongs to someone else, say one short line and name them with @TheirName, which hands this one item over once.",
|
||
bot.name,
|
||
room_mates.join("、")
|
||
));
|
||
}
|
||
let mcp_names: Vec<String> = defs
|
||
.iter()
|
||
.filter(|tool| tool.name.starts_with("mcp_"))
|
||
.map(|tool| tool.name.clone())
|
||
.collect();
|
||
if !mcp_names.is_empty() {
|
||
preamble.push_str(&format!(
|
||
"\n\nMCP tools available: {}. Use them when they help complete the user's request.",
|
||
mcp_names.join(", ")
|
||
));
|
||
}
|
||
if !memory.is_empty() {
|
||
preamble.push_str("\n\n");
|
||
preamble.push_str(&memory);
|
||
}
|
||
if file_skill.is_none() && skill_check.is_none() {
|
||
preamble.push_str("\n\n");
|
||
preamble.push_str(crate::conversation_context::INSTRUCTIONS);
|
||
preamble.push_str(" Historical recall is allowed even in a text-only conversation. It never operates the desktop.");
|
||
preamble.push_str(&format!("\nHistorical handoff through message {summary_seq} (may omit details; use read_conversation for original text):\n{summary}\nCurrent user message sequence: {current_seq}. Only a recent window is included below; omitted history remains retrievable."));
|
||
}
|
||
if plan_gate {
|
||
preamble.push_str("\n\n");
|
||
preamble.push_str(PLAN_INSTRUCTIONS);
|
||
} else if goal_mode {
|
||
preamble.push_str("\n\n");
|
||
preamble.push_str(GOAL_INSTRUCTIONS);
|
||
preamble.push_str("\n\n");
|
||
preamble.push_str(AUTONOMOUS_RECOVERY);
|
||
}
|
||
if !chat_only {
|
||
if let Some(index) = crate::skills::skills_preamble(&skills) {
|
||
preamble.push_str("\n\n");
|
||
preamble.push_str(&index);
|
||
}
|
||
let file_skill_index = crate::file_skills::index(&state.data_dir);
|
||
if !file_skill_index.is_empty() {
|
||
preamble.push_str("\n\n");
|
||
preamble.push_str(&file_skill_index);
|
||
}
|
||
if let Ok(accounts) = crate::vault::list_on(state.pool(), actor, bot_id).await
|
||
&& !accounts.is_empty()
|
||
{
|
||
let names = accounts
|
||
.iter()
|
||
.map(|item| format!("{} ({})", item.site, item.username))
|
||
.collect::<Vec<_>>()
|
||
.join(", ");
|
||
preamble.push_str(&format!(
|
||
"\n\nSaved logins (passwords are not shown): {names}. At a login wall call use_saved_login with the matching accountId from list_accounts."
|
||
));
|
||
}
|
||
}
|
||
|
||
if goal_mode { preamble.push_str("\n"); preamble.push_str(crate::task_report::INSTRUCTIONS); }
|
||
let stable_preamble_len=preamble.len();
|
||
while execution_mode.allows_turn(turns) {
|
||
turns = turns.saturating_add(1);
|
||
if goal_mode || (await_resume && !resume_after_takeover) {
|
||
let steering: Vec<(i32, String)> = sqlx::query_as(
|
||
"SELECT seq, body FROM messages
|
||
WHERE thread_id=$1 AND role='user' AND seq>$2
|
||
ORDER BY seq ASC LIMIT 12",
|
||
)
|
||
.bind(thread_id)
|
||
.bind(steering_seq)
|
||
.fetch_all(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
if let Some((last_seq, _)) = steering.last() {
|
||
sqlx::query("UPDATE runs SET checkpoint=(checkpoint || jsonb_build_object('previousTaskReport',COALESCE(checkpoint->'taskReport',checkpoint->'previousTaskReport')))-'taskReport' WHERE id=$1")
|
||
.bind(run_id).execute(state.pool()).await.map_err(|e|e.to_string())?;
|
||
task_outcome = GoalOutcome::Continue;
|
||
steering_seq = *last_seq;
|
||
let guidance = steering
|
||
.iter()
|
||
.map(|(_, body)| body.trim())
|
||
.filter(|body| !body.is_empty())
|
||
.collect::<Vec<_>>();
|
||
if !guidance.is_empty() {
|
||
let text = if goal_mode {
|
||
format!(
|
||
"The newest user message has highest task priority. If this is a correction, update the current task. If it is a new assignment, park older unfinished work and finish this assignment first; older goals must not block its completion. Do not automatically resume parked work:\n{}",
|
||
guidance.join("\n")
|
||
)
|
||
} else {
|
||
format!(
|
||
"The human sent a new message while the task was paused. If it answers your question, continue from the saved work. If it assigns a different task, prioritize that newest task and park the old work. Do not repeat finished steps.\n{}",
|
||
guidance.join("\n")
|
||
)
|
||
};
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"notice",
|
||
json!({
|
||
"turn": turns,
|
||
"text": format!("收到新指示:{}", crate::monitor::snippet(&guidance.join(" / "), 160)),
|
||
}),
|
||
)
|
||
.await;
|
||
if let Message::User { content } = &mut pending {
|
||
content.push(UserContent::text(text));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if let Some(halt) = renew_or_halt(state, run_id, lease_owner).await? {
|
||
return finish_halt(
|
||
state,
|
||
thread_id,
|
||
run_id,
|
||
halt,
|
||
turns,
|
||
screenshots,
|
||
screenshot_bytes,
|
||
&ctx,
|
||
used_gui,
|
||
)
|
||
.await;
|
||
}
|
||
// Watcher first: a checkpoint question belongs to the model turn that
|
||
// is about to happen, and a derailment or breaker parks the run before
|
||
// another model call is paid for.
|
||
let verdict = guard.on_turn(turns, attempt_clock.elapsed());
|
||
match guard.recover_or_halt(verdict, turns) {
|
||
Verdict::Continue => {}
|
||
Verdict::Reflect(text) => {
|
||
tracing::info!(run_id, turn = turns, "round policy: coaching the run");
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"notice",
|
||
json!({
|
||
"turn": turns,
|
||
"text": format!("系統檢查點:{}", crate::monitor::snippet(&text, 160)),
|
||
}),
|
||
)
|
||
.await;
|
||
if let Message::User { content } = &mut pending {
|
||
content.push(UserContent::text(text));
|
||
}
|
||
}
|
||
Verdict::Halt { reason, note } => {
|
||
let limit = match execution_mode {
|
||
ExecutionMode::Bounded(limit) => limit,
|
||
ExecutionMode::Goal => 0,
|
||
};
|
||
return hand_back(
|
||
state,
|
||
bot_id,
|
||
thread_id,
|
||
run_id,
|
||
lease_owner,
|
||
&ctx,
|
||
PauseRequest {
|
||
reason,
|
||
draft: &final_text,
|
||
history: &history,
|
||
turns,
|
||
steering_seq,
|
||
limit,
|
||
note: Some(note),
|
||
screenshots,
|
||
screenshot_bytes,
|
||
used_gui,
|
||
},
|
||
)
|
||
.await;
|
||
}
|
||
}
|
||
// Keep the current assignment and durable work ledger outside the
|
||
// disposable screenshot/tool history, including after provider overflow.
|
||
preamble.truncate(stable_preamble_len);
|
||
if goal_mode {
|
||
let report: Option<Value> = sqlx::query_scalar("SELECT jsonb_build_object('current',checkpoint->'taskReport','previous',checkpoint->'previousTaskReport') FROM runs WHERE id=$1")
|
||
.bind(run_id).fetch_optional(state.pool()).await.map_err(|e|e.to_string())?.flatten();
|
||
let guidance: Vec<(i32,String)> = sqlx::query_as("SELECT seq,body FROM (SELECT seq,body FROM messages WHERE thread_id=$1 AND role='user' AND seq>$2 AND seq<=$3 ORDER BY seq DESC LIMIT 12) recent ORDER BY seq")
|
||
.bind(thread_id).bind(current_seq).bind(steering_seq).fetch_all(state.pool()).await.map_err(|e|e.to_string())?;
|
||
preamble.push_str(&format!("\n\nOriginal assignment (historical; a newer assignment below takes priority and parks older work): {}\nDurable work ledger (verify before relying on completion claims): {}\nLatest user messages (newest assignment takes priority): {}\nContinue remaining work; retrieve original conversation for missing constraints. Do not restart completed steps merely because tool history was compacted.",
|
||
crate::context_fit::truncate_chars(prompt,12_000),
|
||
crate::context_fit::truncate_chars(&report.unwrap_or(json!({})).to_string(),12_000),
|
||
crate::context_fit::truncate_chars(&serde_json::to_string(&guidance).unwrap_or_default(),8_000)));
|
||
}
|
||
drop_history_screenshots(&mut history, &pending);
|
||
let defs_chars = serde_json::to_string(&defs).map(|s| s.len()).unwrap_or(0);
|
||
let fit = crate::context_fit::fit_model_context(
|
||
&mut history,
|
||
&mut pending,
|
||
&preamble,
|
||
defs_chars,
|
||
None,
|
||
);
|
||
if !history.iter().any(has_screenshot) && !has_screenshot(&pending) {
|
||
ctx.request_force_image();
|
||
}
|
||
if fit.compacted {
|
||
tracing::info!(
|
||
run_id,
|
||
turn = turns,
|
||
dropped = fit.dropped,
|
||
chars_before = fit.chars_before,
|
||
chars_after = fit.chars_after,
|
||
"fitted run history into the model context window"
|
||
);
|
||
let _ = save_harness_checkpoint(
|
||
state,
|
||
run_id,
|
||
lease_owner,
|
||
&history,
|
||
&pending,
|
||
turns,
|
||
steering_seq,
|
||
)
|
||
.await;
|
||
}
|
||
set_run_progress(state, run_id, MODEL_STEP, turns, turn_limit).await;
|
||
let model_started = std::time::Instant::now();
|
||
let content = tokio::select! {
|
||
halt = wait_for_halt(state, run_id) => {
|
||
return finish_halt(
|
||
state,
|
||
thread_id,
|
||
run_id,
|
||
halt,
|
||
turns,
|
||
screenshots,
|
||
screenshot_bytes,
|
||
&ctx,
|
||
used_gui,
|
||
)
|
||
.await;
|
||
}
|
||
result = complete_with_retry(
|
||
&model,
|
||
&mut pending,
|
||
&preamble,
|
||
&mut history,
|
||
&defs,
|
||
Trace { state, run_id, thread_id, bot_id, turn: turns },
|
||
) => result
|
||
}?;
|
||
let assistant = Message::Assistant {
|
||
id: None,
|
||
content: content.clone(),
|
||
};
|
||
history.push(pending.clone());
|
||
history.push(assistant);
|
||
|
||
let mut calls = Vec::new();
|
||
let mut turn_text = String::new();
|
||
for item in &content {
|
||
match item {
|
||
AssistantContent::Text(text) => {
|
||
final_text.push_str(&text.text);
|
||
turn_text.push_str(&text.text);
|
||
}
|
||
AssistantContent::ToolCall(call) => calls.push(call.clone()),
|
||
_ => {}
|
||
}
|
||
}
|
||
let model_elapsed = model_started.elapsed().as_millis() as u64;
|
||
tracing::info!(
|
||
run_id,
|
||
turn = turns,
|
||
elapsed_ms = model_elapsed,
|
||
tool_calls = calls.len(),
|
||
"model turn"
|
||
);
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"model",
|
||
json!({
|
||
"turn": turns,
|
||
"elapsedMs": model_elapsed,
|
||
"toolCalls": calls.len(),
|
||
"text": crate::monitor::snippet(&turn_text, 200),
|
||
}),
|
||
)
|
||
.await;
|
||
if plan_gate {
|
||
let draft = {
|
||
let spoken = turn_text.trim();
|
||
if spoken.is_empty() {
|
||
format!(
|
||
"我打算這樣做:{}\n\n要我開始嗎?",
|
||
crate::monitor::snippet(prompt, 80)
|
||
)
|
||
} else if spoken.contains('?') || spoken.contains('?') {
|
||
spoken.to_string()
|
||
} else {
|
||
format!("{spoken}\n\n要我開始嗎?")
|
||
}
|
||
};
|
||
let limit = match execution_mode {
|
||
ExecutionMode::Bounded(limit) => limit,
|
||
ExecutionMode::Goal => 0,
|
||
};
|
||
return hand_back(
|
||
state,
|
||
bot_id,
|
||
thread_id,
|
||
run_id,
|
||
lease_owner,
|
||
&ctx,
|
||
PauseRequest {
|
||
reason: StopReason::NeedsPlan,
|
||
draft: &draft,
|
||
history: &history,
|
||
turns,
|
||
steering_seq,
|
||
limit,
|
||
note: None,
|
||
screenshots,
|
||
screenshot_bytes,
|
||
used_gui,
|
||
},
|
||
)
|
||
.await;
|
||
}
|
||
if calls.is_empty() {
|
||
if goal_mode {
|
||
let report: Option<Value> = sqlx::query_scalar("SELECT checkpoint->'taskReport' FROM runs WHERE id=$1")
|
||
.bind(run_id).fetch_one(state.pool()).await.map_err(|e|e.to_string())?;
|
||
task_outcome = GoalOutcome::Continue;
|
||
if let Some((outcome,body,artifacts))=report.as_ref().and_then(crate::task_report::terminal_report) {
|
||
task_outcome=outcome;final_text=body;task_artifacts=artifacts;
|
||
}
|
||
}
|
||
// A model that quits a playbook early, or parrots an earlier reply
|
||
// instead of describing the current screen, gets pushed back to
|
||
// the tools a few times before the text is accepted.
|
||
let parroted = earlier_replies
|
||
.iter()
|
||
.any(|earlier| earlier == final_text.trim());
|
||
let declared_done = goal_mode
|
||
&& matches!(
|
||
task_outcome,
|
||
GoalOutcome::Complete | GoalOutcome::NeedsInput
|
||
);
|
||
let nudge_limit = if goal_mode {
|
||
MAX_NUDGES_GOAL
|
||
} else {
|
||
MAX_NUDGES_PLAIN
|
||
};
|
||
let mut verify_chosen = false;
|
||
let nudge = if goal_mode {
|
||
match task_outcome {
|
||
GoalOutcome::Continue => Some(format!("{GOAL_CONTINUE}\n{}", crate::task_report::INSTRUCTIONS)),
|
||
GoalOutcome::Complete | GoalOutcome::NeedsInput => None,
|
||
}
|
||
} else if parroted {
|
||
Some(
|
||
"Your reply repeats an earlier message word for word, so it cannot describe the current screen. Below is what the screen shows RIGHT NOW. Act on it with a tool call. Waiting is done by calling wait or by clicking the control (the click waits for it to enable), never by replying. Reply in text only once the task is finished or you are truly blocked (say why).".to_string(),
|
||
)
|
||
} else if let Some(check) = &skill_check {
|
||
Some(format!(
|
||
"The run is not finished; your text reply ended nothing but your own turn. Check: {check}\nBelow is the current screen. If the next control is [disabled], click it anyway — the click waits up to 45s for it to enable — or call wait. Ids marked [below viewport] scroll automatically. Only reply in text when the check passes or you are truly blocked, and then say exactly what blocks you."
|
||
))
|
||
} else if did_work
|
||
&& !verified
|
||
&& !asks_for_input(&final_text)
|
||
&& !reports_completion(&final_text)
|
||
{
|
||
// First stop attempt of a run that already moved something: make
|
||
// it prove the work is finished, or ask the human properly.
|
||
verify_chosen = true;
|
||
Some(VERIFY_BEFORE_DONE.to_string())
|
||
} else {
|
||
None
|
||
};
|
||
let may_nudge =
|
||
nudges < nudge_limit && execution_mode.allows_turn(turns.saturating_add(2));
|
||
if let Some(text) = may_nudge.then_some(nudge).flatten() {
|
||
nudges = nudges.saturating_add(1);
|
||
verified |= verify_chosen;
|
||
tracing::info!(
|
||
run_id,
|
||
turn = turns,
|
||
parroted,
|
||
verify = verify_chosen,
|
||
"nudging model back to tools"
|
||
);
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"notice",
|
||
json!({
|
||
"turn": turns,
|
||
"text": if verify_chosen {
|
||
format!("要求模型先證明做完了才准結束(第 {nudges} 次攔下)。")
|
||
} else if parroted {
|
||
format!("模型重複了舊的回答,叫它看現在畫面繼續做(第 {nudges} 次)。")
|
||
} else {
|
||
format!("模型想用一句話收尾,叫它用工具繼續(第 {nudges} 次)。")
|
||
},
|
||
}),
|
||
)
|
||
.await;
|
||
earlier_replies.push(final_text.trim().to_string());
|
||
if !goal_mode { remember_spoken(&mut spoken, &final_text); }
|
||
final_text.clear();
|
||
let mut content = vec![UserContent::text(text)];
|
||
// A fresh screenshot after a "done" summary makes the model take
|
||
// the result back. Only re-observe when the reply was empty of
|
||
// substance (parrot / skill check), not after a normal wrap-up.
|
||
let reobserve = used_gui && (parroted || skill_check.is_some());
|
||
if reobserve {
|
||
prepare_run_computer(state, actor, bot_id, run_id, &ctx, true).await?;
|
||
}
|
||
if reobserve && ctx.gui_block.lock().unwrap().is_none() {
|
||
set_run_step(state, run_id, "computer_observe: 重新確認畫面").await;
|
||
let outcome = dispatch(&ctx, "computer_observe", &json!({})).await;
|
||
content.push(UserContent::text(outcome.text));
|
||
if let Some(image) = outcome.image {
|
||
screenshot_bytes += image.len() as u64;
|
||
screenshots += 1;
|
||
content.extend(screenshot_parts(image));
|
||
ctx.commit_image_delivery();
|
||
}
|
||
}
|
||
pending = Message::User { content };
|
||
continue;
|
||
}
|
||
// Nothing left to retry: report where the work stands and let the
|
||
// human decide instead of ending the task in silence.
|
||
let stalled = nudges >= nudge_limit && !declared_done;
|
||
if let Some(reason) = if goal_mode && task_outcome != GoalOutcome::Continue { None } else { stop_reason(execution_mode, turns, if goal_mode { "" } else { &final_text }, did_work, stalled) }
|
||
{
|
||
let limit = match execution_mode {
|
||
ExecutionMode::Bounded(limit) => limit,
|
||
ExecutionMode::Goal => 0,
|
||
};
|
||
let draft = if goal_mode { merge_spoken(&spoken,&final_text).replace("[GOAL_BLOCKED]", "").replace("[NEEDS_INPUT]", "") } else { merge_spoken(&spoken,&final_text) };
|
||
return hand_back(
|
||
state,
|
||
bot_id,
|
||
thread_id,
|
||
run_id,
|
||
lease_owner,
|
||
&ctx,
|
||
PauseRequest {
|
||
reason,
|
||
draft: &draft,
|
||
history: &history,
|
||
turns,
|
||
steering_seq,
|
||
limit,
|
||
note: None,
|
||
screenshots,
|
||
screenshot_bytes,
|
||
used_gui,
|
||
},
|
||
)
|
||
.await;
|
||
}
|
||
break;
|
||
}
|
||
let model_already_spoke = !turn_text.trim().is_empty();
|
||
if !model_already_spoke && !work_announced {
|
||
if let Some(call) = calls.first() {
|
||
turn_text = progress_line(&call.function.name, &call.function.arguments);
|
||
final_text = turn_text.clone();
|
||
}
|
||
}
|
||
if !calls.is_empty()
|
||
&& !model_already_spoke
|
||
&& !turn_text.trim().is_empty()
|
||
&& !work_announced
|
||
{
|
||
state.wakes.live(
|
||
thread_id,
|
||
"reply.progress",
|
||
json!({"runId":run_id,"botId":bot_id,"text":turn_text}),
|
||
);
|
||
}
|
||
work_announced = true;
|
||
if !goal_mode { remember_spoken(&mut spoken, &turn_text); }
|
||
final_text.clear();
|
||
let mut results = Vec::new();
|
||
let mut screen: Option<Vec<u8>> = None;
|
||
let mut fail_key = String::new();
|
||
let mut fail_streak = 0u32;
|
||
for call in calls {
|
||
// A newer instruction invalidates this not-yet-executed tool batch.
|
||
// Finish tool-result pairing, then let the next turn reprioritize.
|
||
let newer: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM messages WHERE thread_id=$1 AND role='user' AND seq>$2)")
|
||
.bind(thread_id).bind(steering_seq).fetch_one(state.pool()).await.map_err(|e|e.to_string())?;
|
||
if goal_mode && newer {
|
||
results.push(UserContent::tool_result_for(call.id.clone(),call.provider.clone(),call.function.name.clone(),vec![ToolResultContent::text("Not executed: a newer user instruction arrived. Re-evaluate this action against the newest assignment before using any tools.")]));
|
||
continue;
|
||
}
|
||
{
|
||
sqlx::query("UPDATE runs SET checkpoint=jsonb_set(COALESCE(checkpoint,'{}'::jsonb),'{taskReport,state}',to_jsonb('progress'::text)) WHERE id=$1 AND checkpoint->'taskReport'->>'state' IN ('complete','needs_input')")
|
||
.bind(run_id).execute(state.pool()).await.map_err(|e|e.to_string())?;
|
||
}
|
||
if let Some(halt) = renew_or_halt(state, run_id, lease_owner).await? {
|
||
return finish_halt(
|
||
state,
|
||
thread_id,
|
||
run_id,
|
||
halt,
|
||
turns,
|
||
screenshots,
|
||
screenshot_bytes,
|
||
&ctx,
|
||
used_gui,
|
||
)
|
||
.await;
|
||
}
|
||
let name = call.function.name.clone();
|
||
if !defs.iter().any(|tool| tool.name == name) {
|
||
results.push(UserContent::tool_result_for(
|
||
call.id.clone(),
|
||
call.provider.clone(),
|
||
name,
|
||
vec![ToolResultContent::text(
|
||
"This turn is conversation only; the desktop is not available. Reply in text.",
|
||
)],
|
||
));
|
||
continue;
|
||
}
|
||
if tool_needs_sandbox(&name) {
|
||
prepare_run_computer(state, actor, bot_id, run_id, &ctx, tool_needs_gui(&name))
|
||
.await?;
|
||
}
|
||
used_gui |= matches!(
|
||
name.as_str(),
|
||
"computer_observe"
|
||
| "computer_act"
|
||
| "open_path"
|
||
| "launch_app"
|
||
| "browser"
|
||
| "connection_check"
|
||
| "wait"
|
||
| "use_saved_login"
|
||
| "request_takeover"
|
||
| "shell"
|
||
| "form_fill"
|
||
);
|
||
did_work = true;
|
||
let step = describe_step(&name, &call.function.arguments);
|
||
set_run_progress(state, run_id, &step, turns, turn_limit).await;
|
||
let fence=sqlx::query("UPDATE runs SET checkpoint=COALESCE(checkpoint,'{}'::jsonb)||jsonb_build_object('toolsStarted',true) WHERE id=$1 AND lease_owner=$2 AND status='running'")
|
||
.bind(run_id).bind(lease_owner).execute(state.pool()).await.map_err(|e| e.to_string())?;
|
||
if fence.rows_affected() != 1 {
|
||
return Err("run lease lost before tool dispatch".into());
|
||
}
|
||
let previous_signature = ctx.previous_signature.lock().unwrap().clone();
|
||
let tool_started = std::time::Instant::now();
|
||
let mut tool_timed_out = false;
|
||
state.wakes.live(
|
||
thread_id,
|
||
"tool.started",
|
||
json!({"runId":run_id,"botId":bot_id,"name":name,"step":step}),
|
||
);
|
||
let outcome = tokio::select! {
|
||
halt = wait_for_halt(state, run_id) => {
|
||
return finish_halt(
|
||
state,
|
||
thread_id,
|
||
run_id,
|
||
halt,
|
||
turns,
|
||
screenshots,
|
||
screenshot_bytes,
|
||
&ctx,
|
||
used_gui,
|
||
)
|
||
.await;
|
||
}
|
||
outcome = tokio::time::timeout(
|
||
Duration::from_secs(150),
|
||
dispatch(&ctx, &name, &call.function.arguments),
|
||
) => match outcome {
|
||
Ok(outcome) => outcome,
|
||
// A slow tool is a normal turn, not a dead run: say the
|
||
// effects are unknown and let the model re-observe.
|
||
Err(_) => {
|
||
tool_timed_out = true;
|
||
ToolOutcome {
|
||
text: format!("tool {name} timed out after 150 seconds. Its effects are unknown: observe the current screen or files before anything else, and never repeat a step that already worked."),
|
||
image: None,
|
||
pause: false,
|
||
blocks: Vec::new(),
|
||
error_code: None,
|
||
}
|
||
}
|
||
}
|
||
};
|
||
tracing::info!(
|
||
run_id,
|
||
turn = turns,
|
||
step = %step,
|
||
elapsed_ms = tool_started.elapsed().as_millis() as u64,
|
||
result_chars = outcome.text.chars().count(),
|
||
screenshot = outcome.image.is_some(),
|
||
pause = outcome.pause,
|
||
"tool call"
|
||
);
|
||
let status = if outcome.error_code.is_some() { "error" } else { tool_status(tool_timed_out, outcome.pause, &outcome.text) };
|
||
if status == "ok" {
|
||
fail_key.clear();
|
||
fail_streak = 0;
|
||
// Count consecutive prose-only stalls, not every progress
|
||
// update across a long goal that is doing real tool work.
|
||
if name != "report_task" { nudges = 0; }
|
||
} else {
|
||
let key = format!("{name}:{}", call.function.arguments);
|
||
if key == fail_key {
|
||
fail_streak = fail_streak.saturating_add(1);
|
||
} else {
|
||
fail_key = key;
|
||
fail_streak = 1;
|
||
}
|
||
}
|
||
// Background jobs answer with JSON carrying jobId; surface it so the
|
||
// monitor can show which job a status/cancel line refers to.
|
||
let job_id = serde_json::from_str::<Value>(&outcome.text)
|
||
.ok()
|
||
.and_then(|value| value.get("jobId")?.as_str().map(str::to_string));
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"tool",
|
||
json!({
|
||
"turn": turns,
|
||
"name": name.clone(),
|
||
"step": step.clone(),
|
||
"status": status,
|
||
"errorCode": outcome.error_code,
|
||
"jobId": job_id,
|
||
"operationId": ctx.last_operation_id.lock().unwrap().clone(),
|
||
"elapsedMs": tool_started.elapsed().as_millis() as u64,
|
||
"snippet": crate::monitor::snippet(&outcome.text, 200),
|
||
}),
|
||
)
|
||
.await;
|
||
if status == "ok" {
|
||
if let (Some(before),Some(after)) = (previous_signature.as_ref(),ctx.previous_signature.lock().unwrap().as_ref()) {
|
||
if !lazyboy_control::signatures_similar(before,after) { guard.confirm_progress(turns); }
|
||
}
|
||
}
|
||
if status == "ok" && matches!(name.as_str(), "read_file" | "stat_file") {
|
||
let key = format!("{name}:{}", call.function.arguments);
|
||
if let Some(before) = observed_files.insert(key, outcome.text.clone()) {
|
||
if before != outcome.text { guard.confirm_progress(turns); }
|
||
}
|
||
}
|
||
let verdict = guard.on_action(&ActionObserved {
|
||
name: call.function.name.as_str(),
|
||
args: &call.function.arguments,
|
||
label: &step,
|
||
ok: status == "ok",
|
||
turn: turns,
|
||
changes_state: action_changes_state(&name, &call.function.arguments),
|
||
});
|
||
match guard.recover_or_halt(verdict, turns) {
|
||
Verdict::Continue => {}
|
||
Verdict::Reflect(text) => {
|
||
tracing::info!(run_id, turn = turns, step = %step, "round policy: coaching mid-turn");
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"notice",
|
||
json!({
|
||
"turn": turns,
|
||
"text": format!("系統提示:{}", crate::monitor::snippet(&text, 160)),
|
||
}),
|
||
)
|
||
.await;
|
||
results.push(UserContent::text(text));
|
||
}
|
||
Verdict::Halt { reason, note } => {
|
||
let limit = match execution_mode {
|
||
ExecutionMode::Bounded(limit) => limit,
|
||
ExecutionMode::Goal => 0,
|
||
};
|
||
let draft = if goal_mode { merge_spoken(&spoken,&final_text).replace("[GOAL_BLOCKED]", "").replace("[NEEDS_INPUT]", "") } else { merge_spoken(&spoken,&final_text) };
|
||
return hand_back(
|
||
state,
|
||
bot_id,
|
||
thread_id,
|
||
run_id,
|
||
lease_owner,
|
||
&ctx,
|
||
PauseRequest {
|
||
reason,
|
||
draft: &draft,
|
||
history: &history,
|
||
turns,
|
||
steering_seq,
|
||
limit,
|
||
note: Some(note),
|
||
screenshots,
|
||
screenshot_bytes,
|
||
used_gui,
|
||
},
|
||
)
|
||
.await;
|
||
}
|
||
}
|
||
if fail_streak >= 2 {
|
||
results.push(UserContent::text(
|
||
"LAST FAILURE — do not repeat this action unchanged. Use a different tool, different arguments, or a fresh observation. If only the human can unblock this, say what you already did and end with a standalone [NEEDS_INPUT] line.".to_string(),
|
||
));
|
||
}
|
||
// xAI rejects images inside tool results. Attach a changed
|
||
// screenshot as a following user image instead.
|
||
if let Some(image) = outcome.image {
|
||
screen = Some(image);
|
||
}
|
||
results.push(UserContent::tool_result_for(
|
||
call.id.clone(),
|
||
call.provider.clone(),
|
||
name,
|
||
vec![ToolResultContent::text(crate::context_fit::truncate_chars(
|
||
&outcome.text,
|
||
crate::context_fit::HOT_PART,
|
||
))],
|
||
));
|
||
if !outcome.blocks.is_empty() && !outcome.pause {
|
||
let _ = append_bot_message_with(
|
||
state,
|
||
thread_id,
|
||
run_id,
|
||
bot_id,
|
||
&outcome.text,
|
||
json!(outcome.blocks.clone()),
|
||
)
|
||
.await;
|
||
}
|
||
if outcome.pause {
|
||
// Keep the work record alongside fresh-screen resume. Store as context,
|
||
// not pending tool calls, since a takeover may interrupt a tool batch.
|
||
let handoff_context = crate::context_fit::truncate_chars(&serde_json::to_string(&history).unwrap_or_default(), 24000);
|
||
sqlx::query("UPDATE runs SET checkpoint=COALESCE(checkpoint,'{}'::jsonb)||jsonb_build_object('handoffContext',$2::text) WHERE id=$1")
|
||
.bind(run_id).bind(handoff_context).execute(state.pool()).await.map_err(|e|e.to_string())?;
|
||
sqlx::query(
|
||
"UPDATE runs SET status = 'waiting_takeover',
|
||
checkpoint = COALESCE(checkpoint, '{}'::jsonb)
|
||
|| jsonb_build_object('resumeAfterTakeover', true),
|
||
updated_at = now()
|
||
WHERE id = $1",
|
||
)
|
||
.bind(run_id)
|
||
.execute(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
let _ = computer::takeover(state, actor, bot_id).await;
|
||
append_bot_message_with(
|
||
state,
|
||
thread_id,
|
||
run_id,
|
||
bot_id,
|
||
&outcome.text,
|
||
json!(outcome.blocks),
|
||
)
|
||
.await?;
|
||
let click_misses = *ctx.click_misses.lock().unwrap();
|
||
record_run_metrics(
|
||
state,
|
||
thread_id,
|
||
run_id,
|
||
"run.paused",
|
||
turns,
|
||
screenshots,
|
||
screenshot_bytes,
|
||
click_misses,
|
||
used_gui,
|
||
true,
|
||
)
|
||
.await;
|
||
return Ok(());
|
||
}
|
||
}
|
||
if let Some(png) = screen {
|
||
screenshot_bytes += png.len() as u64;
|
||
screenshots += 1;
|
||
results.extend(screenshot_parts(png));
|
||
ctx.commit_image_delivery();
|
||
}
|
||
pending = Message::User { content: results };
|
||
save_harness_checkpoint(
|
||
state,
|
||
run_id,
|
||
lease_owner,
|
||
&history,
|
||
&pending,
|
||
turns,
|
||
steering_seq,
|
||
)
|
||
.await?;
|
||
if goal_mode {
|
||
// A validated final report already contains the user-facing result.
|
||
// Do not spend another model round merely to repeat it. New steering
|
||
// always gets a turn before finalizing this task.
|
||
let report: Option<Value> = sqlx::query_scalar("SELECT checkpoint->'taskReport' FROM runs r WHERE id=$1 AND NOT EXISTS (SELECT 1 FROM messages m WHERE m.thread_id=r.thread_id AND m.role='user' AND m.seq>$2)")
|
||
.bind(run_id).bind(steering_seq).fetch_optional(state.pool()).await.map_err(|e|e.to_string())?.flatten();
|
||
if let Some((outcome,body,artifacts))=report.as_ref().and_then(crate::task_report::terminal_report) {
|
||
task_outcome=outcome;final_text=body;task_artifacts=artifacts;
|
||
history.push(pending.clone());
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
let status: Option<String> = sqlx::query_scalar("SELECT status FROM runs WHERE id = $1")
|
||
.bind(run_id)
|
||
.fetch_optional(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
if let Some(halt) = halt_from_status(status.as_deref()) {
|
||
return finish_halt(
|
||
state,
|
||
thread_id,
|
||
run_id,
|
||
halt,
|
||
turns,
|
||
screenshots,
|
||
screenshot_bytes,
|
||
&ctx,
|
||
used_gui,
|
||
)
|
||
.await;
|
||
}
|
||
// The turn budget ran out straight after a tool batch, so the model never
|
||
// got a turn to explain itself. Park the run with its state instead of
|
||
// delivering an empty answer that looks like a finished task.
|
||
if let Some(reason) = stop_reason(execution_mode, turns, "", did_work, false) {
|
||
let limit = match execution_mode {
|
||
ExecutionMode::Bounded(limit) => limit,
|
||
ExecutionMode::Goal => 0,
|
||
};
|
||
return hand_back(
|
||
state,
|
||
bot_id,
|
||
thread_id,
|
||
run_id,
|
||
lease_owner,
|
||
&ctx,
|
||
PauseRequest {
|
||
reason,
|
||
draft: &final_text,
|
||
history: &history,
|
||
turns,
|
||
steering_seq,
|
||
limit,
|
||
note: None,
|
||
screenshots,
|
||
screenshot_bytes,
|
||
used_gui,
|
||
},
|
||
)
|
||
.await;
|
||
}
|
||
let needs_input = goal_mode && task_outcome == GoalOutcome::NeedsInput;
|
||
if needs_input {
|
||
let next = Message::User {
|
||
content: vec![UserContent::text(
|
||
"The goal was paused for required user input. Read the user's new information and continue from completed work.",
|
||
)],
|
||
};
|
||
save_harness_checkpoint(
|
||
state,
|
||
run_id,
|
||
lease_owner,
|
||
&history,
|
||
&next,
|
||
turns,
|
||
steering_seq,
|
||
)
|
||
.await?;
|
||
}
|
||
let final_text = merge_spoken(
|
||
&spoken,
|
||
&final_text
|
||
.replace("[GOAL_COMPLETE]", "")
|
||
.replace("[GOAL_BLOCKED]", "")
|
||
.replace("[NEEDS_INPUT]", ""),
|
||
);
|
||
append_bot_message_with(state, thread_id, run_id, bot_id, &final_text, task_artifacts).await?;
|
||
let completed = sqlx::query(
|
||
"UPDATE runs
|
||
SET status=$3, completed_at=CASE WHEN $3='completed' THEN now() ELSE NULL END, updated_at=now(),
|
||
lease_owner=NULL, lease_expires_at=NULL
|
||
WHERE id=$1 AND lease_owner=$2 AND status='running'",
|
||
)
|
||
.bind(run_id)
|
||
.bind(lease_owner)
|
||
.bind(if needs_input { "waiting_input" } else { "completed" })
|
||
.execute(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
if completed.rows_affected() != 1 {
|
||
return Err("run lease was lost before completion".into());
|
||
}
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"run",
|
||
json!({
|
||
"event": if needs_input { "waiting_input" } else { "completed" },
|
||
"reason": if needs_input { Some(final_text.as_str()) } else { None },
|
||
"turns": turns,
|
||
}),
|
||
)
|
||
.await;
|
||
let click_misses = *ctx.click_misses.lock().unwrap();
|
||
let takeover = *ctx.takeover_requested.lock().unwrap();
|
||
record_run_metrics(
|
||
state,
|
||
thread_id,
|
||
run_id,
|
||
if needs_input {
|
||
"run.paused"
|
||
} else {
|
||
"run.completed"
|
||
},
|
||
turns,
|
||
screenshots,
|
||
screenshot_bytes,
|
||
click_misses,
|
||
used_gui,
|
||
takeover,
|
||
)
|
||
.await;
|
||
computer::release_screen_execution(state, run_id).await?;
|
||
sqlx::query(
|
||
"UPDATE computers SET execution_bot_id = NULL, execution_run_id = NULL, execution_lease_expires_at = NULL, updated_at = now()
|
||
WHERE execution_run_id = $1",
|
||
)
|
||
.bind(run_id)
|
||
.execute(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
Ok(())
|
||
}
|
||
|
||
/// Resolve the model a bot runs on (bot override → workspace default).
|
||
/// `session` names the conversation the calls belong to; providers such as
|
||
/// OpenCode Go require it on every request. Returns the connected model and
|
||
/// whether it accepts images.
|
||
pub(crate) async fn bot_model(
|
||
state: &AppState,
|
||
actor: &Actor,
|
||
bot: &crate::db::BotRow,
|
||
session: &str,
|
||
) -> Result<(DynModel, bool, String), String> {
|
||
let space = state
|
||
.db
|
||
.get_space(actor)
|
||
.await
|
||
.map_err(|error| error.to_string())?
|
||
.ok_or_else(|| "workspace not found".to_string())?;
|
||
let provider = bot
|
||
.model_provider
|
||
.as_deref()
|
||
.or(Some(space.default_model_provider.as_str()))
|
||
.unwrap_or("xai")
|
||
.parse::<ModelProvider>()
|
||
.map_err(|error| error.to_string())?;
|
||
let model_id = bot
|
||
.model_id
|
||
.clone()
|
||
.filter(|value| !value.is_empty())
|
||
.or_else(|| Some(space.default_model_id.clone()).filter(|value| !value.is_empty()));
|
||
let (base_url, api_key) =
|
||
crate::workspace::credentials_for_provider(&state.db, actor, &space, provider.as_str())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
let backend = resolve_backend(ResolveModelRequest {
|
||
provider,
|
||
model_id,
|
||
base_url,
|
||
credentials: CredentialChain {
|
||
bot: None,
|
||
space: api_key,
|
||
},
|
||
})
|
||
.map_err(|error| error.to_string())?;
|
||
let model = connect_model(&backend, session).map_err(|error| error.to_string())?;
|
||
Ok((model, backend.capabilities.vision, backend.model_id))
|
||
}
|
||
|
||
pub(crate) async fn complete_once(
|
||
model: &DynModel,
|
||
pending: Message,
|
||
preamble: &str,
|
||
history: &[Message],
|
||
defs: &[ToolDefinition],
|
||
) -> Result<Vec<AssistantContent>, String> {
|
||
match model {
|
||
DynModel::Xai(model) => complete_with(model, pending, preamble, history, defs).await,
|
||
DynModel::OpenAi(model) => complete_with(model, pending, preamble, history, defs).await,
|
||
DynModel::OpenAiResponses(model) => {
|
||
complete_with(model, pending, preamble, history, defs).await
|
||
}
|
||
}
|
||
}
|
||
|
||
fn retryable_run_error(error: &str) -> bool {
|
||
let Some((_, suffix)) = error.split_once("status ") else {
|
||
return true;
|
||
};
|
||
let Some(code) = suffix.get(..3).and_then(|value| value.parse::<u16>().ok()) else {
|
||
return true;
|
||
};
|
||
!matches!(code, 400..=499 if !matches!(code, 408 | 409 | 425 | 429))
|
||
}
|
||
|
||
/// Where a helper that is not the run loop itself writes its diagnostics.
|
||
#[derive(Clone, Copy)]
|
||
struct Trace<'a> {
|
||
state: &'a AppState,
|
||
run_id: &'a str,
|
||
thread_id: &'a str,
|
||
bot_id: &'a str,
|
||
turn: u32,
|
||
}
|
||
|
||
async fn complete_with_retry(
|
||
model: &DynModel,
|
||
pending: &mut Message,
|
||
preamble: &str,
|
||
history: &mut Vec<Message>,
|
||
defs: &[ToolDefinition],
|
||
trace: Trace<'_>,
|
||
) -> Result<Vec<AssistantContent>, String> {
|
||
let mut last = String::new();
|
||
let defs_chars = serde_json::to_string(defs).map(|s| s.len()).unwrap_or(0);
|
||
for attempt in 0..2 {
|
||
if attempt > 0 {
|
||
let failure = crate::monitor::classify_run_error(&last);
|
||
trace.state.wakes.live(trace.thread_id, "reply.progress",
|
||
json!({"runId":trace.run_id,"botId":trace.bot_id,"statusOnly":true,
|
||
"text":format!("{} 正在自動恢復;已完成的操作會保留。", failure.headline)}));
|
||
set_run_step(
|
||
trace.state,
|
||
trace.run_id,
|
||
&format!(
|
||
"{} 正在自動重試模型(第 {}/2 次);保留已完成的操作。",
|
||
failure.headline,
|
||
attempt + 1
|
||
),
|
||
)
|
||
.await;
|
||
}
|
||
let started = std::time::Instant::now();
|
||
let result = stream_once(model, pending.clone(), preamble, history, defs, trace).await;
|
||
match result {
|
||
Ok(content) => return Ok(content),
|
||
Err(error) => {
|
||
let overflow = crate::context_fit::is_context_overflow(&error);
|
||
crate::monitor::record(
|
||
trace.state,
|
||
trace.run_id,
|
||
"retry",
|
||
json!({
|
||
"turn": trace.turn,
|
||
"attempt": attempt + 1,
|
||
"error": error,
|
||
"gaveUp": attempt == 1 || (!overflow && !retryable_run_error(&error)),
|
||
}),
|
||
)
|
||
.await;
|
||
if overflow {
|
||
let now = crate::context_fit::estimate_payload_chars(
|
||
preamble, history, pending, defs_chars,
|
||
);
|
||
let budget = crate::context_fit::overflow_char_budget(&error, now);
|
||
let fit = crate::context_fit::fit_model_context(
|
||
history,
|
||
pending,
|
||
preamble,
|
||
defs_chars,
|
||
Some(budget),
|
||
);
|
||
tracing::warn!(
|
||
attempt = attempt + 1,
|
||
dropped = fit.dropped,
|
||
chars_before = fit.chars_before,
|
||
chars_after = fit.chars_after,
|
||
budget,
|
||
"model rejected an oversized prompt; compacted and retrying"
|
||
);
|
||
crate::monitor::record(
|
||
trace.state,
|
||
trace.run_id,
|
||
"notice",
|
||
json!({
|
||
"turn": trace.turn,
|
||
"text": "模型說這輪對話超過上下文上限,已再壓縮一次後重試,不會重做已完成的步驟。",
|
||
}),
|
||
)
|
||
.await;
|
||
last = error;
|
||
} else if !retryable_run_error(&error) {
|
||
return Err(error);
|
||
} else {
|
||
tracing::warn!(
|
||
attempt = attempt + 1,
|
||
elapsed_ms = started.elapsed().as_millis() as u64,
|
||
"model attempt failed: {error}"
|
||
);
|
||
last = error;
|
||
}
|
||
}
|
||
}
|
||
if attempt < 1 {
|
||
tokio::time::sleep(Duration::from_millis(500 * (1 << attempt))).await;
|
||
}
|
||
}
|
||
Err(last)
|
||
}
|
||
|
||
// Each attempt has its own generation so a late frame from a failed try cannot
|
||
// join the replacement. Already-shown text stays on screen; the next attempt
|
||
// appends. Only public Text events leave this layer.
|
||
async fn stream_once(
|
||
model: &DynModel,
|
||
pending: Message,
|
||
preamble: &str,
|
||
history: &[Message],
|
||
defs: &[ToolDefinition],
|
||
trace: Trace<'_>,
|
||
) -> Result<Vec<AssistantContent>, String> {
|
||
let generation = Uuid::new_v4().to_string();
|
||
// Reconnects restore only this attempt, not every historical text delta.
|
||
// Merge into the harness checkpoint so normal progress saves preserve it.
|
||
sqlx::query("UPDATE runs SET checkpoint=COALESCE(checkpoint,'{}'::jsonb)||$2 WHERE id=$1 AND status='running'")
|
||
.bind(trace.run_id)
|
||
.bind(json!({"replyGeneration":generation}))
|
||
.execute(trace.state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
let _ = crate::sessions::append_event(
|
||
trace.state,
|
||
trace.thread_id,
|
||
"reply.started",
|
||
json!({"runId":trace.run_id,"botId":trace.bot_id,"generation":generation,"quiet":defs.iter().any(|tool|tool.name=="report_task")}),
|
||
)
|
||
.await;
|
||
let result = match model {
|
||
DynModel::Xai(model) => {
|
||
stream_with(model, pending, preamble, history, defs, trace, &generation).await
|
||
}
|
||
DynModel::OpenAi(model) => {
|
||
stream_with(model, pending, preamble, history, defs, trace, &generation).await
|
||
}
|
||
DynModel::OpenAiResponses(model) => {
|
||
stream_with(model, pending, preamble, history, defs, trace, &generation).await
|
||
}
|
||
};
|
||
if result.is_err() {
|
||
let _ = crate::sessions::append_event(
|
||
trace.state,
|
||
trace.thread_id,
|
||
"reply.reset",
|
||
json!({"runId":trace.run_id,"generation":generation}),
|
||
)
|
||
.await;
|
||
}
|
||
result
|
||
}
|
||
|
||
async fn stream_with<M: CompletionModel + Clone>(
|
||
model: &M,
|
||
pending: Message,
|
||
preamble: &str,
|
||
history: &[Message],
|
||
defs: &[ToolDefinition],
|
||
trace: Trace<'_>,
|
||
generation: &str,
|
||
) -> Result<Vec<AssistantContent>, String> {
|
||
let request = model
|
||
.completion_request(pending)
|
||
.preamble(preamble.to_string())
|
||
.messages(history.to_vec())
|
||
.tools(defs.to_vec())
|
||
.build();
|
||
let response = model
|
||
.stream(request)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
let publish_prose = !defs.iter().any(|tool| tool.name == "report_task");
|
||
collect_reply_stream(response, |text| {
|
||
if !publish_prose { return; }
|
||
trace.state.wakes.live(
|
||
trace.thread_id,
|
||
"reply.delta",
|
||
json!({"runId":trace.run_id,"generation":generation,"text":text}),
|
||
);
|
||
})
|
||
.await
|
||
}
|
||
|
||
async fn collect_reply_stream<F>(
|
||
mut response: rig_core::streaming::StreamingCompletionResponse,
|
||
mut publish: F,
|
||
) -> Result<Vec<AssistantContent>, String>
|
||
where
|
||
F: FnMut(String),
|
||
{
|
||
let mut text = String::new();
|
||
let mut terminal = false;
|
||
let mut flush = tokio::time::interval(Duration::from_millis(40));
|
||
flush.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||
loop {
|
||
tokio::select! {
|
||
part = response.next() => {
|
||
let Some(part) = part else { break };
|
||
match part.map_err(|error| error.to_string())? {
|
||
StreamedAssistantContent::Text(part) => text.push_str(&part.text),
|
||
StreamedAssistantContent::Final(_) => terminal = true,
|
||
_ => {}
|
||
}
|
||
}
|
||
_ = flush.tick() => {
|
||
if !text.is_empty() { publish(std::mem::take(&mut text)); }
|
||
}
|
||
}
|
||
}
|
||
// A truncated stream must not execute partially assembled tool calls.
|
||
if !terminal {
|
||
return Err("model stream ended before its final response".into());
|
||
}
|
||
if !text.is_empty() {
|
||
publish(text);
|
||
}
|
||
Ok(response.choice)
|
||
}
|
||
|
||
pub(crate) async fn context_completion(model:&DynModel, preamble:&str, input:&str)->Result<String,String> {
|
||
let pending=Message::User {content:vec![UserContent::text(input)]};
|
||
let content=match model {
|
||
DynModel::Xai(model)=>complete_with(model,pending,preamble,&[],&[]).await?,
|
||
DynModel::OpenAi(model)=>complete_with(model,pending,preamble,&[],&[]).await?,
|
||
DynModel::OpenAiResponses(model)=>complete_with(model,pending,preamble,&[],&[]).await?,
|
||
};
|
||
Ok(content.into_iter().filter_map(|item|match item {AssistantContent::Text(text)=>Some(text.text),_=>None}).collect::<Vec<_>>().join("\n"))
|
||
}
|
||
|
||
async fn complete_with<M>(
|
||
model: &M,
|
||
pending: Message,
|
||
preamble: &str,
|
||
history: &[Message],
|
||
defs: &[ToolDefinition],
|
||
) -> Result<Vec<AssistantContent>, String>
|
||
where
|
||
M: CompletionModel + Clone,
|
||
{
|
||
let request = model
|
||
.completion_request(pending)
|
||
.preamble(preamble.to_string())
|
||
.messages(history.to_vec())
|
||
.tools(defs.to_vec())
|
||
.build();
|
||
let response = model.completion(request)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
Ok(response.choice.into_iter().collect())
|
||
}
|
||
|
||
fn keep_thought(text: &str) -> Option<String> {
|
||
let thought = text
|
||
.replace("[GOAL_COMPLETE]", "")
|
||
.replace("[GOAL_BLOCKED]", "")
|
||
.replace("[NEEDS_INPUT]", "");
|
||
let thought = thought.trim();
|
||
if thought.chars().count() < 8 {
|
||
None
|
||
} else {
|
||
Some(thought.to_string())
|
||
}
|
||
}
|
||
|
||
fn remember_spoken(spoken: &mut Vec<String>, text: &str) {
|
||
if let Some(thought) = keep_thought(text) {
|
||
if spoken.last() != Some(&thought) {
|
||
spoken.push(thought);
|
||
}
|
||
}
|
||
}
|
||
|
||
fn merge_spoken(spoken: &[String], final_text: &str) -> String {
|
||
let body = final_text.trim();
|
||
let lead: Vec<&str> = spoken
|
||
.iter()
|
||
.map(|part| part.trim())
|
||
.filter(|part| !part.is_empty() && (body.is_empty() || !body.contains(*part)))
|
||
.collect();
|
||
if lead.is_empty() {
|
||
return body.to_string();
|
||
}
|
||
if body.is_empty() {
|
||
return lead.join("\n\n");
|
||
}
|
||
format!("{}\n\n{body}", lead.join("\n\n"))
|
||
}
|
||
|
||
fn screenshot_parts(image: Vec<u8>) -> Vec<UserContent> {
|
||
let media = if image.starts_with(&[0xFF, 0xD8, 0xFF]) {
|
||
ImageMediaType::JPEG
|
||
} else {
|
||
ImageMediaType::PNG
|
||
};
|
||
let encoded = base64::engine::general_purpose::STANDARD.encode(image);
|
||
// `Low` makes OpenAI-compatible backends downscale the 1280x800 frame to
|
||
// ~512px before the model sees it: small text vanishes and coordinate
|
||
// clicks land 2-3x off. Coordinates only work at native resolution.
|
||
vec![
|
||
UserContent::text(SCREENSHOT_CAPTION),
|
||
UserContent::image_base64(encoded, Some(media), Some(ImageDetail::High)),
|
||
]
|
||
}
|
||
|
||
fn has_screenshot(message: &Message) -> bool {
|
||
match message {
|
||
Message::User { content } => content
|
||
.iter()
|
||
.any(|part| matches!(part, UserContent::Image(_))),
|
||
_ => false,
|
||
}
|
||
}
|
||
|
||
/// Keep exactly one screenshot in the model's context: the one in `pending`
|
||
/// if it carries a fresh frame, otherwise the most recent one already in
|
||
/// history. Without the fallback a "(screen unchanged)" turn would leave the
|
||
/// model with no picture of the desktop at all.
|
||
/// Plain-text bodies of the assistant turns in a history, used to catch a model
|
||
/// that answers by repeating an earlier reply instead of reading the screen.
|
||
fn assistant_texts(history: &[Message]) -> Vec<String> {
|
||
history
|
||
.iter()
|
||
.filter_map(|message| match message {
|
||
Message::Assistant { content, .. } => Some(
|
||
content
|
||
.iter()
|
||
.filter_map(|item| match item {
|
||
AssistantContent::Text(text) => Some(text.text.trim().to_string()),
|
||
_ => None,
|
||
})
|
||
.collect::<String>(),
|
||
),
|
||
_ => None,
|
||
})
|
||
.filter(|text| !text.is_empty())
|
||
.collect()
|
||
}
|
||
|
||
/// A checkpoint that cannot be written freezes the run: progress stops and every
|
||
/// later hiccup turns fatal. Drop the oldest turns and cap long tool dumps until
|
||
/// the row fits again.
|
||
fn shrink_checkpoint(history: &mut Vec<Message>, pending: &mut Message) {
|
||
const LIMIT: usize = 768 * 1024;
|
||
const MAX_PART: usize = 48 * 1024;
|
||
crate::context_fit::cap_message_parts(pending, MAX_PART);
|
||
history
|
||
.iter_mut()
|
||
.for_each(|message| crate::context_fit::cap_message_parts(message, MAX_PART));
|
||
let fits = |turns: &[Message]| {
|
||
json!({"harnessHistory": turns, "harnessPending": pending})
|
||
.to_string()
|
||
.len()
|
||
<= LIMIT
|
||
};
|
||
while !fits(history) && history.len() > 1 {
|
||
history.remove(0);
|
||
}
|
||
}
|
||
|
||
async fn save_harness_checkpoint(
|
||
state: &AppState,
|
||
run_id: &str,
|
||
owner: &str,
|
||
history: &[Message],
|
||
pending: &Message,
|
||
turns: u32,
|
||
steering_seq: i32,
|
||
) -> Result<(), String> {
|
||
let mut history = history.to_vec();
|
||
let mut pending = pending.clone();
|
||
for message in history.iter_mut().chain(std::iter::once(&mut pending)) {
|
||
if let Message::User { content } = message {
|
||
content.retain(|part| {
|
||
!matches!(part, UserContent::Image(_))
|
||
&& !matches!(part,UserContent::Text(text) if text.text==SCREENSHOT_CAPTION)
|
||
});
|
||
}
|
||
}
|
||
shrink_checkpoint(&mut history, &mut pending);
|
||
let value = json!({"harnessHistory":history,"harnessPending":pending,"toolsStarted":false,"harnessTurns":turns,"steeringSeq":steering_seq});
|
||
// Only a checkpoint that still cannot fit after shrinking fails closed, and
|
||
// loudly: the run keeps its uncertain-effects flag instead of freezing.
|
||
if value.to_string().len() > 1024 * 1024 {
|
||
tracing::error!(
|
||
run_id,
|
||
"harness checkpoint still exceeds 1 MB after shrinking"
|
||
);
|
||
return Ok(());
|
||
}
|
||
let result=sqlx::query("UPDATE runs SET checkpoint=COALESCE(checkpoint,'{}'::jsonb)||$3,updated_at=now() WHERE id=$1 AND lease_owner=$2 AND status='running'")
|
||
.bind(run_id).bind(owner).bind(value).execute(state.pool()).await.map_err(|e|e.to_string())?;
|
||
if result.rows_affected() != 1 {
|
||
return Err("run lease lost while checkpointing".into());
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
fn drop_history_screenshots(history: &mut [Message], pending: &Message) {
|
||
let keep = if has_screenshot(pending) {
|
||
None
|
||
} else {
|
||
history.iter().rposition(has_screenshot)
|
||
};
|
||
for (index, message) in history.iter_mut().enumerate() {
|
||
if keep == Some(index) {
|
||
continue;
|
||
}
|
||
let Message::User { content } = message else {
|
||
continue;
|
||
};
|
||
content.retain(|part| match part {
|
||
UserContent::Image(_) => false,
|
||
UserContent::Text(text) if text.text == SCREENSHOT_CAPTION => false,
|
||
_ => true,
|
||
});
|
||
}
|
||
}
|
||
|
||
/// Cancel every unfinished run of a bot and free the screen/execution leases
|
||
/// it held, so the desktop is available to a human immediately.
|
||
pub(crate) async fn cancel_active_runs(
|
||
state: &AppState,
|
||
bot_id: &str,
|
||
) -> Result<Vec<String>, String> {
|
||
let run_ids: Vec<String> = sqlx::query_scalar(
|
||
"UPDATE runs SET status = 'cancelled', completed_at = now(), updated_at = now()
|
||
WHERE bot_id = $1 AND status IN ('queued','leased','running','waiting_input','waiting_takeover')
|
||
RETURNING id",
|
||
)
|
||
.bind(bot_id)
|
||
.fetch_all(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
for run_id in &run_ids {
|
||
if let Ok(thread_id) =
|
||
sqlx::query_scalar::<_, String>("SELECT thread_id FROM runs WHERE id=$1")
|
||
.bind(run_id)
|
||
.fetch_one(state.pool())
|
||
.await
|
||
{
|
||
let _ = crate::sessions::append_event(
|
||
state,
|
||
&thread_id,
|
||
"run.cancelled",
|
||
json!({"runId":run_id}),
|
||
)
|
||
.await;
|
||
}
|
||
computer::release_screen_execution(state, run_id).await?;
|
||
}
|
||
sqlx::query(
|
||
"UPDATE computers SET execution_bot_id = NULL, execution_run_id = NULL,
|
||
execution_lease_expires_at = NULL, updated_at = now()
|
||
WHERE execution_bot_id = $1",
|
||
)
|
||
.bind(bot_id)
|
||
.execute(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
Ok(run_ids)
|
||
}
|
||
|
||
pub(crate) async fn append_bot_message(
|
||
state: &AppState,
|
||
thread_id: &str,
|
||
run_id: &str,
|
||
bot_id: &str,
|
||
body: &str,
|
||
) -> Result<(), String> {
|
||
append_bot_message_with(state, thread_id, run_id, bot_id, body, json!([])).await
|
||
}
|
||
|
||
pub(crate) async fn append_bot_message_with(
|
||
state: &AppState,
|
||
thread_id: &str,
|
||
run_id: &str,
|
||
bot_id: &str,
|
||
body: &str,
|
||
blocks: Value,
|
||
) -> Result<(), String> {
|
||
// A stop with nothing to say is a bug, not a message. An empty assistant
|
||
// bubble is exactly what made "finished" and "died mid-task" look alike.
|
||
let has_blocks = blocks.as_array().is_some_and(|items| !items.is_empty());
|
||
if body.trim().is_empty() && !has_blocks {
|
||
return Ok(());
|
||
}
|
||
// A bot that names a member may hand this piece over: one hop only, and
|
||
// never to a member who is already working. The lookup happens before the
|
||
// transaction so the message write stays short.
|
||
let handoff = crate::routing::handoff_target(state, run_id, bot_id, body)
|
||
.await
|
||
.unwrap_or_else(|error| {
|
||
tracing::warn!("room hand-off lookup for run {run_id}: {error}");
|
||
None
|
||
});
|
||
let mut tx = state
|
||
.pool()
|
||
.begin()
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
let seq: i32 = sqlx::query_scalar(
|
||
"UPDATE threads SET next_message_seq=next_message_seq+1,updated_at=now()
|
||
WHERE id=$1 RETURNING next_message_seq-1",
|
||
)
|
||
.bind(thread_id)
|
||
.fetch_one(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
let message_id = Uuid::new_v4().to_string();
|
||
sqlx::query(
|
||
"INSERT INTO messages (id,thread_id,seq,role,body,blocks,run_id,speaker_bot_id,reply_bot_ids)
|
||
VALUES ($1,$2,$3,'assistant',$4,$5,$6,$7,$8)",
|
||
)
|
||
.bind(&message_id)
|
||
.bind(thread_id)
|
||
.bind(seq)
|
||
.bind(body)
|
||
.bind(&blocks)
|
||
.bind(run_id)
|
||
.bind(bot_id)
|
||
.bind(handoff.clone().map(|target| vec![target]))
|
||
.execute(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
if let Some(target) = handoff.as_ref() {
|
||
let owner: Option<(String, String, String)> = sqlx::query_as(
|
||
"SELECT r.space_id, r.user_id, b.name
|
||
FROM runs r JOIN bots b ON b.id=r.bot_id WHERE r.id=$1",
|
||
)
|
||
.bind(run_id)
|
||
.fetch_optional(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
if let Some((space_id, user_id, speaker)) = owner {
|
||
tracing::info!(run_id, target, "room hand-off queued");
|
||
sqlx::query(
|
||
"INSERT INTO runs (id,space_id,bot_id,thread_id,user_id,status,trigger,prompt,checkpoint)
|
||
VALUES ($1,$2,$3,$4,$5,'queued','handoff',$6,$7)",
|
||
)
|
||
.bind(Uuid::new_v4().to_string())
|
||
.bind(space_id)
|
||
.bind(target)
|
||
.bind(thread_id)
|
||
.bind(user_id)
|
||
.bind(format!("{speaker} 在群組裡把這件事交給你:\n{body}"))
|
||
.bind(json!({"handoffFrom": run_id}))
|
||
.execute(&mut *tx)
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
}
|
||
}
|
||
tx.commit().await.map_err(|error| error.to_string())?;
|
||
let _ = crate::sessions::append_event(
|
||
state,
|
||
thread_id,
|
||
"message.created",
|
||
json!({"id":message_id,"seq":seq,"role":"assistant","body":body,"runId":run_id}),
|
||
)
|
||
.await;
|
||
Ok(())
|
||
}
|
||
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||
enum RunHalt {
|
||
Cancelled,
|
||
Takeover,
|
||
}
|
||
|
||
fn halt_from_status(status: Option<&str>) -> Option<RunHalt> {
|
||
match status {
|
||
Some("cancelled") => Some(RunHalt::Cancelled),
|
||
Some("waiting_takeover") => Some(RunHalt::Takeover),
|
||
_ => None,
|
||
}
|
||
}
|
||
|
||
async fn run_status_halt(state: &AppState, run_id: &str) -> Result<Option<RunHalt>, String> {
|
||
let status: Option<String> = sqlx::query_scalar("SELECT status FROM runs WHERE id = $1")
|
||
.bind(run_id)
|
||
.fetch_optional(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
Ok(halt_from_status(status.as_deref()))
|
||
}
|
||
|
||
async fn wait_for_halt(state: &AppState, run_id: &str) -> RunHalt {
|
||
loop {
|
||
if let Ok(Some(halt)) = run_status_halt(state, run_id).await {
|
||
return halt;
|
||
}
|
||
tokio::time::sleep(Duration::from_millis(750)).await;
|
||
}
|
||
}
|
||
|
||
async fn renew_or_halt(
|
||
state: &AppState,
|
||
run_id: &str,
|
||
lease_owner: &str,
|
||
) -> Result<Option<RunHalt>, String> {
|
||
if let Some(halt) = run_status_halt(state, run_id).await? {
|
||
return Ok(Some(halt));
|
||
}
|
||
match renew_lease(state, run_id, lease_owner).await {
|
||
Ok(()) => run_status_halt(state, run_id).await,
|
||
Err(error) => {
|
||
if let Some(halt) = run_status_halt(state, run_id).await? {
|
||
Ok(Some(halt))
|
||
} else {
|
||
Err(error)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
async fn finish_halt(
|
||
state: &AppState,
|
||
thread_id: &str,
|
||
run_id: &str,
|
||
halt: RunHalt,
|
||
turns: u32,
|
||
screenshots: u32,
|
||
screenshot_bytes: u64,
|
||
ctx: &crate::tools::ToolCtx,
|
||
used_gui: bool,
|
||
) -> Result<(), String> {
|
||
match halt {
|
||
RunHalt::Cancelled => Ok(()),
|
||
RunHalt::Takeover => {
|
||
let _ = sqlx::query(
|
||
"UPDATE runs SET lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||
WHERE id=$1 AND status='waiting_takeover'",
|
||
)
|
||
.bind(run_id)
|
||
.execute(state.pool())
|
||
.await;
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"run",
|
||
json!({"event": "paused", "reason": "takeover", "turns": turns}),
|
||
)
|
||
.await;
|
||
let click_misses = *ctx.click_misses.lock().unwrap();
|
||
record_run_metrics(
|
||
state,
|
||
thread_id,
|
||
run_id,
|
||
"run.paused",
|
||
turns,
|
||
screenshots,
|
||
screenshot_bytes,
|
||
click_misses,
|
||
used_gui,
|
||
true,
|
||
)
|
||
.await;
|
||
Ok(())
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Work that stopped before it was verified as finished. Real blockers park in
|
||
/// `waiting_input` with a one-click question. Stalls and spent budgets report
|
||
/// where they stopped and complete, so the human is not asked to babysit.
|
||
struct PauseRequest<'a> {
|
||
reason: StopReason,
|
||
draft: &'a str,
|
||
history: &'a [Message],
|
||
turns: u32,
|
||
steering_seq: i32,
|
||
limit: u32,
|
||
/// Concrete evidence from the round policy, shown instead of a generic
|
||
/// category when the run parked on a loop.
|
||
note: Option<String>,
|
||
screenshots: u32,
|
||
screenshot_bytes: u64,
|
||
used_gui: bool,
|
||
}
|
||
|
||
const PAUSED_PENDING: &str = "This run is paused for the human's answer. When their reply arrives, continue the work that has already started: observe the current screen before any new mutation and never repeat finished steps.";
|
||
|
||
fn should_gate_on_plan(
|
||
_chat_only: bool,
|
||
_resume_after_takeover: bool,
|
||
_await_resume: bool,
|
||
_plan_shown: bool,
|
||
_work_needs_plan: bool,
|
||
) -> bool {
|
||
// A clear task starts immediately; planning is explanatory only.
|
||
false
|
||
}
|
||
|
||
fn pause_for_human(reason: StopReason, draft: &str) -> bool {
|
||
match reason {
|
||
StopReason::NeedsPlan => true,
|
||
StopReason::MidTaskText => asks_for_input(draft),
|
||
StopReason::BudgetExhausted | StopReason::LoopDetected => false,
|
||
}
|
||
}
|
||
|
||
async fn mark_plan_shown(state: &AppState, thread_id: &str) {
|
||
if let Err(error) = sqlx::query(
|
||
"UPDATE threads SET plan_shown=TRUE, updated_at=now() WHERE id=$1 AND plan_shown=FALSE",
|
||
)
|
||
.bind(thread_id)
|
||
.execute(state.pool())
|
||
.await
|
||
{
|
||
tracing::warn!(thread_id, "failed to mark plan shown: {error}");
|
||
}
|
||
}
|
||
|
||
fn stall_report(reason: StopReason, draft: &str, turns: u32, note: Option<&str>) -> String {
|
||
let stall = note.map(str::to_string).unwrap_or_else(|| match reason {
|
||
StopReason::BudgetExhausted => "已達本輪執行上限".to_string(),
|
||
StopReason::MidTaskText => "目前推不下去了".to_string(),
|
||
StopReason::LoopDetected => "重複同一個動作,任務沒有新的進展".to_string(),
|
||
StopReason::NeedsPlan => "等待你確認計畫".to_string(),
|
||
});
|
||
let body = draft.replace(NEEDS_INPUT_MARKER, "").trim().to_string();
|
||
if !body.is_empty() {
|
||
if body.contains(&stall) {
|
||
return body;
|
||
}
|
||
return format!("{body}\n\n{stall}");
|
||
}
|
||
match reason {
|
||
StopReason::BudgetExhausted => {
|
||
format!("我在這個任務上用了 {turns} 輪,還沒做到可以確認完成的地步,先停在這裡。")
|
||
}
|
||
StopReason::LoopDetected => {
|
||
format!("我在同一個動作上打轉,先停在目前進度。{stall}")
|
||
}
|
||
StopReason::MidTaskText => format!("我目前推不下去了。{stall}"),
|
||
StopReason::NeedsPlan => "我先寫一下打算怎麼做。".to_string(),
|
||
}
|
||
}
|
||
|
||
async fn hand_back(
|
||
state: &AppState,
|
||
bot_id: &str,
|
||
thread_id: &str,
|
||
run_id: &str,
|
||
lease_owner: &str,
|
||
ctx: &ToolCtx,
|
||
req: PauseRequest<'_>,
|
||
) -> Result<(), String> {
|
||
if req.reason == StopReason::NeedsPlan {
|
||
mark_plan_shown(state, thread_id).await;
|
||
}
|
||
if pause_for_human(req.reason, req.draft) {
|
||
return pause_for_answer(state, bot_id, thread_id, run_id, lease_owner, ctx, req).await;
|
||
}
|
||
complete_stalled(state, bot_id, thread_id, run_id, lease_owner, ctx, req).await
|
||
}
|
||
|
||
async fn complete_stalled(
|
||
state: &AppState,
|
||
bot_id: &str,
|
||
thread_id: &str,
|
||
run_id: &str,
|
||
lease_owner: &str,
|
||
ctx: &ToolCtx,
|
||
req: PauseRequest<'_>,
|
||
) -> Result<(), String> {
|
||
let spoken = stall_report(req.reason, req.draft, req.turns, req.note.as_deref());
|
||
append_bot_message(state, thread_id, run_id, bot_id, &spoken).await?;
|
||
let completed = sqlx::query(
|
||
"UPDATE runs
|
||
SET status='failed', error=$3, completed_at=now(), updated_at=now(),
|
||
lease_owner=NULL, lease_expires_at=NULL
|
||
WHERE id=$1 AND lease_owner=$2 AND status='running'",
|
||
)
|
||
.bind(run_id)
|
||
.bind(lease_owner)
|
||
.bind(&spoken)
|
||
.execute(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
if completed.rows_affected() != 1 {
|
||
return Err("run lease was lost before completion".into());
|
||
}
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"run",
|
||
json!({
|
||
"event": "failed",
|
||
"reason": spoken,
|
||
"turns": req.turns,
|
||
}),
|
||
)
|
||
.await;
|
||
let click_misses = *ctx.click_misses.lock().unwrap();
|
||
record_run_metrics(
|
||
state,
|
||
thread_id,
|
||
run_id,
|
||
"run.failed",
|
||
req.turns,
|
||
req.screenshots,
|
||
req.screenshot_bytes,
|
||
click_misses,
|
||
req.used_gui,
|
||
false,
|
||
)
|
||
.await;
|
||
computer::release_screen_execution(state, run_id).await?;
|
||
sqlx::query(
|
||
"UPDATE computers SET execution_bot_id = NULL, execution_run_id = NULL, execution_lease_expires_at = NULL, updated_at = now()
|
||
WHERE execution_run_id = $1",
|
||
)
|
||
.bind(run_id)
|
||
.execute(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
tracing::info!(
|
||
run_id,
|
||
reason = req.reason.as_str(),
|
||
turns = req.turns,
|
||
"run failed after recovery was exhausted"
|
||
);
|
||
Ok(())
|
||
}
|
||
|
||
async fn pause_for_answer(
|
||
state: &AppState,
|
||
bot_id: &str,
|
||
thread_id: &str,
|
||
run_id: &str,
|
||
lease_owner: &str,
|
||
ctx: &ToolCtx,
|
||
req: PauseRequest<'_>,
|
||
) -> Result<(), String> {
|
||
let draft = req.draft.replace(NEEDS_INPUT_MARKER, "").trim().to_string();
|
||
// The round policy arrives with concrete evidence ("the same click six
|
||
// times in a row"), which always beats a category name for the human.
|
||
let stall = req.note.clone().unwrap_or_else(|| match req.reason {
|
||
StopReason::BudgetExhausted => "已達本輪執行上限".to_string(),
|
||
StopReason::MidTaskText => {
|
||
"模型多次未能提供可執行的下一步,系統已要求它重新確認並繼續,但仍無法推進".to_string()
|
||
}
|
||
StopReason::LoopDetected => "重複同一個動作,任務沒有新的進展".to_string(),
|
||
StopReason::NeedsPlan => "等待你確認計畫".to_string(),
|
||
});
|
||
let draft = if matches!(req.reason, StopReason::NeedsPlan) {
|
||
if draft.is_empty() {
|
||
"我先寫一下打算怎麼做。要我開始嗎?".to_string()
|
||
} else {
|
||
draft
|
||
}
|
||
} else if draft.is_empty() {
|
||
match req.reason {
|
||
StopReason::BudgetExhausted => format!(
|
||
"我在這個任務上用了 {} 輪,還沒有做到可以幫你確認完成的地步,先停在目前的畫面。要我繼續嗎?\n\n任務尚未確認完成。停止原因:{}。已保存操作進度;回覆下一步指示或接管確認現況後可繼續。",
|
||
req.turns, stall
|
||
),
|
||
StopReason::MidTaskText => format!(
|
||
"模型多次未提供可執行的下一步,系統無法確認任務已完成。請補充下一步指示或接管確認現況後繼續。\n\n任務尚未確認完成。停止原因:{}。已保存操作進度;回覆下一步指示或接管確認現況後可繼續。",
|
||
stall
|
||
),
|
||
StopReason::LoopDetected => format!(
|
||
"我在這個任務上用了 {} 輪,一直在同一個動作上打轉,先停在目前的畫面。要我換個做法繼續嗎?\n\n任務尚未確認完成。停止原因:{}。已保存操作進度;告訴我該怎麼做,或直接接管電腦。",
|
||
req.turns, stall
|
||
),
|
||
StopReason::NeedsPlan => "我先寫一下打算怎麼做。要我開始嗎?".to_string(),
|
||
}
|
||
} else if !asks_for_input(req.draft) {
|
||
// A model's optimistic draft must not hide a harness-detected stall.
|
||
format!(
|
||
"{draft}\n\n任務尚未確認完成。停止原因:{stall}。已保存操作進度;回覆下一步指示或接管確認現況後可繼續。"
|
||
)
|
||
} else {
|
||
draft
|
||
};
|
||
let next = Message::User {
|
||
content: vec![UserContent::text(PAUSED_PENDING)],
|
||
};
|
||
save_harness_checkpoint(
|
||
state,
|
||
run_id,
|
||
lease_owner,
|
||
req.history,
|
||
&next,
|
||
req.turns,
|
||
req.steering_seq,
|
||
)
|
||
.await?;
|
||
let paused = sqlx::query(
|
||
"UPDATE runs
|
||
SET status='waiting_input', lease_owner=NULL, lease_expires_at=NULL, updated_at=now(),
|
||
checkpoint=COALESCE(checkpoint,'{}'::jsonb)||jsonb_build_object('awaitResume',
|
||
jsonb_build_object('reason',$3,'turns',$4,'limit',$5))
|
||
WHERE id=$1 AND lease_owner=$2 AND status='running'",
|
||
)
|
||
.bind(run_id)
|
||
.bind(lease_owner)
|
||
.bind(req.reason.as_str())
|
||
.bind(req.turns as i64)
|
||
.bind(req.limit as i64)
|
||
.execute(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
if paused.rows_affected() != 1 {
|
||
return Err("run lease was lost before pausing".into());
|
||
}
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"run",
|
||
json!({
|
||
"event": "paused",
|
||
"reason": draft,
|
||
"turns": req.turns,
|
||
"limit": req.limit,
|
||
}),
|
||
)
|
||
.await;
|
||
append_bot_message_with(
|
||
state,
|
||
thread_id,
|
||
run_id,
|
||
bot_id,
|
||
&draft,
|
||
json!([{
|
||
"kind": "resume",
|
||
"reason": req.reason.as_str(),
|
||
"turns": req.turns,
|
||
"limit": req.limit,
|
||
}]),
|
||
)
|
||
.await?;
|
||
let click_misses = *ctx.click_misses.lock().unwrap();
|
||
record_run_metrics(
|
||
state,
|
||
thread_id,
|
||
run_id,
|
||
"run.paused",
|
||
req.turns,
|
||
req.screenshots,
|
||
req.screenshot_bytes,
|
||
click_misses,
|
||
req.used_gui,
|
||
false,
|
||
)
|
||
.await;
|
||
computer::release_screen_execution(state, run_id).await?;
|
||
sqlx::query(
|
||
"UPDATE computers SET execution_bot_id = NULL, execution_run_id = NULL, execution_lease_expires_at = NULL, updated_at = now()
|
||
WHERE execution_run_id = $1",
|
||
)
|
||
.bind(run_id)
|
||
.execute(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
tracing::info!(
|
||
run_id,
|
||
reason = req.reason.as_str(),
|
||
turns = req.turns,
|
||
"run paused for an answer"
|
||
);
|
||
Ok(())
|
||
}
|
||
|
||
// This future is scoped to execute_run by select!, so it cannot outlive the
|
||
// worker or renew a completed/cancelled task. Waiting on a provider is alive.
|
||
async fn keep_run_alive(state: &AppState, run_id: &str, lease_owner: &str) -> String {
|
||
loop {
|
||
tokio::time::sleep(Duration::from_secs(10)).await;
|
||
match tokio::time::timeout(Duration::from_secs(10), renew_lease(state, run_id, lease_owner)).await {
|
||
Ok(Ok(())) => {},
|
||
Ok(Err(error)) => return error,
|
||
Err(_) => return "run heartbeat could not reach the database".into(),
|
||
}
|
||
}
|
||
}
|
||
|
||
async fn renew_lease(state: &AppState, run_id: &str, lease_owner: &str) -> Result<(), String> {
|
||
let renewed = sqlx::query(
|
||
"UPDATE runs SET lease_expires_at=now()+interval '5 minutes',updated_at=now(),
|
||
checkpoint=COALESCE(checkpoint,'{}'::jsonb)||jsonb_build_object('heartbeatAt',now())
|
||
WHERE id=$1 AND lease_owner=$2 AND status IN ('leased','running')",
|
||
)
|
||
.bind(run_id)
|
||
.bind(lease_owner)
|
||
.execute(state.pool())
|
||
.await
|
||
.map_err(|error| error.to_string())?;
|
||
if renewed.rows_affected() == 1 {
|
||
Ok(())
|
||
} else {
|
||
Err("run lease was lost".into())
|
||
}
|
||
}
|
||
|
||
fn is_punct(c: char) -> bool {
|
||
matches!(
|
||
c,
|
||
'!' | '?'
|
||
| '.'
|
||
| ','
|
||
| ';'
|
||
| ':'
|
||
| '~'
|
||
| '"'
|
||
| '\''
|
||
| '('
|
||
| ')'
|
||
| '!'
|
||
| '?'
|
||
| '。'
|
||
| ','
|
||
| '、'
|
||
| ';'
|
||
| ':'
|
||
| '~'
|
||
| '…'
|
||
| '・'
|
||
| '·'
|
||
| '「'
|
||
| '」'
|
||
| '『'
|
||
| '』'
|
||
| '('
|
||
| ')'
|
||
| '【'
|
||
| '】'
|
||
| '《'
|
||
| '》'
|
||
)
|
||
}
|
||
|
||
fn normalize_prompt(prompt: &str) -> String {
|
||
let lowered = prompt.trim().to_lowercase();
|
||
let mut out = String::new();
|
||
let mut pending_space = false;
|
||
for c in lowered.chars() {
|
||
if c.is_whitespace() || is_punct(c) {
|
||
if !out.is_empty() {
|
||
pending_space = true;
|
||
}
|
||
continue;
|
||
}
|
||
if pending_space {
|
||
out.push(' ');
|
||
pending_space = false;
|
||
}
|
||
out.push(c);
|
||
}
|
||
out
|
||
}
|
||
|
||
fn first_token(text: &str) -> &str {
|
||
text.split_whitespace().next().unwrap_or("")
|
||
}
|
||
|
||
fn looks_like_url(text: &str) -> bool {
|
||
let lower = text.to_lowercase();
|
||
if lower.contains("://") || lower.contains("www.") {
|
||
return true;
|
||
}
|
||
lower.split_whitespace().any(|token| {
|
||
let host = token.split('/').next().unwrap_or(token);
|
||
let host = host.split('?').next().unwrap_or(host);
|
||
let Some((_, tld)) = host.rsplit_once('.') else {
|
||
return false;
|
||
};
|
||
matches!(
|
||
tld,
|
||
"com"
|
||
| "org"
|
||
| "net"
|
||
| "io"
|
||
| "ai"
|
||
| "app"
|
||
| "dev"
|
||
| "co"
|
||
| "edu"
|
||
| "gov"
|
||
| "tv"
|
||
| "me"
|
||
| "cc"
|
||
| "info"
|
||
| "xyz"
|
||
| "tw"
|
||
| "cn"
|
||
| "hk"
|
||
| "jp"
|
||
) && host
|
||
.chars()
|
||
.all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-')
|
||
})
|
||
}
|
||
|
||
fn is_text_only_request(normalized: &str) -> bool {
|
||
const PHRASES: &[&str] = &[
|
||
"what is ",
|
||
"what are ",
|
||
"who is ",
|
||
"why ",
|
||
"how does ",
|
||
"how do ",
|
||
"tell me about ",
|
||
"explain ",
|
||
"can you explain ",
|
||
"could you explain ",
|
||
"please explain ",
|
||
"是什麼",
|
||
"是什么",
|
||
"什麼是",
|
||
"什么是",
|
||
"為什麼",
|
||
"为什么",
|
||
"請解釋",
|
||
"请解释",
|
||
"幫我解釋",
|
||
"帮我解释",
|
||
"請說明",
|
||
"请说明",
|
||
];
|
||
PHRASES.iter().any(|phrase| normalized.contains(phrase))
|
||
}
|
||
|
||
fn prompt_needs_memory(prompt: &str) -> bool {
|
||
let normalized = normalize_prompt(prompt);
|
||
const PHRASES: &[&str] = &[
|
||
"remember ",
|
||
"remember that",
|
||
"do you remember",
|
||
"forget ",
|
||
"記住",
|
||
"记住",
|
||
"記得",
|
||
"记得",
|
||
"忘記",
|
||
"忘记",
|
||
];
|
||
PHRASES.iter().any(|phrase| normalized.contains(phrase))
|
||
}
|
||
|
||
fn prompt_needs_desktop(prompt: &str) -> bool {
|
||
let trimmed = prompt.trim();
|
||
if trimmed.is_empty() {
|
||
return false;
|
||
}
|
||
if looks_like_url(trimmed) {
|
||
return true;
|
||
}
|
||
let lower = trimmed.to_lowercase();
|
||
let normalized = normalize_prompt(trimmed);
|
||
// Mentioning a site or app in an informational question is not a request
|
||
// to operate it ("YouTube 是什麼?", "how do I open Chrome?").
|
||
if is_text_only_request(&normalized) {
|
||
return false;
|
||
}
|
||
const MARKERS: &[&str] = &[
|
||
"open ",
|
||
"open the",
|
||
"launch",
|
||
"click",
|
||
"browser",
|
||
"chrome",
|
||
"chromium",
|
||
"firefox",
|
||
"desktop",
|
||
"screenshot",
|
||
"terminal",
|
||
"xterm",
|
||
"install ",
|
||
"download",
|
||
"upload",
|
||
"type into",
|
||
"navigate",
|
||
"visit ",
|
||
"go to ",
|
||
"google ",
|
||
"youtube",
|
||
"video",
|
||
"watch ",
|
||
"email",
|
||
"gmail",
|
||
"on the computer",
|
||
"on my computer",
|
||
"use the computer",
|
||
"use the desktop",
|
||
"file manager",
|
||
"run this",
|
||
"run the ",
|
||
"打開",
|
||
"开启",
|
||
"開啟",
|
||
"启动",
|
||
"啟動",
|
||
"點擊",
|
||
"点击",
|
||
"點一下",
|
||
"点一下",
|
||
"瀏覽器",
|
||
"浏览器",
|
||
"桌面",
|
||
"電腦",
|
||
"电脑",
|
||
"螢幕",
|
||
"屏幕",
|
||
"截圖",
|
||
"截图",
|
||
"終端",
|
||
"终端",
|
||
"安裝",
|
||
"安装",
|
||
"下載",
|
||
"下载",
|
||
"上傳",
|
||
"上传",
|
||
"檔案",
|
||
"档案",
|
||
"資料夾",
|
||
"文件夹",
|
||
"網頁",
|
||
"网页",
|
||
"網站",
|
||
"网站",
|
||
"上網",
|
||
"上网",
|
||
"搜尋",
|
||
"搜索",
|
||
"登入",
|
||
"登录",
|
||
"進電腦",
|
||
"进电脑",
|
||
"操作電腦",
|
||
"操作电脑",
|
||
"用電腦",
|
||
"用电脑",
|
||
"看畫面",
|
||
"看画面",
|
||
"看一下影片",
|
||
"看一下視頻",
|
||
"看一下视频",
|
||
"影片",
|
||
"視頻",
|
||
"视频",
|
||
"幫我開",
|
||
"帮我开",
|
||
"幫我點",
|
||
"帮我点",
|
||
"幫我搜",
|
||
"帮我搜",
|
||
"執行指令",
|
||
"执行指令",
|
||
"在電腦",
|
||
"在电脑",
|
||
"到網站",
|
||
"到网站",
|
||
"去網站",
|
||
"去网站",
|
||
"執行「",
|
||
"执行「",
|
||
"執行\"",
|
||
"执行\"",
|
||
"排程",
|
||
"以後每天",
|
||
"以后每天",
|
||
"每個工作日",
|
||
"每个工作日",
|
||
"every day",
|
||
"weekdays",
|
||
"every monday",
|
||
"schedule",
|
||
];
|
||
if MARKERS.iter().any(|marker| lower.contains(marker)) {
|
||
return true;
|
||
}
|
||
if trimmed.contains('?') || trimmed.contains('?') {
|
||
return false;
|
||
}
|
||
const SHELL: &[&str] = &[
|
||
"ls", "pwd", "cd", "cat", "chmod", "chown", "rm", "mv", "cp", "mkdir", "touch", "htop",
|
||
"top", "ps", "df", "du", "whoami", "uname", "curl", "wget", "git", "npm", "npx", "pip",
|
||
"pip3", "python", "python3", "node", "cargo", "make", "docker", "apt", "apt-get", "sudo",
|
||
"bash", "sh", "zsh",
|
||
];
|
||
SHELL.contains(&first_token(&normalized))
|
||
}
|
||
|
||
fn is_follow_up_task(normalized: &str) -> bool {
|
||
const FOLLOW: &[&str] = &[
|
||
"繼續",
|
||
"继续",
|
||
"接著",
|
||
"接着",
|
||
"接著做",
|
||
"接着做",
|
||
"再來",
|
||
"再来",
|
||
"然後",
|
||
"然后",
|
||
"continue",
|
||
"keep going",
|
||
"go on",
|
||
"go ahead",
|
||
"resume",
|
||
"keep at it",
|
||
"從畫面",
|
||
"从画面",
|
||
];
|
||
FOLLOW
|
||
.iter()
|
||
.any(|item| normalized == *item || normalized.starts_with(&format!("{item} ")))
|
||
}
|
||
|
||
fn has_task_verb(normalized: &str) -> bool {
|
||
const VERBS: &[&str] = &[
|
||
"幫我",
|
||
"帮我",
|
||
"幫忙",
|
||
"帮忙",
|
||
"請你",
|
||
"请你",
|
||
"請幫",
|
||
"请帮",
|
||
"please",
|
||
"can you",
|
||
"could you",
|
||
"would you",
|
||
"will you",
|
||
];
|
||
VERBS.iter().any(|verb| normalized.contains(verb))
|
||
}
|
||
|
||
/// Plain doing-words that never show up in a greeting. `prompt_needs_desktop`
|
||
/// cannot name every app, site, or file, so a request is also a task when it
|
||
/// asks for an action to be performed on something.
|
||
fn has_work_verb(normalized: &str) -> bool {
|
||
const VERBS: &[&str] = &[
|
||
"整理",
|
||
"彙整",
|
||
"彙總",
|
||
"存到",
|
||
"存進",
|
||
"存入",
|
||
"存成",
|
||
"儲存",
|
||
"存檔",
|
||
"建立",
|
||
"新增",
|
||
"產生",
|
||
"產出",
|
||
"改名",
|
||
"重命名",
|
||
"移動",
|
||
"複製",
|
||
"刪除",
|
||
"刪掉",
|
||
"翻譯",
|
||
"摘要",
|
||
"總結",
|
||
"歸納",
|
||
"比對",
|
||
"比較",
|
||
"填入",
|
||
"填寫",
|
||
"提交",
|
||
"送出",
|
||
"歸檔",
|
||
"轉檔",
|
||
"轉換",
|
||
"壓縮",
|
||
"解凍",
|
||
"分割",
|
||
"合併",
|
||
"報名",
|
||
"預訂",
|
||
"預約",
|
||
"訂閱",
|
||
"退訂",
|
||
"追蹤",
|
||
"回覆",
|
||
"寄送",
|
||
"領取",
|
||
"打卡",
|
||
"簽到",
|
||
"紀錄",
|
||
"記錄",
|
||
"檢查",
|
||
"測試",
|
||
"執行",
|
||
"下載",
|
||
"上傳",
|
||
"安裝",
|
||
"更新",
|
||
"設定",
|
||
"organize",
|
||
"rename",
|
||
"move ",
|
||
"copy",
|
||
"delete",
|
||
"save",
|
||
"create",
|
||
"generate",
|
||
"download",
|
||
"upload",
|
||
"translate",
|
||
"summarize",
|
||
"submit",
|
||
"book",
|
||
"reserve",
|
||
"archive",
|
||
"convert",
|
||
"compare",
|
||
"send",
|
||
"reply",
|
||
"schedule",
|
||
"install",
|
||
];
|
||
VERBS.iter().any(|verb| normalized.contains(verb))
|
||
}
|
||
|
||
fn is_greeting(normalized: &str) -> bool {
|
||
const EXACT: &[&str] = &[
|
||
"hi",
|
||
"hi hi",
|
||
"hii",
|
||
"hiii",
|
||
"hey",
|
||
"hey there",
|
||
"hey hey",
|
||
"hello",
|
||
"hello there",
|
||
"hello hi",
|
||
"hi hello",
|
||
"yo",
|
||
"sup",
|
||
"howdy",
|
||
"hiya",
|
||
"good morning",
|
||
"good afternoon",
|
||
"good evening",
|
||
"good night",
|
||
"morning",
|
||
"evening",
|
||
"how are you",
|
||
"how are you doing",
|
||
"how's it going",
|
||
"hows it going",
|
||
"whats up",
|
||
"what's up",
|
||
"what up",
|
||
"who are you",
|
||
"what can you do",
|
||
"what do you do",
|
||
"what are you",
|
||
"thanks",
|
||
"thank you",
|
||
"thx",
|
||
"ty",
|
||
"ok",
|
||
"okay",
|
||
"cool",
|
||
"nice",
|
||
"got it",
|
||
"understood",
|
||
"嗨",
|
||
"嗨嗨",
|
||
"你好",
|
||
"您好",
|
||
"哈囉",
|
||
"哈罗",
|
||
"嗨你好",
|
||
"你好啊",
|
||
"你好呀",
|
||
"你好嗨",
|
||
"早安",
|
||
"午安",
|
||
"晚安",
|
||
"在嗎",
|
||
"在嘛",
|
||
"在不在",
|
||
"在吗",
|
||
"你好嗎",
|
||
"你好吗",
|
||
"你是誰",
|
||
"你是谁",
|
||
"你會什麼",
|
||
"你会什么",
|
||
"你能做什麼",
|
||
"你能做什么",
|
||
"你可以做什麼",
|
||
"你可以做什么",
|
||
"謝謝",
|
||
"谢谢",
|
||
"感謝",
|
||
"感谢",
|
||
"好",
|
||
"嗯",
|
||
"喔",
|
||
"哦",
|
||
"哈哈",
|
||
"呵",
|
||
"聊聊",
|
||
"聊天",
|
||
"陪我聊天",
|
||
"說說話",
|
||
"说说话",
|
||
"👋",
|
||
"🙋",
|
||
"😊",
|
||
"🙂",
|
||
"😀",
|
||
];
|
||
if EXACT.contains(&normalized) {
|
||
return true;
|
||
}
|
||
const PREFIXES: &[&str] = &["hi ", "hey ", "hello ", "嗨", "你好"];
|
||
PREFIXES.iter().any(|prefix| {
|
||
normalized.starts_with(prefix)
|
||
&& normalized.chars().count() <= 16
|
||
&& !has_task_verb(normalized)
|
||
})
|
||
}
|
||
|
||
fn is_chat_intent(normalized: &str) -> bool {
|
||
const PHRASES: &[&str] = &[
|
||
"陪我聊天",
|
||
"跟我聊天",
|
||
"來聊天",
|
||
"来聊天",
|
||
"只是聊天",
|
||
"隨便聊聊",
|
||
"随便聊聊",
|
||
"just chatting",
|
||
"just saying hi",
|
||
"let's chat",
|
||
"lets chat",
|
||
"wanna chat",
|
||
"want to chat",
|
||
];
|
||
PHRASES.iter().any(|phrase| normalized.contains(phrase))
|
||
}
|
||
|
||
/// True when this user message should stay in text chat: no desktop tools,
|
||
/// no container boot, no screenshot.
|
||
fn is_plain_chat(prompt: &str) -> bool {
|
||
if prompt_needs_desktop(prompt) || prompt_needs_memory(prompt) {
|
||
return false;
|
||
}
|
||
let normalized = normalize_prompt(prompt);
|
||
if normalized.is_empty() {
|
||
return true;
|
||
}
|
||
if is_text_only_request(&normalized) {
|
||
return true;
|
||
}
|
||
if is_follow_up_task(&normalized) {
|
||
return false;
|
||
}
|
||
if is_greeting(&normalized) {
|
||
return true;
|
||
}
|
||
let chars = normalized.chars().count();
|
||
// Withholding every tool is the most damaging mistake this function can
|
||
// make: the task silently degrades into a paragraph and looks like the run
|
||
// gave up. Only an obvious pleasantry counts as chat; anything with a work
|
||
// verb keeps its tools.
|
||
if chars <= 12 && !has_task_verb(&normalized) && !has_work_verb(&normalized) {
|
||
return true;
|
||
}
|
||
is_chat_intent(&normalized) && chars <= 24 && !has_work_verb(&normalized)
|
||
}
|
||
|
||
fn skips_plan(normalized: &str) -> bool {
|
||
const SKIP: &[&str] = &[
|
||
"直接做",
|
||
"不用問",
|
||
"不用等我",
|
||
"開始做",
|
||
"开始做",
|
||
"just do it",
|
||
"don't ask",
|
||
"dont ask",
|
||
"no need to ask",
|
||
"go ahead",
|
||
];
|
||
if is_follow_up_task(normalized) {
|
||
return true;
|
||
}
|
||
SKIP.iter().any(|marker| normalized.contains(marker))
|
||
}
|
||
|
||
fn is_simple_act(normalized: &str) -> bool {
|
||
if normalized.chars().count() > 28 {
|
||
return false;
|
||
}
|
||
const CHAIN: &[&str] = &[
|
||
" then ", " and ", "save", "export", "整理", "存成", "存為", "存为", "然後", "然后",
|
||
"並且", "并且", "接著", "接着",
|
||
];
|
||
if CHAIN.iter().any(|marker| normalized.contains(marker)) {
|
||
return false;
|
||
}
|
||
const SIMPLE: &[&str] = &[
|
||
"打開",
|
||
"開啟",
|
||
"开启",
|
||
"幫我開",
|
||
"帮我开",
|
||
"幫我搜",
|
||
"帮我搜",
|
||
"搜一下",
|
||
"點一下",
|
||
"点一下",
|
||
"看一下",
|
||
"open ",
|
||
"search ",
|
||
"click ",
|
||
"go to ",
|
||
"visit ",
|
||
];
|
||
SIMPLE.iter().any(|marker| normalized.contains(marker))
|
||
}
|
||
|
||
/// Work that should show a plan before booting the computer. Short open/search
|
||
/// stays immediate; anything that will take several tool turns asks first.
|
||
fn needs_plan_first(prompt: &str) -> bool {
|
||
if is_plain_chat(prompt) {
|
||
return false;
|
||
}
|
||
let normalized = normalize_prompt(prompt);
|
||
if skips_plan(&normalized) || is_text_only_request(&normalized) {
|
||
return false;
|
||
}
|
||
if is_simple_act(&normalized) {
|
||
return false;
|
||
}
|
||
prompt_needs_desktop(prompt) || has_work_verb(&normalized) || has_task_verb(&normalized)
|
||
}
|
||
|
||
fn tool_needs_sandbox(name: &str) -> bool {
|
||
name.starts_with("mcp_")
|
||
|| matches!(
|
||
name,
|
||
"shell"
|
||
| "exec"
|
||
| "terminal_start"
|
||
| "terminal_interact"
|
||
| "list_files"
|
||
| "read_file_bytes"
|
||
| "search_files"
|
||
| "move_file"
|
||
| "patch_file"
|
||
| "stat_file"
|
||
| "file_operation_status"
|
||
| "read_file"
|
||
| "write_file"
|
||
| "computer_observe"
|
||
| "computer_act"
|
||
| "browser"
|
||
| "connection_check"
|
||
| "open_path"
|
||
| "launch_app"
|
||
| "wait"
|
||
| "use_saved_login"
|
||
| "request_takeover"
|
||
| "form_fill"
|
||
| "computer_mcp"
|
||
)
|
||
}
|
||
|
||
fn tool_needs_gui(name: &str) -> bool {
|
||
matches!(
|
||
name,
|
||
"shell"
|
||
| "computer_observe"
|
||
| "computer_act"
|
||
| "browser"
|
||
| "connection_check"
|
||
| "open_path"
|
||
| "launch_app"
|
||
| "wait"
|
||
| "use_saved_login"
|
||
| "request_takeover"
|
||
| "form_fill"
|
||
)
|
||
}
|
||
|
||
async fn prepare_run_computer(
|
||
state: &AppState,
|
||
actor: &Actor,
|
||
bot_id: &str,
|
||
run_id: &str,
|
||
ctx: &ToolCtx,
|
||
need_gui: bool,
|
||
) -> Result<(), String> {
|
||
if ctx.computer.lock().unwrap().is_none() {
|
||
let bot = state
|
||
.db
|
||
.get_bot(actor, bot_id)
|
||
.await
|
||
.map_err(|error| error.to_string())?
|
||
.ok_or_else(|| "bot not found".to_string())?;
|
||
let computer = state
|
||
.db
|
||
.get_computer(bot.computer_id.as_deref().unwrap_or(""))
|
||
.await
|
||
.map_err(|error| error.to_string())?
|
||
.ok_or_else(|| "computer not found".to_string())?;
|
||
if computer.state != "running" {
|
||
let step = if computer.state == "suspended" {
|
||
computer::STEP_WAKING
|
||
} else {
|
||
computer::STEP_BOOTING
|
||
};
|
||
set_run_step(state, run_id, step).await;
|
||
}
|
||
computer::boot_for(state, actor, bot_id, need_gui).await?;
|
||
let computer = state
|
||
.db
|
||
.get_computer(bot.computer_id.as_deref().unwrap_or(""))
|
||
.await
|
||
.map_err(|error| error.to_string())?
|
||
.ok_or_else(|| "computer not found".to_string())?;
|
||
let computer_ref = computer::computer_ref(&computer)
|
||
.ok_or_else(|| "computer is not running".to_string())?;
|
||
*ctx.computer.lock().unwrap() = Some(computer_ref);
|
||
ctx.context.lock().unwrap().computer_generation = Some(computer.generation);
|
||
}
|
||
if !need_gui || ctx.adapter().display.is_some() {
|
||
return Ok(());
|
||
}
|
||
let bot = state
|
||
.db
|
||
.get_bot(actor, bot_id)
|
||
.await
|
||
.map_err(|error| error.to_string())?
|
||
.ok_or_else(|| "bot not found".to_string())?;
|
||
let computer = state
|
||
.db
|
||
.get_computer(bot.computer_id.as_deref().unwrap_or(""))
|
||
.await
|
||
.map_err(|error| error.to_string())?
|
||
.ok_or_else(|| "computer not found".to_string())?;
|
||
let bound = computer::ensure_bot_screen(state, actor, bot_id, &computer, Some(run_id)).await?;
|
||
let mut gui_block = bound.gui_block;
|
||
let screen = if let Some(row) = bound.row {
|
||
let held_by_other = row
|
||
.execution_run_id
|
||
.as_deref()
|
||
.is_some_and(|held| held != run_id);
|
||
let user_holding = row.control_holder == "user";
|
||
if held_by_other || user_holding {
|
||
set_run_step(state, run_id, computer::STEP_HANDOFF).await;
|
||
}
|
||
let row = computer::take_screen_execution(state, &row, run_id).await?;
|
||
if gui_block.is_none() {
|
||
gui_block =
|
||
computer::take_profile_lock(state, &computer, bot_id, &bot.name, run_id, &row)
|
||
.await?;
|
||
}
|
||
Some(row)
|
||
} else {
|
||
None
|
||
};
|
||
*ctx.gui_block.lock().unwrap() = gui_block;
|
||
*ctx.context.lock().unwrap() =
|
||
adapter_context_for(actor, bot_id, "run", screen.as_ref(), Some(run_id));
|
||
ctx.context.lock().unwrap().computer_generation = Some(computer.generation);
|
||
Ok(())
|
||
}
|
||
|
||
fn history_window_start(summary_seq: i32, current_seq: i32) -> i32 {
|
||
summary_seq.min(current_seq.saturating_sub(1)).max(0)
|
||
}
|
||
|
||
/// Recent chat turns sent to the model. The transcript itself is unchanged.
|
||
fn thread_history_limit(_has_summary: bool) -> i64 {
|
||
32
|
||
}
|
||
|
||
/// Durable assistant text can include on-screen thinking. The model only needs
|
||
/// the closing summary; the full body stays on the message row.
|
||
fn model_facing_reply(body: &str) -> &str {
|
||
let body = body.trim();
|
||
if body.chars().count() <= 480 {
|
||
return body;
|
||
}
|
||
body.rsplit("\n\n")
|
||
.find(|part| !part.trim().is_empty())
|
||
.map(str::trim)
|
||
.filter(|tail| !tail.is_empty())
|
||
.unwrap_or(body)
|
||
}
|
||
|
||
const MODEL_STEP: &str = "思考中";
|
||
|
||
fn progress_line(name: &str, args: &Value) -> String {
|
||
fn short(value: Option<&str>, max: usize) -> String {
|
||
let text = value.unwrap_or("").replace('\n', " ");
|
||
if text.chars().count() > max {
|
||
format!("{}…", text.chars().take(max).collect::<String>())
|
||
} else {
|
||
text
|
||
}
|
||
}
|
||
let get = |key: &str| args.get(key).and_then(Value::as_str);
|
||
match name {
|
||
"computer_observe" => "正在看畫面".into(),
|
||
"computer_act" => "正在操作桌面".into(),
|
||
"connection_check" => "正在確認連線".into(),
|
||
"browser" => match get("action").unwrap_or("snapshot") {
|
||
"snapshot" | "observe" => "正在看網頁".into(),
|
||
"navigate" | "goto" | "open" => {
|
||
let url = short(get("url").or(get("text")), 40);
|
||
if url.is_empty() {
|
||
"正在打開網頁".into()
|
||
} else {
|
||
format!("正在打開 {url}")
|
||
}
|
||
}
|
||
"click" => "正在點頁面上的按鈕".into(),
|
||
"type" | "fill" => "正在輸入文字".into(),
|
||
_ => "正在操作網頁".into(),
|
||
},
|
||
"write_file" | "patch_file" => {
|
||
let path = short(get("path"), 40);
|
||
if path.is_empty() {
|
||
"正在寫檔".into()
|
||
} else {
|
||
format!("正在寫入 {path}")
|
||
}
|
||
}
|
||
"read_file" | "list_files" => "正在讀檔".into(),
|
||
"exec" => "正在執行指令".into(),
|
||
"wait" => "正在等畫面就緒".into(),
|
||
"open_path" | "launch_app" => "正在打開應用程式".into(),
|
||
"use_skill" => "正在讀取技能".into(),
|
||
"form_fill" => "正在填表".into(),
|
||
_ => "正在處理".into(),
|
||
}
|
||
}
|
||
|
||
/// Human-readable label for what the run is doing right now. Surfaced through
|
||
/// `computer.status` so the chat can show "working: browser click" instead of a
|
||
/// bare spinner while a tool runs.
|
||
fn describe_step(name: &str, args: &Value) -> String {
|
||
fn short(value: Option<&str>, max: usize) -> String {
|
||
let text = value.unwrap_or("").replace('\n', " ");
|
||
if text.chars().count() > max {
|
||
format!("{}…", text.chars().take(max).collect::<String>())
|
||
} else {
|
||
text
|
||
}
|
||
}
|
||
let get = |key: &str| args.get(key).and_then(Value::as_str);
|
||
let detail = match name {
|
||
"computer_observe" => "看畫面".to_string(),
|
||
"connection_check" => "嘗試連線驗證並確認結果".to_string(),
|
||
"computer_act" => args
|
||
.get("actions")
|
||
.and_then(Value::as_array)
|
||
.map(|actions| {
|
||
actions
|
||
.iter()
|
||
.take(4)
|
||
.map(|action| {
|
||
let field = |key: &str| action.get(key).and_then(Value::as_str);
|
||
let kind = field("kind").or(field("type")).unwrap_or("?");
|
||
let target = if let Some(text) = field("text").or(field("keys")) {
|
||
short(Some(text), 24)
|
||
} else if let Some(id) = lazyboy_control::element_id(action.get("element"))
|
||
{
|
||
format!("#{id}")
|
||
} else if let (Some(x), Some(y)) = (
|
||
action.get("x").and_then(Value::as_i64),
|
||
action.get("y").and_then(Value::as_i64),
|
||
) {
|
||
format!("({x},{y})")
|
||
} else {
|
||
String::new()
|
||
};
|
||
format!("{kind} {target}").trim().to_string()
|
||
})
|
||
.collect::<Vec<_>>()
|
||
.join(", ")
|
||
})
|
||
.unwrap_or_default(),
|
||
"browser" => {
|
||
let target = get("url")
|
||
.or(get("text"))
|
||
.or(get("selector"))
|
||
.map(|s| short(Some(s), 40));
|
||
let target = target.or_else(|| {
|
||
lazyboy_control::element_id(args.get("element")).map(|id| format!("#{id}"))
|
||
});
|
||
format!(
|
||
"{} {}",
|
||
get("action").unwrap_or("snapshot"),
|
||
target.unwrap_or_default()
|
||
)
|
||
.trim()
|
||
.to_string()
|
||
}
|
||
"exec" => {
|
||
if let Some(argv) = args.get("argv").and_then(Value::as_array) {
|
||
let line = argv
|
||
.iter()
|
||
.filter_map(Value::as_str)
|
||
.take(6)
|
||
.collect::<Vec<_>>()
|
||
.join(" ");
|
||
short(Some(&line), 60)
|
||
} else {
|
||
short(get("command"), 60)
|
||
}
|
||
}
|
||
"list_files" | "read_file_bytes" | "search_files" | "move_file" | "patch_file"
|
||
| "stat_file" | "read_file" | "write_file" | "list_dir" => short(get("path"), 40),
|
||
"shell" => {
|
||
// The terminal does four different things; the feed says which.
|
||
let session = get("session").filter(|name| !name.trim().is_empty() && *name != "main");
|
||
let suffix = session.map(|name| format!(" ·{name}")).unwrap_or_default();
|
||
if args.get("reset").and_then(Value::as_bool).unwrap_or(false) {
|
||
format!("重開終端機{suffix}")
|
||
} else if let Some(keys) = get("keys") {
|
||
format!("輸入 {}{suffix}", short(Some(keys), 20))
|
||
} else {
|
||
match get("command")
|
||
.or(get("cmd"))
|
||
.filter(|line| !line.trim().is_empty())
|
||
{
|
||
Some(command) => format!("{}{suffix}", short(Some(command), 60)),
|
||
None => format!("讀終端機{suffix}"),
|
||
}
|
||
}
|
||
}
|
||
"wait" => format!(
|
||
"{}s {}",
|
||
args.get("seconds")
|
||
.and_then(Value::as_f64)
|
||
.unwrap_or(0.0)
|
||
.round(),
|
||
short(get("reason"), 30)
|
||
)
|
||
.trim()
|
||
.to_string(),
|
||
"launch_app" | "open_path" => short(get("app").or(get("path")), 40),
|
||
"use_skill" => format!("讀取技能 {}", short(get("name"), 30)),
|
||
"use_saved_login" => "填入已存帳號".into(),
|
||
"list_accounts" => "列出已存帳號".into(),
|
||
"create_schedule" => format!("排程 {}", short(get("name"), 30)),
|
||
"list_schedules" => "列出排程".into(),
|
||
"cancel_schedule" => "取消排程".into(),
|
||
"request_takeover" => short(get("site").or(get("reason")), 40),
|
||
"form_fill" => format!(
|
||
"{} fields",
|
||
args.get("fields")
|
||
.and_then(Value::as_array)
|
||
.map(|fields| fields.len())
|
||
.unwrap_or(0)
|
||
),
|
||
"computer_mcp" => short(get("packageId").or(get("text")), 40),
|
||
_ => String::new(),
|
||
};
|
||
if detail.is_empty() {
|
||
name.to_string()
|
||
} else {
|
||
format!("{name}: {detail}")
|
||
}
|
||
}
|
||
|
||
/// The error the claim sweep writes when a worker died holding a tool open.
|
||
const INTERRUPTED_ERROR: &str =
|
||
"Worker interrupted after tool execution; inspect current state before continuing.";
|
||
|
||
/// Step plus turn counter: what the bubble header reads out of the checkpoint.
|
||
pub(crate) async fn set_run_progress(
|
||
state: &AppState,
|
||
run_id: &str,
|
||
step: &str,
|
||
turn: u32,
|
||
limit: Option<i64>,
|
||
) {
|
||
let result = sqlx::query(
|
||
"UPDATE runs SET checkpoint = COALESCE(checkpoint, '{}'::jsonb)
|
||
|| jsonb_build_object('step', $2::text, 'stepAt', now(),
|
||
'turn', $3::bigint, 'turnLimit', $4::bigint),
|
||
updated_at = now()
|
||
WHERE id = $1",
|
||
)
|
||
.bind(run_id)
|
||
.bind(step)
|
||
.bind(turn as i64)
|
||
.bind(limit)
|
||
.execute(state.pool())
|
||
.await;
|
||
if let Err(error) = result {
|
||
tracing::warn!(run_id, "failed to record run progress: {error}");
|
||
}
|
||
}
|
||
|
||
/// Tell the human a run died, in their language, with the one next action. The
|
||
/// raw error stays available through the run activity endpoint.
|
||
async fn report_run_failure(
|
||
state: &AppState,
|
||
run_id: &str,
|
||
thread_id: &str,
|
||
bot_id: &str,
|
||
error: &str,
|
||
) {
|
||
let progress: Option<(Option<String>, Option<i64>)> = sqlx::query_as(
|
||
"SELECT checkpoint->>'step', (checkpoint->>'turn')::bigint FROM runs WHERE id=$1",
|
||
)
|
||
.bind(run_id)
|
||
.fetch_optional(state.pool())
|
||
.await
|
||
.unwrap_or(None);
|
||
let (last_step, turn) = progress.unwrap_or((None, None));
|
||
let failure = crate::monitor::classify_run_error(error);
|
||
let body = crate::monitor::failure_message(&failure, last_step.as_deref(), turn);
|
||
let _ = append_bot_message_with(
|
||
state,
|
||
thread_id,
|
||
run_id,
|
||
bot_id,
|
||
&body,
|
||
json!([{
|
||
"kind": "error",
|
||
"code": failure.code,
|
||
"retryable": failure.retryable,
|
||
"runId": run_id,
|
||
"turn": turn,
|
||
"step": last_step,
|
||
}]),
|
||
)
|
||
.await;
|
||
crate::monitor::record(
|
||
state,
|
||
run_id,
|
||
"run",
|
||
json!({"event": "failed", "error": error}),
|
||
)
|
||
.await;
|
||
}
|
||
|
||
/// Display-only reading of a tool result. Tools report failure in prose, so the
|
||
/// trail marks the shapes it recognises rather than pretending to know more.
|
||
fn tool_status(timed_out: bool, pause: bool, text: &str) -> &'static str {
|
||
if timed_out {
|
||
return "timed_out";
|
||
}
|
||
if pause {
|
||
return "paused";
|
||
}
|
||
const FAILURES: [&str; 14] = [
|
||
"error",
|
||
"failed",
|
||
"failure",
|
||
"unable to",
|
||
"cannot ",
|
||
"can't ",
|
||
"timed out",
|
||
"denied",
|
||
"no such",
|
||
"exception",
|
||
"not available",
|
||
"失敗",
|
||
"錯誤",
|
||
"無法",
|
||
];
|
||
let head: String = text.to_lowercase().chars().take(160).collect();
|
||
if FAILURES.iter().any(|needle| head.contains(needle)) {
|
||
"error"
|
||
} else {
|
||
"ok"
|
||
}
|
||
}
|
||
|
||
/// Does this tool call change the world? The round policy only counts
|
||
/// mutating actions, so polling the screen, waiting, or reading a file can
|
||
/// never look like a derailment on its own.
|
||
fn action_changes_state(name: &str, args: &Value) -> bool {
|
||
match name {
|
||
"computer_act" | "shell" | "write_file" | "launch_app" | "open_path"
|
||
| "create_schedule" | "cancel_schedule" | "remember" | "forget_memory"
|
||
| "use_saved_login" | "request_takeover" | "form_fill" | "computer_mcp"
|
||
| "terminal_start" | "terminal_interact" => true,
|
||
"exec" => !matches!(
|
||
args.get("action").and_then(Value::as_str),
|
||
Some("status" | "output")
|
||
),
|
||
"browser" => matches!(
|
||
args.get("action").and_then(Value::as_str),
|
||
Some("click") | Some("type") | Some("navigate") | Some("press")
|
||
),
|
||
// Observations, waits, reads, and connected-service calls (mcp_*) are
|
||
// allowed to repeat: they are how a run learns what changed.
|
||
_ => false,
|
||
}
|
||
}
|
||
|
||
pub(crate) async fn set_run_step(state: &AppState, run_id: &str, step: &str) {
|
||
let result = sqlx::query(
|
||
"UPDATE runs SET checkpoint = COALESCE(checkpoint, '{}'::jsonb)
|
||
|| jsonb_build_object('step', $2::text, 'stepAt', now()),
|
||
updated_at = now()
|
||
WHERE id = $1",
|
||
)
|
||
.bind(run_id)
|
||
.bind(step)
|
||
.execute(state.pool())
|
||
.await;
|
||
if let Err(error) = result {
|
||
tracing::warn!(run_id, "failed to record run step: {error}");
|
||
}
|
||
}
|
||
|
||
async fn record_run_metrics(
|
||
state: &AppState,
|
||
thread_id: &str,
|
||
run_id: &str,
|
||
event: &str,
|
||
turns: u32,
|
||
screenshots: u32,
|
||
screenshot_bytes: u64,
|
||
click_misses: u32,
|
||
used_gui: bool,
|
||
takeover: bool,
|
||
) {
|
||
let payload = json!({
|
||
"runId": run_id,
|
||
"turns": turns,
|
||
"screenshotsToModel": screenshots,
|
||
"screenshotBytes": screenshot_bytes,
|
||
"clickMisses": click_misses,
|
||
"usedGui": used_gui,
|
||
"takeover": takeover,
|
||
});
|
||
tracing::info!(
|
||
run_id,
|
||
turns,
|
||
screenshots,
|
||
screenshot_bytes,
|
||
click_misses,
|
||
used_gui,
|
||
takeover,
|
||
"run metrics"
|
||
);
|
||
let _ = crate::sessions::append_event(state, thread_id, event, payload).await;
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::{
|
||
RunHalt, SCREENSHOT_CAPTION, TAKEOVER_RESUME_PROMPT, describe_step,
|
||
drop_history_screenshots, halt_from_status, history_window_start, is_plain_chat,
|
||
keep_thought, merge_spoken, model_facing_reply, needs_plan_first, pause_for_human,
|
||
retryable_run_error, screenshot_parts, should_gate_on_plan, stall_report,
|
||
thread_history_limit, tool_needs_gui, tool_needs_sandbox, tool_status,
|
||
};
|
||
use rig_core::completion::message::{Message, UserContent};
|
||
use serde_json::json;
|
||
|
||
#[tokio::test]
|
||
async fn reply_stream_publishes_before_model_finishes() {
|
||
use futures_util::StreamExt;
|
||
use rig_core::streaming::{RawStreamingChoice, StreamFinal, StreamingCompletionResponse};
|
||
let source =
|
||
futures_util::stream::iter(vec![Ok(RawStreamingChoice::Message("visible now".into()))])
|
||
.chain(futures_util::stream::once(async {
|
||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||
Ok(RawStreamingChoice::FinalResponse(StreamFinal::new(
|
||
"fixture",
|
||
Default::default(),
|
||
)))
|
||
}));
|
||
let response = StreamingCompletionResponse::stream("fixture", Box::pin(source));
|
||
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
|
||
let task = tokio::spawn(super::collect_reply_stream(response, move |text| {
|
||
tx.send(text).unwrap();
|
||
}));
|
||
let first = tokio::time::timeout(std::time::Duration::from_millis(500), rx.recv())
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(first.as_deref(), Some("visible now"));
|
||
assert!(
|
||
!task.is_finished(),
|
||
"text must arrive before the terminal response"
|
||
);
|
||
task.await.unwrap().unwrap();
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn reply_stream_requires_terminal_and_preserves_text() {
|
||
use rig_core::streaming::{RawStreamingChoice, StreamFinal, StreamingCompletionResponse};
|
||
let stream = StreamingCompletionResponse::stream(
|
||
"fixture",
|
||
Box::pin(futures_util::stream::iter(vec![
|
||
Ok(RawStreamingChoice::Message("你".into())),
|
||
Ok(RawStreamingChoice::Message("好".into())),
|
||
Ok(RawStreamingChoice::FinalResponse(StreamFinal::new(
|
||
"fixture",
|
||
Default::default(),
|
||
))),
|
||
])),
|
||
);
|
||
let mut published = String::new();
|
||
let content = super::collect_reply_stream(stream, |text| {
|
||
published.push_str(&text);
|
||
})
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(published, "你好");
|
||
assert!(
|
||
matches!(&content[0], rig_core::message::AssistantContent::Text(text) if text.text == "你好")
|
||
);
|
||
let truncated = StreamingCompletionResponse::stream(
|
||
"fixture",
|
||
Box::pin(futures_util::stream::iter(vec![Ok(
|
||
RawStreamingChoice::Message("incomplete".into()),
|
||
)])),
|
||
);
|
||
assert!(
|
||
super::collect_reply_stream(truncated, |_| {})
|
||
.await
|
||
.unwrap_err()
|
||
.contains("before its final response")
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn spoken_thoughts_stay_in_front_of_the_summary() {
|
||
assert_eq!(
|
||
merge_spoken(
|
||
&["先打開 Threads 搜尋頁。".into()],
|
||
"摘要已寫進 notes.md。做好了。"
|
||
),
|
||
"先打開 Threads 搜尋頁。\n\n摘要已寫進 notes.md。做好了。"
|
||
);
|
||
assert_eq!(
|
||
merge_spoken(
|
||
&["摘要已寫進 notes.md。做好了。".into()],
|
||
"摘要已寫進 notes.md。做好了。"
|
||
),
|
||
"摘要已寫進 notes.md。做好了。"
|
||
);
|
||
assert_eq!(keep_thought("好了"), None);
|
||
assert_eq!(keep_thought("[GOAL_COMPLETE]"), None);
|
||
assert_eq!(
|
||
keep_thought("先搜尋 Threads\n[GOAL_COMPLETE]"),
|
||
Some("先搜尋 Threads".into())
|
||
);
|
||
let stored = merge_spoken(
|
||
&["我先看畫面並搜尋 Threads。".repeat(40)],
|
||
"摘要已寫進 notes.md。做好了。",
|
||
);
|
||
assert!(stored.contains("我先看畫面並搜尋 Threads。"));
|
||
assert_eq!(model_facing_reply(&stored), "摘要已寫進 notes.md。做好了。");
|
||
assert_eq!(model_facing_reply("短回覆就好"), "短回覆就好");
|
||
assert_eq!(thread_history_limit(true), 32);
|
||
assert_eq!(thread_history_limit(false), 32);
|
||
}
|
||
|
||
#[test]
|
||
fn takeover_resume_invalidates_pre_handoff_refs() {
|
||
assert!(
|
||
TAKEOVER_RESUME_PROMPT
|
||
.contains("Element ids and page refs from before the handoff are invalid")
|
||
);
|
||
assert!(TAKEOVER_RESUME_PROMPT.contains("fresh observation"));
|
||
assert!(tool_needs_gui("browser"));
|
||
assert!(tool_needs_gui("computer_observe"));
|
||
}
|
||
|
||
#[test]
|
||
fn the_trail_reads_a_tool_result_as_ok_error_timeout_or_pause() {
|
||
assert_eq!(tool_status(false, false, "clicked element #12"), "ok");
|
||
assert_eq!(
|
||
tool_status(false, false, "Error: selector not found"),
|
||
"error"
|
||
);
|
||
assert_eq!(tool_status(false, false, "操作失敗:視窗關閉"), "error");
|
||
assert_eq!(tool_status(true, false, "tool timed out"), "timed_out");
|
||
assert_eq!(tool_status(false, true, "需要人先登入"), "paused");
|
||
}
|
||
|
||
#[test]
|
||
fn chat_tools_do_not_need_the_desktop() {
|
||
assert!(!tool_needs_sandbox("remember"));
|
||
assert!(!tool_needs_sandbox("recall_memory"));
|
||
assert!(!tool_needs_sandbox("use_skill"));
|
||
assert!(tool_needs_sandbox("shell"));
|
||
assert!(tool_needs_sandbox("exec"));
|
||
assert!(tool_needs_sandbox("list_files"));
|
||
assert!(tool_needs_sandbox("read_file_bytes"));
|
||
assert!(!tool_needs_gui("read_file_bytes"));
|
||
assert!(tool_needs_sandbox("search_files"));
|
||
assert!(!tool_needs_gui("search_files"));
|
||
assert!(tool_needs_sandbox("move_file"));
|
||
assert!(!tool_needs_gui("move_file"));
|
||
assert!(tool_needs_sandbox("patch_file"));
|
||
assert!(!tool_needs_gui("patch_file"));
|
||
assert!(tool_needs_sandbox("file_operation_status"));
|
||
assert!(!tool_needs_gui("file_operation_status"));
|
||
assert!(!crate::operations::mutating_tool(
|
||
"file_operation_status",
|
||
&serde_json::json!({"targetOperationId":"op","readback":true})
|
||
));
|
||
assert!(tool_needs_sandbox("stat_file"));
|
||
assert!(!tool_needs_gui("stat_file"));
|
||
assert!(tool_needs_gui("computer_observe"));
|
||
assert!(tool_needs_gui("browser"));
|
||
assert!(tool_needs_gui("shell"));
|
||
assert!(!tool_needs_gui("list_files"));
|
||
assert!(!tool_needs_gui("read_file"));
|
||
assert!(!tool_needs_gui("write_file"));
|
||
assert!(!tool_needs_gui("exec"));
|
||
assert!(tool_needs_sandbox("form_fill"));
|
||
assert!(tool_needs_gui("form_fill"));
|
||
assert!(tool_needs_sandbox("computer_mcp"));
|
||
assert!(!tool_needs_gui("computer_mcp"));
|
||
}
|
||
|
||
#[test]
|
||
fn native_tools_do_not_boot_the_desktop() {
|
||
for name in [
|
||
"exec",
|
||
"list_files",
|
||
"read_file",
|
||
"write_file",
|
||
"computer_mcp",
|
||
] {
|
||
assert!(tool_needs_sandbox(name), "{name}");
|
||
assert!(!tool_needs_gui(name), "{name}");
|
||
}
|
||
assert!(!lazyboy_control::viewer_blocks_native());
|
||
assert!(!lazyboy_control::desktop_required_for_native());
|
||
assert!(lazyboy_control::native_work_allowed(
|
||
lazyboy_control::ComponentReadiness {
|
||
runner: true,
|
||
..lazyboy_control::ComponentReadiness::default()
|
||
}
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn greetings_stay_in_chat() {
|
||
for prompt in [
|
||
"hi",
|
||
"Hi!",
|
||
"hello",
|
||
"hey there",
|
||
"你好",
|
||
"嗨",
|
||
"哈囉!",
|
||
"在嗎",
|
||
"你是誰",
|
||
"thanks",
|
||
"謝謝",
|
||
"聊聊",
|
||
"👋",
|
||
"今天心情不好",
|
||
"寫一首詩",
|
||
"量子力學是什麼",
|
||
"hello, can you explain Rust ownership?",
|
||
"YouTube 是什麼?",
|
||
"how do I open Chrome?",
|
||
"請解釋量子力學",
|
||
] {
|
||
assert!(is_plain_chat(prompt), "{prompt} should stay in chat");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn computer_tasks_are_not_plain_chat() {
|
||
for prompt in [
|
||
"打開 youtube",
|
||
"open chrome",
|
||
"幫我開 gmail",
|
||
"go to https://example.com",
|
||
"ls",
|
||
"git status",
|
||
"繼續",
|
||
"接著做",
|
||
"執行「完成 STAR 訓練」",
|
||
"看畫面現在怎樣",
|
||
"download this file",
|
||
"幫我點 Next",
|
||
"幫我看一下 YouTube 上的 Rust 教學",
|
||
"幫我看一下這個影片",
|
||
"watch this YouTube video",
|
||
"hi check my email",
|
||
"remember that I prefer dark mode",
|
||
"記住我喜歡繁體中文",
|
||
] {
|
||
assert!(!is_plain_chat(prompt), "{prompt} should keep desktop tools");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn a_work_verb_outranks_a_short_prompt() {
|
||
// The old length rule silently stripped every tool from a short
|
||
// imperative; that is the "it stopped halfway" bug in its purest form.
|
||
for prompt in [
|
||
"把這張圖壓縮到 800px",
|
||
"把報價整理成表格",
|
||
"翻譯這段",
|
||
"幫我把檔案改名",
|
||
"幫我把这份報告存成 pdf",
|
||
"rename the screenshots",
|
||
"submit the form",
|
||
"然後把結果存成 md",
|
||
] {
|
||
assert!(!is_plain_chat(prompt), "{prompt} must keep its tools");
|
||
}
|
||
for prompt in ["你好", "YouTube 是什麼?", "今天天氣如何", "寫一首詩"] {
|
||
assert!(is_plain_chat(prompt), "{prompt} should stay in chat");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn only_multi_step_work_asks_for_a_plan() {
|
||
for prompt in [
|
||
"嗨",
|
||
"為什麼孕婦會頭痛",
|
||
"为什么孕妇会头痛",
|
||
"打開 youtube",
|
||
"幫我搜 Rust",
|
||
"go to https://example.com",
|
||
"繼續",
|
||
"直接做",
|
||
"不用問,開始做",
|
||
] {
|
||
assert!(
|
||
!needs_plan_first(prompt),
|
||
"{prompt} should not pause for a plan"
|
||
);
|
||
}
|
||
for prompt in [
|
||
"把 Threads 搜尋結果存成 md",
|
||
"幫我把这份報告存成 pdf",
|
||
"把報價整理成表格",
|
||
"先登入再把表單填完存檔",
|
||
"search then save as markdown",
|
||
"幫我到 Threads 找資料",
|
||
] {
|
||
assert!(
|
||
needs_plan_first(prompt),
|
||
"{prompt} should show a plan first"
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn clear_work_starts_without_a_confirmation_turn() {
|
||
assert!(!should_gate_on_plan(false, false, false, false, true));
|
||
assert!(!should_gate_on_plan(false, false, false, true, true));
|
||
assert!(!should_gate_on_plan(true, false, false, false, true));
|
||
assert!(!should_gate_on_plan(false, true, false, false, true));
|
||
assert!(!should_gate_on_plan(false, false, true, false, true));
|
||
assert!(!should_gate_on_plan(false, false, false, false, false));
|
||
}
|
||
|
||
#[test]
|
||
fn only_real_blockers_pause_for_the_human() {
|
||
use lazyboy_harness::execution::StopReason;
|
||
assert!(pause_for_human(StopReason::NeedsPlan, "我打算先搜再存檔"));
|
||
assert!(pause_for_human(
|
||
StopReason::MidTaskText,
|
||
"需要你登入才能繼續\n[NEEDS_INPUT]"
|
||
));
|
||
assert!(!pause_for_human(
|
||
StopReason::MidTaskText,
|
||
"卡在 captcha,需要你接手"
|
||
));
|
||
assert!(!pause_for_human(StopReason::MidTaskText, "我先跳到下一步"));
|
||
assert!(!pause_for_human(StopReason::BudgetExhausted, "還在做"));
|
||
assert!(!pause_for_human(StopReason::LoopDetected, "還在做"));
|
||
let spoken = stall_report(StopReason::LoopDetected, "", 12, Some("同一個點擊"));
|
||
assert!(spoken.contains("同一個點擊"));
|
||
assert!(!spoken.contains("要我"));
|
||
assert!(!spoken.contains("繼續嗎"));
|
||
}
|
||
|
||
#[test]
|
||
fn step_labels_summarize_tool_arguments() {
|
||
assert_eq!(
|
||
describe_step("computer_observe", &json!({})),
|
||
"computer_observe: 看畫面"
|
||
);
|
||
assert_eq!(
|
||
describe_step(
|
||
"computer_act",
|
||
&json!({"actions":[{"kind":"click","x":10,"y":20},{"kind":"type","text":"hello world"}]})
|
||
),
|
||
"computer_act: click (10,20), type hello world"
|
||
);
|
||
assert_eq!(
|
||
describe_step("browser", &json!({"action":"click","element":12})),
|
||
"browser: click #12"
|
||
);
|
||
assert_eq!(
|
||
describe_step("shell", &json!({"command":"ls\n-la"})),
|
||
"shell: ls -la"
|
||
);
|
||
assert_eq!(describe_step("shell", &json!({})), "shell: 讀終端機");
|
||
assert_eq!(
|
||
describe_step("shell", &json!({"keys":"C-c","session":"build"})),
|
||
"shell: 輸入 C-c ·build"
|
||
);
|
||
assert_eq!(
|
||
describe_step("shell", &json!({"command":"make","session":"main"})),
|
||
"shell: make"
|
||
);
|
||
assert_eq!(
|
||
describe_step("shell", &json!({"reset":true,"session":"build"})),
|
||
"shell: 重開終端機 ·build"
|
||
);
|
||
assert_eq!(
|
||
describe_step("mcp_search", &json!({"query":"x"})),
|
||
"mcp_search"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn history_never_reads_past_the_current_prompt() {
|
||
assert_eq!(history_window_start(10, 5), 4);
|
||
assert_eq!(history_window_start(3, 20), 3);
|
||
assert_eq!(history_window_start(-1, 1), 0);
|
||
}
|
||
|
||
#[test]
|
||
fn screenshot_parts_keeps_caption_and_image() {
|
||
let parts = screenshot_parts(vec![0xFF, 0xD8, 0xFF, 0x00]);
|
||
assert_eq!(parts.len(), 2);
|
||
assert!(matches!(parts[0], UserContent::Text(_)));
|
||
assert!(matches!(parts[1], UserContent::Image(_)));
|
||
}
|
||
|
||
fn user_with_shot(label: &str, byte: u8) -> Message {
|
||
let mut content = vec![UserContent::text(label)];
|
||
content.extend(screenshot_parts(vec![0xFF, 0xD8, 0xFF, byte]));
|
||
Message::User { content }
|
||
}
|
||
|
||
fn image_count(history: &[Message]) -> usize {
|
||
history
|
||
.iter()
|
||
.filter_map(|message| match message {
|
||
Message::User { content } => Some(content),
|
||
_ => None,
|
||
})
|
||
.flatten()
|
||
.filter(|part| matches!(part, UserContent::Image(_)))
|
||
.count()
|
||
}
|
||
|
||
#[test]
|
||
fn history_keeps_latest_screenshot_when_new_turn_has_none() {
|
||
let mut history = vec![
|
||
user_with_shot("a", 1),
|
||
Message::Assistant {
|
||
id: None,
|
||
content: vec![],
|
||
},
|
||
user_with_shot("b", 2),
|
||
];
|
||
let pending = Message::User {
|
||
content: vec![UserContent::text("(screen unchanged)")],
|
||
};
|
||
drop_history_screenshots(&mut history, &pending);
|
||
assert_eq!(image_count(&history), 1);
|
||
let Message::User { content } = &history[2] else {
|
||
panic!("expected user message");
|
||
};
|
||
assert!(
|
||
content
|
||
.iter()
|
||
.any(|part| matches!(part, UserContent::Image(_)))
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn history_drops_every_screenshot_when_new_turn_has_one() {
|
||
let mut history = vec![user_with_shot("a", 1), user_with_shot("b", 2)];
|
||
let pending = user_with_shot("c", 3);
|
||
drop_history_screenshots(&mut history, &pending);
|
||
assert_eq!(image_count(&history), 0);
|
||
assert!(history.iter().all(|message| match message {
|
||
Message::User { content } => !content.iter().any(|part| matches!(
|
||
part,
|
||
UserContent::Text(text) if text.text == SCREENSHOT_CAPTION
|
||
)),
|
||
_ => true,
|
||
}));
|
||
}
|
||
|
||
#[test]
|
||
fn halt_maps_paused_and_cancelled_runs() {
|
||
assert_eq!(
|
||
halt_from_status(Some("cancelled")),
|
||
Some(RunHalt::Cancelled)
|
||
);
|
||
assert_eq!(
|
||
halt_from_status(Some("waiting_takeover")),
|
||
Some(RunHalt::Takeover)
|
||
);
|
||
assert_eq!(halt_from_status(Some("running")), None);
|
||
assert_eq!(halt_from_status(None), None);
|
||
}
|
||
|
||
#[test]
|
||
fn permanent_provider_errors_are_not_retried() {
|
||
assert!(!retryable_run_error(
|
||
"ProviderResponseError: status 403 Forbidden"
|
||
));
|
||
assert!(!retryable_run_error(
|
||
"ProviderResponseError: status 422 Unprocessable"
|
||
));
|
||
assert!(retryable_run_error(
|
||
"ProviderResponseError: status 429 Too Many Requests"
|
||
));
|
||
assert!(retryable_run_error("connection reset"));
|
||
// 400 overflow is handled inside the model turn (compact + retry).
|
||
// Re-queuing the same oversized checkpoint at the worker would just
|
||
// bounce. Fitting happens when the run is claimed again.
|
||
assert!(!retryable_run_error(
|
||
"ProviderResponseError: status 400 Bad Request: exceed_context_size_error"
|
||
));
|
||
}
|
||
}
|