AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
// The work loop: ask the model, run the tools it asks for (asking the person first when the
|
|
|
|
|
|
// rules say so), give it the results, and repeat until it answers in plain words.
|
|
|
|
|
|
// Long chats are folded into a summary before they outgrow the model's context.
|
|
|
|
|
|
|
|
|
|
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
|
|
|
|
use std::sync::mpsc::{channel, Sender};
|
|
|
|
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
|
|
use std::time::Duration;
|
|
|
|
|
|
|
|
|
|
|
|
use crate::appctl;
|
|
|
|
|
|
use crate::computer;
|
|
|
|
|
|
use crate::model::{Message, Role, ToolStatus};
|
|
|
|
|
|
use crate::providers::{self, AppCall, Approval, ApprovalAnswer, ApprovalKind, ChatRequest, StreamEvent};
|
|
|
|
|
|
use crate::tools::{self, Decision};
|
|
|
|
|
|
use crate::web;
|
|
|
|
|
|
|
|
|
|
|
|
/// Context size assumed when the person hasn't set one (estimated).
|
|
|
|
|
|
pub const DEFAULT_CONTEXT_TOKENS: u64 = 128_000;
|
|
|
|
|
|
/// Summarize once the chat fills this share of the context:
|
|
|
|
|
|
/// main-app.cjs `client_speculative_summarization_config.tokenUsageThresholdPercentage:70`.
|
|
|
|
|
|
pub const COMPACT_THRESHOLD_PERCENT: u64 = 70;
|
|
|
|
|
|
/// Extra room allowed in the middle of a task before summarizing again:
|
|
|
|
|
|
/// same config, `tolerancePercentage:5`.
|
|
|
|
|
|
pub const COMPACT_TOLERANCE_PERCENT: u64 = 5;
|
2026-11-06 08:51:25 +00:00
|
|
|
|
/// Most model calls in one turn: a guard against a runaway turn, not a budget. Grok Bot leaves
|
|
|
|
|
|
/// its turn window open (main-app.cjs `cloud_agent_stream_conversation_turn_window`
|
|
|
|
|
|
/// `fallbackValues:{maxTurns:null}`) and browser-use stops at 100; 25 cut off real work (fixing
|
|
|
|
|
|
/// a server's Docker took 50+ commands). Going in circles is stopped long before this by step_key.
|
|
|
|
|
|
pub const MAX_STEPS: usize = 100;
|
|
|
|
|
|
/// Every this many steps the bot is asked whether it is getting anywhere (estimated).
|
|
|
|
|
|
pub const CHECK_IN_STEPS: usize = 25;
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
/// Newest messages always kept word for word (estimated).
|
|
|
|
|
|
pub const KEEP_RECENT_MESSAGES: usize = 6;
|
|
|
|
|
|
/// Characters of each old message given to the summarizer (estimated).
|
|
|
|
|
|
const SUMMARY_SNIPPET_CHARS: usize = 2_000;
|
|
|
|
|
|
|
|
|
|
|
|
pub struct Job {
|
|
|
|
|
|
pub req: ChatRequest,
|
|
|
|
|
|
/// Computers this bot may work on (empty when computer tools are off).
|
|
|
|
|
|
pub computers: Vec<tools::Target>,
|
|
|
|
|
|
pub default_computer: String,
|
2026-11-05 08:47:28 +00:00
|
|
|
|
/// Which computer is the bot's own, which one the person is using, and whether this is a server.
|
|
|
|
|
|
pub roles: tools::Roles,
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
/// 在這台電腦上執行 is on.
|
|
|
|
|
|
pub local_execution: bool,
|
|
|
|
|
|
/// This bot's computer tools are on (when off, using them asks the person first).
|
|
|
|
|
|
pub tools_enabled: bool,
|
|
|
|
|
|
/// 直接操作 mode; the window can switch it on mid-turn ("全部允許").
|
|
|
|
|
|
pub direct: Arc<AtomicBool>,
|
|
|
|
|
|
/// 在這台電腦上執行 is locked off by an admin policy: refuse, don't ask.
|
|
|
|
|
|
pub local_locked: bool,
|
|
|
|
|
|
pub grants: Vec<String>,
|
|
|
|
|
|
pub context_tokens: u64,
|
|
|
|
|
|
pub via: String,
|
|
|
|
|
|
/// The bot's name, for the pop-up text.
|
|
|
|
|
|
pub via_name: String,
|
|
|
|
|
|
/// Messages the person sends while the turn runs (they steer it; no new turn).
|
|
|
|
|
|
pub steer: Arc<Mutex<Vec<String>>>,
|
|
|
|
|
|
/// 自動審核 rules, when on.
|
|
|
|
|
|
pub review: Option<Review>,
|
|
|
|
|
|
/// 示範教學 this bot has learned (for view_demo_step).
|
|
|
|
|
|
pub demos: Vec<crate::teach::Demo>,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[derive(Clone, Default)]
|
|
|
|
|
|
pub struct Review {
|
|
|
|
|
|
/// 「無需詢問直接允許」規則
|
|
|
|
|
|
pub allow: Vec<String>,
|
|
|
|
|
|
/// 「一律詢問」規則 (wins when both match)
|
|
|
|
|
|
pub ask: Vec<String>,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub enum ReviewVerdict {
|
|
|
|
|
|
Allow(String),
|
|
|
|
|
|
Ask(String),
|
|
|
|
|
|
NoMatch,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Ask the model whether an action matches the person's rules. "Ask first" wins on conflict
|
|
|
|
|
|
/// (NFRGrQ: 規則衝突時,「先詢問」優先); built-in safety checks have already run before this.
|
|
|
|
|
|
fn review_action(job: &Job, review: &Review, action: &str, cancel: &AtomicBool) -> ReviewVerdict {
|
|
|
|
|
|
if review.allow.is_empty() && review.ask.is_empty() {
|
|
|
|
|
|
return ReviewVerdict::NoMatch;
|
|
|
|
|
|
}
|
|
|
|
|
|
let list = |v: &[String]| if v.is_empty() { "(無)".to_string() } else { v.iter().enumerate().map(|(i, r)| format!("{}. {r}", i + 1)).collect::<Vec<_>>().join("\n") };
|
|
|
|
|
|
let req = ChatRequest {
|
|
|
|
|
|
system: "你是動作審核員。只根據使用者寫的規則判斷,不要自己加規則。只回一行 JSON:{\"verdict\":\"ask\"|\"allow\"|\"none\",\"rule\":\"符合的規則原文\"}。如果符合任何「一律詢問」規則就回 ask(優先);否則符合「無需詢問直接允許」規則就回 allow;都不符合回 none。".into(),
|
|
|
|
|
|
history: vec![Message::user(&format!("要審核的動作:{action}\n\n「一律詢問」規則:\n{}\n\n「無需詢問直接允許」規則:\n{}", list(&review.ask), list(&review.allow)))],
|
|
|
|
|
|
tools: vec![],
|
|
|
|
|
|
..job.req.clone()
|
|
|
|
|
|
};
|
|
|
|
|
|
let Ok(out) = providers::run_step(&req, cancel, &|_| {}) else { return ReviewVerdict::NoMatch };
|
|
|
|
|
|
let text = out.text.trim();
|
2026-11-05 07:48:20 +00:00
|
|
|
|
// A truncated answer can put the last "}" before the first "{" — slicing then panics.
|
|
|
|
|
|
let json_part = text.find('{').and_then(|a| text.rfind('}').filter(|b| *b >= a).map(|b| &text[a..=b])).unwrap_or(text);
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
let v: serde_json::Value = serde_json::from_str(json_part).unwrap_or_default();
|
|
|
|
|
|
let rule = v["rule"].as_str().unwrap_or("").to_string();
|
|
|
|
|
|
match v["verdict"].as_str() {
|
|
|
|
|
|
Some("ask") => ReviewVerdict::Ask(rule),
|
|
|
|
|
|
Some("allow") => ReviewVerdict::Allow(rule),
|
|
|
|
|
|
_ => ReviewVerdict::NoMatch,
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Take what the person typed mid-turn and add it to the chat as steering.
|
|
|
|
|
|
fn drain_steer(job: &Job, history: &mut Vec<Message>, send: &dyn Fn(StreamEvent)) -> bool {
|
|
|
|
|
|
let queued: Vec<String> = std::mem::take(&mut *job.steer.lock().unwrap());
|
|
|
|
|
|
for t in &queued {
|
|
|
|
|
|
let m = Message { steer: true, ..Message::user(t) };
|
|
|
|
|
|
send(StreamEvent::Steered(m.clone()));
|
|
|
|
|
|
history.push(m);
|
|
|
|
|
|
}
|
|
|
|
|
|
!queued.is_empty()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Which computer a call asks for (permission to use it is checked separately).
|
|
|
|
|
|
fn target_for(job: &Job, args: &str) -> Result<tools::Target, String> {
|
|
|
|
|
|
let v = tools::args_of(args);
|
|
|
|
|
|
let want = v.get("computer").and_then(|c| c.as_str()).filter(|c| !c.is_empty()).unwrap_or(&job.default_computer).to_string();
|
Remote brain: a computer that just came on is known at once; an offline computer is said, not swapped; snapshots survive an open browser; end-to-end test
- The brain looked the account's computers up once a minute: for up to a minute after the Mac
came on, 「我的電腦」 was unknown and steps went elsewhere. Now every 15 s, and at once when a
message comes from a computer it doesn't know yet.
- A step naming a computer that is off says so in words instead of running on the chosen one
with a note that the person hadn't asked.
- Snapshots: tar's 'file changed' (the open browser) is not a failure; browser caches are left
out; ten minutes for a large home folder.
- box_end_to_end clicks the terminal where it opened; restore only with ANYBOT_TEST_RESTORE=1.
- deploy/test/remote_e2e.py: account server + headless brain + hands + scripted model, 27 checks
(computers, redirect, agent-desktop on the hands, hand-off 3 ways, phone view, sync copy,
edits, hands offline/back, brain restart, clear). anybot --sample-state for such tests.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 10:14:23 +00:00
|
|
|
|
job.computers.iter().find(|t| t.name == want).cloned().ok_or_else(|| not_here(job, &want))
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-05 07:00:31 +00:00
|
|
|
|
enum BrowserAt {
|
|
|
|
|
|
Here,
|
|
|
|
|
|
Box,
|
|
|
|
|
|
Device(String),
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-05 08:47:28 +00:00
|
|
|
|
/// Whose browser a browser step uses: the computer it names, else the bot's chosen computer; a
|
|
|
|
|
|
/// computer without a browser (a server, an SSH one) hands it to the person's computer.
|
|
|
|
|
|
fn browser_place(job: &Job, name: &str, args: &str) -> (BrowserAt, String) {
|
2026-11-05 07:00:31 +00:00
|
|
|
|
if name == "web_fetch" {
|
2026-11-05 08:47:28 +00:00
|
|
|
|
return (BrowserAt::Here, String::new());
|
2026-11-05 07:00:31 +00:00
|
|
|
|
}
|
|
|
|
|
|
let v = tools::args_of(args);
|
|
|
|
|
|
let want = v.get("computer").and_then(|c| c.as_str()).map(str::trim).filter(|c| !c.is_empty()).unwrap_or(&job.default_computer).to_string();
|
2026-11-05 08:47:28 +00:00
|
|
|
|
let at = |t: &tools::Target| -> Option<BrowserAt> {
|
|
|
|
|
|
match &t.place {
|
2026-11-05 13:28:32 +00:00
|
|
|
|
tools::Place::Device { host, .. } => Some(BrowserAt::Device(host.clone())),
|
2026-11-05 08:47:28 +00:00
|
|
|
|
tools::Place::Box { .. } if crate::cloudbox::available() => Some(BrowserAt::Box),
|
|
|
|
|
|
tools::Place::Local { .. } if crate::botbrowser::available() => Some(BrowserAt::Here),
|
|
|
|
|
|
_ => None,
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
if let Some((b, n)) = job.computers.iter().find(|t| t.name == want).and_then(|t| at(t).map(|b| (b, t.name.clone()))) {
|
|
|
|
|
|
return (b, n);
|
|
|
|
|
|
}
|
2026-11-05 11:41:49 +00:00
|
|
|
|
let person = roles_now(job).person.and_then(|p| job.computers.iter().find(|t| t.name == p));
|
2026-11-05 08:47:28 +00:00
|
|
|
|
match person.and_then(|t| at(t).map(|b| (b, t.name.clone()))).or_else(|| job.computers.iter().find_map(|t| at(t).map(|b| (b, t.name.clone())))) {
|
|
|
|
|
|
Some(x) => x,
|
|
|
|
|
|
None => (BrowserAt::Here, String::new()),
|
2026-11-05 07:00:31 +00:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-06 08:51:25 +00:00
|
|
|
|
/// Is this browser step for the bot's own browser (no permission, no person's computer)? The
|
|
|
|
|
|
/// box's Chromium always is — whether *this* computer has a Chrome doesn't matter there. (Asking
|
|
|
|
|
|
/// this computer made a cloud brain, which has no Chrome, send every browser_open/read meant for
|
|
|
|
|
|
/// the box to the person's browser on the server: 「打不開瀏覽器」, 「Safari 沒有回應」.)
|
|
|
|
|
|
fn bots_browser(place: &BrowserAt, name: &str, args: &str) -> bool {
|
|
|
|
|
|
matches!(place, BrowserAt::Box) || !web::uses_this_computer(name, args)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-05 07:48:20 +00:00
|
|
|
|
/// Substring search with word boundaries on the ASCII parts: "mac" must not light up inside
|
|
|
|
|
|
/// "macOS", and a device named "Ann" must not light up on "anniversary". Loose matching let
|
|
|
|
|
|
/// unrelated words — including words from page text that arrives inside a message — unlock a
|
|
|
|
|
|
/// computer the person never named, defeating the guard below.
|
|
|
|
|
|
fn finds(hay: &str, needle: &str) -> bool {
|
|
|
|
|
|
let hb: Vec<char> = hay.chars().collect();
|
|
|
|
|
|
let nb: Vec<char> = needle.chars().collect();
|
|
|
|
|
|
if nb.is_empty() || nb.len() > hb.len() {
|
|
|
|
|
|
return false;
|
|
|
|
|
|
}
|
|
|
|
|
|
let alnum = |c: char| c.is_ascii_alphanumeric();
|
|
|
|
|
|
'start: for i in 0..=hb.len() - nb.len() {
|
|
|
|
|
|
if hb[i] != nb[0] {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
for j in 1..nb.len() {
|
|
|
|
|
|
if hb[i + j] != nb[j] {
|
|
|
|
|
|
continue 'start;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
if alnum(nb[0]) && i > 0 && alnum(hb[i - 1]) {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
if alnum(*nb.last().unwrap()) && i + nb.len() < hb.len() && alnum(hb[i + nb.len()]) {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
return true;
|
|
|
|
|
|
}
|
|
|
|
|
|
false
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-05 08:47:28 +00:00
|
|
|
|
/// Words that mean a computer in the person's message: its name (or the part after 「的」, so
|
|
|
|
|
|
/// 「Mac mini」 names 「王小明的Mac mini」), and for the bot's own computer and the one the person
|
|
|
|
|
|
/// is using, the everyday words for them.
|
|
|
|
|
|
fn mentions(text: &str, t: &tools::Target, roles: &tools::Roles) -> bool {
|
2026-11-05 07:00:31 +00:00
|
|
|
|
let low = text.to_lowercase();
|
2026-11-05 08:47:28 +00:00
|
|
|
|
let name = t.name.to_lowercase();
|
|
|
|
|
|
if finds(&low, &name) {
|
|
|
|
|
|
return true;
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Some((_, tail)) = name.rsplit_once('的') {
|
|
|
|
|
|
let tail = tail.trim();
|
|
|
|
|
|
if tail.chars().count() >= 3 && tail != "電腦" && finds(&low, tail) {
|
|
|
|
|
|
return true;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
if finds(&name, "mac") && finds(&low, "mac") {
|
2026-11-05 07:00:31 +00:00
|
|
|
|
return true;
|
|
|
|
|
|
}
|
2026-11-05 08:47:28 +00:00
|
|
|
|
let words: &[&str] = if t.name == roles.own {
|
|
|
|
|
|
&["bot 的電腦", "bot的電腦", "你的電腦", "你自己的電腦", "你的虛擬", "虛擬電腦", "雲端電腦"]
|
|
|
|
|
|
} else if roles.person.as_deref() == Some(t.name.as_str()) {
|
|
|
|
|
|
&["我的電腦", "我電腦", "我這台", "這台電腦", "本機", "我的筆電", "my computer"]
|
|
|
|
|
|
} else if roles.server && matches!(t.place, tools::Place::Local { .. }) {
|
|
|
|
|
|
&["伺服器", "server"]
|
|
|
|
|
|
} else {
|
|
|
|
|
|
&[]
|
2026-11-05 07:00:31 +00:00
|
|
|
|
};
|
2026-11-05 08:47:28 +00:00
|
|
|
|
if t.name == roles.own && finds(&low, &name.replace(' ', "")) {
|
|
|
|
|
|
return true;
|
|
|
|
|
|
}
|
2026-11-05 07:48:20 +00:00
|
|
|
|
words.iter().any(|w| finds(&low, w))
|
2026-11-05 07:00:31 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// A step naming a computer other than the chosen one, which the person didn't ask for, goes
|
2026-11-05 08:47:28 +00:00
|
|
|
|
/// to the chosen one — and the bot is told so (quietly moving it made a bot "ask your Mac" six
|
|
|
|
|
|
/// times while every step landed on its own computer).
|
|
|
|
|
|
fn keep_to_chosen(job: &Job, history: &[Message], args: &str) -> (String, Option<String>) {
|
2026-11-05 07:00:31 +00:00
|
|
|
|
let mut v = tools::args_of(args);
|
2026-11-05 08:47:28 +00:00
|
|
|
|
let Some(named) = v.get("computer").and_then(|c| c.as_str()).map(|c| c.trim().to_string()).filter(|c| !c.is_empty()) else { return (args.to_string(), None) };
|
2026-11-05 07:00:31 +00:00
|
|
|
|
if named == job.default_computer || !job.computers.iter().any(|t| t.name == job.default_computer) {
|
2026-11-05 08:47:28 +00:00
|
|
|
|
return (args.to_string(), None);
|
2026-11-05 07:00:31 +00:00
|
|
|
|
}
|
|
|
|
|
|
let said = history.iter().rev().filter(|m| m.role == Role::User && !m.summary).take(3).map(|m| m.text.as_str()).collect::<Vec<_>>().join("\n");
|
|
|
|
|
|
match job.computers.iter().find(|t| t.name == named) {
|
2026-11-05 11:41:49 +00:00
|
|
|
|
Some(t) if mentions(&said, t, &roles_now(job)) => (args.to_string(), None),
|
Remote brain: a computer that just came on is known at once; an offline computer is said, not swapped; snapshots survive an open browser; end-to-end test
- The brain looked the account's computers up once a minute: for up to a minute after the Mac
came on, 「我的電腦」 was unknown and steps went elsewhere. Now every 15 s, and at once when a
message comes from a computer it doesn't know yet.
- A step naming a computer that is off says so in words instead of running on the chosen one
with a note that the person hadn't asked.
- Snapshots: tar's 'file changed' (the open browser) is not a failure; browser caches are left
out; ten minutes for a large home folder.
- box_end_to_end clicks the terminal where it opened; restore only with ANYBOT_TEST_RESTORE=1.
- deploy/test/remote_e2e.py: account server + headless brain + hands + scripted model, 27 checks
(computers, redirect, agent-desktop on the hands, hand-off 3 ways, phone view, sync copy,
edits, hands offline/back, brain restart, clear). anybot --sample-state for such tests.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 10:14:23 +00:00
|
|
|
|
// A computer that isn't on the list now (switched off, its AnyBot closed): the step says
|
|
|
|
|
|
// so instead of quietly running somewhere else.
|
|
|
|
|
|
None => (args.to_string(), None),
|
2026-11-05 07:00:31 +00:00
|
|
|
|
_ => {
|
|
|
|
|
|
v["computer"] = serde_json::Value::String(job.default_computer.clone());
|
2026-11-05 08:47:28 +00:00
|
|
|
|
let note = format!("(這一步你指定了「{named}」,但使用者選的電腦是「{}」,而且他沒有說要換,所以這一步在「{}」上做。真的需要別台,先問使用者。)\n", job.default_computer, job.default_computer);
|
|
|
|
|
|
(v.to_string(), Some(note))
|
2026-11-05 07:00:31 +00:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-05 10:25:46 +00:00
|
|
|
|
/// What element `e7` is, from the latest app list in the chat (「按鈕「7」」).
|
|
|
|
|
|
fn element_named(history: &[Message], r: &str) -> Option<String> {
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let tag = format!("[{}] ", crate::marks::number_of(r)?);
|
2026-11-05 10:25:46 +00:00
|
|
|
|
history.iter().rev().filter(|m| m.role == Role::Tool).take(12).find_map(|m| {
|
|
|
|
|
|
m.text.lines().find_map(|l| l.trim_start().strip_prefix(&tag).map(|rest| rest.trim().to_string())).filter(|e| !e.is_empty())
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
/// What makes two screen or browser steps "the same" (None for other tools): the tool, the
|
|
|
|
|
|
/// computer and the target, a click's spot on a 16-pixel grid (an inch beside an empty strip is
|
|
|
|
|
|
/// the same miss).
|
|
|
|
|
|
fn step_key(name: &str, args: &str) -> Option<String> {
|
|
|
|
|
|
if !(computer::is_computer_tool(name) || web::is_web_tool(name)) || matches!(name, "computer_use_screenshot" | "browser_read" | "browser_screenshot" | "browser_wait") {
|
|
|
|
|
|
return None;
|
|
|
|
|
|
}
|
|
|
|
|
|
let mut v = tools::args_of(args);
|
|
|
|
|
|
if let Some(o) = v.as_object_mut() {
|
|
|
|
|
|
for k in ["x", "y"] {
|
|
|
|
|
|
if let Some(n) = o.get(k).and_then(serde_json::Value::as_f64) {
|
|
|
|
|
|
o.insert(k.into(), serde_json::Value::from((n / 16.0).round() as i64));
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
Some(format!("{name} {v}"))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-06 08:51:25 +00:00
|
|
|
|
/// Every CHECK_IN_STEPS steps, the newest result carries a question for the bot: still getting
|
|
|
|
|
|
/// somewhere, or stuck? Only the model sees it (the chat keeps the result as it was).
|
|
|
|
|
|
fn check_in(history: &mut [Message], step: usize) {
|
|
|
|
|
|
if step == 0 || !step.is_multiple_of(CHECK_IN_STEPS) {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Some(m) = history.last_mut().filter(|m| m.role == Role::Tool) {
|
|
|
|
|
|
m.text.push_str(&format!("\n\n(這一回合已經做了 {step} 步,最多 {MAX_STEPS} 步。方向對、有進展就繼續做完;如果最近幾步都沒有進展,停下來用一兩句話告訴使用者卡在哪、需要他做什麼。)"));
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-05 08:47:28 +00:00
|
|
|
|
/// A step's result, with a word first when the step was moved back to the chosen computer.
|
|
|
|
|
|
fn noted(moved: &std::collections::HashMap<String, String>, id: &str, text: &str) -> String {
|
|
|
|
|
|
match moved.get(id) {
|
|
|
|
|
|
Some(n) => format!("{n}{text}"),
|
|
|
|
|
|
None => text.to_string(),
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
Remote brain: a computer that just came on is known at once; an offline computer is said, not swapped; snapshots survive an open browser; end-to-end test
- The brain looked the account's computers up once a minute: for up to a minute after the Mac
came on, 「我的電腦」 was unknown and steps went elsewhere. Now every 15 s, and at once when a
message comes from a computer it doesn't know yet.
- A step naming a computer that is off says so in words instead of running on the chosen one
with a note that the person hadn't asked.
- Snapshots: tar's 'file changed' (the open browser) is not a failure; browser caches are left
out; ten minutes for a large home folder.
- box_end_to_end clicks the terminal where it opened; restore only with ANYBOT_TEST_RESTORE=1.
- deploy/test/remote_e2e.py: account server + headless brain + hands + scripted model, 27 checks
(computers, redirect, agent-desktop on the hands, hand-off 3 ways, phone view, sync copy,
edits, hands offline/back, brain restart, clear). anybot --sample-state for such tests.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 10:14:23 +00:00
|
|
|
|
/// A computer the step names that isn't among this bot's computers right now, in plain words.
|
|
|
|
|
|
fn not_here(job: &Job, name: &str) -> String {
|
|
|
|
|
|
format!("「{name}」現在連不到:那台電腦可能沒開,或上面的 AnyBot 沒開、沒登入。請告訴使用者,等它連上再做(或問他要不要改用別台)。現在可以用:{}", job.computers.iter().map(|t| t.name.as_str()).collect::<Vec<_>>().join("、"))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-05 11:41:49 +00:00
|
|
|
|
/// Which computer is the person's right now: a message typed on one of the account's computers
|
|
|
|
|
|
/// while the turn runs (a steer, or the answer to a hand-off) moves it there — the turn's own
|
|
|
|
|
|
/// idea was fixed when it started, maybe from the phone, which isn't a computer.
|
|
|
|
|
|
fn roles_now(job: &Job) -> tools::Roles {
|
|
|
|
|
|
let mut r = job.roles.clone();
|
|
|
|
|
|
let host = crate::remote::person_host();
|
|
|
|
|
|
if !host.is_empty() {
|
2026-11-05 13:28:32 +00:00
|
|
|
|
if let Some(t) = job.computers.iter().find(|t| matches!(&t.place, tools::Place::Device { host: h, .. } if *h == host)) {
|
2026-11-05 11:41:49 +00:00
|
|
|
|
r.person = Some(t.name.clone());
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
r
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-05 06:46:42 +00:00
|
|
|
|
/// Does the call name a computer?
|
|
|
|
|
|
fn names_a_computer(args: &str) -> bool {
|
|
|
|
|
|
tools::args_of(args).get("computer").and_then(|c| c.as_str()).map(|c| !c.trim().is_empty()).unwrap_or(false)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
/// Rough token count: about 4 Latin characters or 1 Chinese character per token (estimated).
|
|
|
|
|
|
/// Which screen a computer_use call is for: the one named, else the bot's default when it has a
|
|
|
|
|
|
/// screen, else the bot's own desktop when there is one, else this Mac.
|
|
|
|
|
|
fn screen_target(job: &Job, args: &str) -> Result<tools::Target, String> {
|
|
|
|
|
|
let v = tools::args_of(args);
|
|
|
|
|
|
let has_screen = |t: &tools::Target| matches!(t.place, tools::Place::Box { .. } | tools::Place::Local { .. } | tools::Place::Device { .. });
|
|
|
|
|
|
let named = v.get("computer").and_then(|c| c.as_str()).map(str::trim).filter(|c| !c.is_empty());
|
|
|
|
|
|
if let Some(n) = named {
|
Remote brain: a computer that just came on is known at once; an offline computer is said, not swapped; snapshots survive an open browser; end-to-end test
- The brain looked the account's computers up once a minute: for up to a minute after the Mac
came on, 「我的電腦」 was unknown and steps went elsewhere. Now every 15 s, and at once when a
message comes from a computer it doesn't know yet.
- A step naming a computer that is off says so in words instead of running on the chosen one
with a note that the person hadn't asked.
- Snapshots: tar's 'file changed' (the open browser) is not a failure; browser caches are left
out; ten minutes for a large home folder.
- box_end_to_end clicks the terminal where it opened; restore only with ANYBOT_TEST_RESTORE=1.
- deploy/test/remote_e2e.py: account server + headless brain + hands + scripted model, 27 checks
(computers, redirect, agent-desktop on the hands, hand-off 3 ways, phone view, sync copy,
edits, hands offline/back, brain restart, clear). anybot --sample-state for such tests.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 10:14:23 +00:00
|
|
|
|
let t = job.computers.iter().find(|t| t.name == n).ok_or_else(|| not_here(job, n))?;
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
if !has_screen(t) {
|
2026-11-05 08:47:28 +00:00
|
|
|
|
return Err(if n == job.roles.own {
|
|
|
|
|
|
format!("{n}現在沒有螢幕:AnyBot 所在的電腦沒有 Docker({})。要用別台請使用者指名。", if cfg!(target_os = "macos") { "安裝 OrbStack 或 Docker Desktop 就會有" } else { "安裝 Docker Engine 就會有" })
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
} else {
|
|
|
|
|
|
format!("「{n}」不能遠端操作螢幕,只能用檔案和指令工具。")
|
|
|
|
|
|
});
|
|
|
|
|
|
}
|
|
|
|
|
|
return Ok(t.clone());
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Some(t) = job.computers.iter().find(|t| t.name == job.default_computer && has_screen(t)) {
|
|
|
|
|
|
return Ok(t.clone());
|
|
|
|
|
|
}
|
2026-11-05 00:05:29 +00:00
|
|
|
|
// Not named: a screen that is really there — the bot's desktop, this computer's, then one of
|
|
|
|
|
|
// the account's other computers (a brain on a server has no screen of its own).
|
|
|
|
|
|
let real = |t: &&tools::Target| match t.place {
|
|
|
|
|
|
tools::Place::Box { .. } => crate::cloudbox::available(),
|
|
|
|
|
|
tools::Place::Local { .. } => computer::screen_supported(),
|
|
|
|
|
|
_ => false,
|
|
|
|
|
|
};
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
job.computers
|
|
|
|
|
|
.iter()
|
2026-11-05 00:05:29 +00:00
|
|
|
|
.find(|t| matches!(t.place, tools::Place::Box { .. }) && real(t))
|
|
|
|
|
|
.or_else(|| job.computers.iter().find(|t| matches!(t.place, tools::Place::Local { .. }) && real(t)))
|
|
|
|
|
|
.or_else(|| job.computers.iter().find(|t| matches!(t.place, tools::Place::Device { .. })))
|
|
|
|
|
|
.or_else(|| job.computers.iter().find(|t| matches!(t.place, tools::Place::Box { .. })))
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
.or_else(|| job.computers.iter().find(|t| matches!(t.place, tools::Place::Local { .. })))
|
|
|
|
|
|
.cloned()
|
|
|
|
|
|
.ok_or_else(|| "找不到可以操作螢幕的電腦".to_string())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub fn estimate_tokens(s: &str) -> u64 {
|
|
|
|
|
|
let (mut ascii, mut other) = (0u64, 0u64);
|
|
|
|
|
|
for c in s.chars() {
|
|
|
|
|
|
if c.is_ascii() { ascii += 1 } else { other += 1 }
|
|
|
|
|
|
}
|
|
|
|
|
|
ascii / 4 + other
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn history_tokens(system: &str, h: &[Message]) -> u64 {
|
|
|
|
|
|
estimate_tokens(system) + h.iter().map(|m| estimate_tokens(&m.text) + m.tool_calls.iter().map(|c| estimate_tokens(&c.args)).sum::<u64>() + 4).sum::<u64>()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Where to cut so the newest messages stay and no tool call is split from its result:
|
|
|
|
|
|
/// the summary covers everything before a user message.
|
|
|
|
|
|
pub fn compaction_cut(h: &[Message]) -> Option<usize> {
|
|
|
|
|
|
if h.len() <= KEEP_RECENT_MESSAGES + 1 {
|
|
|
|
|
|
return None;
|
|
|
|
|
|
}
|
|
|
|
|
|
(1..=h.len() - KEEP_RECENT_MESSAGES).rev().find(|&i| h[i].role == Role::User && !h[..i].iter().all(|m| m.summary))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn flatten(h: &[Message]) -> String {
|
|
|
|
|
|
let clip = |s: &str| -> String {
|
|
|
|
|
|
let mut t: String = s.chars().take(SUMMARY_SNIPPET_CHARS).collect();
|
|
|
|
|
|
if s.chars().count() > SUMMARY_SNIPPET_CHARS {
|
|
|
|
|
|
t.push('…');
|
|
|
|
|
|
}
|
|
|
|
|
|
t
|
|
|
|
|
|
};
|
|
|
|
|
|
h.iter()
|
|
|
|
|
|
.map(|m| {
|
|
|
|
|
|
if m.summary {
|
|
|
|
|
|
return format!("【先前的摘要】\n{}", m.text);
|
|
|
|
|
|
}
|
|
|
|
|
|
match m.role {
|
|
|
|
|
|
Role::User => format!("使用者:{}", clip(&m.text)),
|
|
|
|
|
|
Role::Assistant => {
|
|
|
|
|
|
let calls: Vec<String> = m.tool_calls.iter().map(|c| format!("[使用工具 {}:{}]", c.name, tools::describe(&c.name, &c.args))).collect();
|
|
|
|
|
|
format!("助手:{} {}", clip(&m.text), calls.join(" "))
|
|
|
|
|
|
}
|
|
|
|
|
|
Role::Tool => format!("工具結果({}):{}", m.tool_name, clip(&m.text)),
|
|
|
|
|
|
}
|
|
|
|
|
|
})
|
|
|
|
|
|
.collect::<Vec<_>>()
|
|
|
|
|
|
.join("\n\n")
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Fold older messages into one summary when the chat is over `percent` of the context.
|
|
|
|
|
|
/// Returns how many messages were replaced.
|
|
|
|
|
|
fn maybe_compact(job: &Job, history: &mut Vec<Message>, percent: u64, cancel: &AtomicBool, send: &dyn Fn(StreamEvent)) -> Option<usize> {
|
|
|
|
|
|
let used = history_tokens(&job.req.system, history);
|
|
|
|
|
|
if used * 100 < job.context_tokens * percent {
|
|
|
|
|
|
return None;
|
|
|
|
|
|
}
|
|
|
|
|
|
let cut = compaction_cut(history)?;
|
|
|
|
|
|
send(StreamEvent::Retrying("對話很長了,正在把較早的內容整理成摘要…".into()));
|
|
|
|
|
|
let req = ChatRequest {
|
|
|
|
|
|
system: "你負責把一段對話整理成摘要,讓之後的對話能接著進行。請用條列寫下:使用者的目標與偏好、已經做出的決定、重要的事實與數字、提到的檔案路徑與指令結果、還沒完成的事。不要加入對話裡沒有的內容。".into(),
|
|
|
|
|
|
history: vec![Message::user(&format!("請整理下面這段對話:\n\n{}", flatten(&history[..cut])))],
|
|
|
|
|
|
tools: vec![],
|
|
|
|
|
|
..job.req.clone()
|
|
|
|
|
|
};
|
|
|
|
|
|
let quiet = |ev: StreamEvent| {
|
|
|
|
|
|
if matches!(ev, StreamEvent::Alive | StreamEvent::Retrying(_)) {
|
|
|
|
|
|
send(StreamEvent::Alive);
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
let out = providers::run_step(&req, cancel, &quiet).ok()?;
|
|
|
|
|
|
if out.text.trim().is_empty() || cancel.load(Ordering::Relaxed) {
|
|
|
|
|
|
return None;
|
|
|
|
|
|
}
|
|
|
|
|
|
let summary = Message { role: Role::Assistant, text: out.text.trim().to_string(), summary: true, via: job.via.clone(), at: crate::model::now_ms(), ..Default::default() };
|
2026-11-05 07:48:20 +00:00
|
|
|
|
// The window shows the chat unfiltered (group chats drop some messages on the way here);
|
|
|
|
|
|
// the timestamp of the first kept message is the landmark it can find the cut by.
|
|
|
|
|
|
let boundary = history[cut].at;
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
history.splice(..cut, [summary.clone()]);
|
2026-11-05 07:48:20 +00:00
|
|
|
|
send(StreamEvent::Compacted(cut, summary, boundary));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
Some(cut)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Wait for the person's answer; stopping the reply counts as a no.
|
|
|
|
|
|
fn wait_answer(rx: std::sync::mpsc::Receiver<ApprovalAnswer>, cancel: &AtomicBool) -> ApprovalAnswer {
|
|
|
|
|
|
loop {
|
|
|
|
|
|
if cancel.load(Ordering::Relaxed) {
|
|
|
|
|
|
return ApprovalAnswer::Reject;
|
|
|
|
|
|
}
|
|
|
|
|
|
match rx.recv_timeout(Duration::from_millis(200)) {
|
|
|
|
|
|
Ok(a) => return a,
|
|
|
|
|
|
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue,
|
|
|
|
|
|
Err(_) => return ApprovalAnswer::Reject,
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc<AtomicBool>, ctx: eframe::egui::Context) {
|
|
|
|
|
|
std::thread::spawn(move || {
|
|
|
|
|
|
let send = |ev: StreamEvent| {
|
|
|
|
|
|
let _ = tx.send((bot_id, ev));
|
|
|
|
|
|
ctx.request_repaint();
|
|
|
|
|
|
};
|
|
|
|
|
|
let mut history = job.req.history.clone();
|
|
|
|
|
|
let mut grants = job.grants.clone();
|
|
|
|
|
|
// Turned on by an "always" answer mid-turn (the window saves the setting too).
|
|
|
|
|
|
let mut tools_on = job.tools_enabled;
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
// The screen and browser steps of this turn (to stop the same step being tried over and over).
|
|
|
|
|
|
let mut tried: Vec<String> = Vec::new();
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
let mut local_on = job.local_execution;
|
|
|
|
|
|
maybe_compact(&job, &mut history, COMPACT_THRESHOLD_PERCENT, &cancel, &send);
|
|
|
|
|
|
for step in 0..MAX_STEPS {
|
|
|
|
|
|
if step > 0 {
|
|
|
|
|
|
maybe_compact(&job, &mut history, COMPACT_THRESHOLD_PERCENT + COMPACT_TOLERANCE_PERCENT, &cancel, &send);
|
|
|
|
|
|
drain_steer(&job, &mut history, &send);
|
2026-11-06 08:51:25 +00:00
|
|
|
|
check_in(&mut history, step);
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
send(StreamEvent::NextStep(job.via.clone()));
|
|
|
|
|
|
}
|
|
|
|
|
|
let req = ChatRequest { history: history.clone(), ..job.req.clone() };
|
|
|
|
|
|
let out = match providers::run_step(&req, &cancel, &send) {
|
|
|
|
|
|
Ok(o) => o,
|
|
|
|
|
|
Err(e) => {
|
|
|
|
|
|
send(StreamEvent::Error(e));
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
if cancel.load(Ordering::Relaxed) {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
2026-11-05 07:00:31 +00:00
|
|
|
|
let mut out = out;
|
2026-11-05 08:47:28 +00:00
|
|
|
|
// Steps moved back to the chosen computer: what the bot is told with each result.
|
|
|
|
|
|
let mut moved: std::collections::HashMap<String, String> = std::collections::HashMap::new();
|
2026-11-05 07:00:31 +00:00
|
|
|
|
for c in out.tool_calls.iter_mut() {
|
2026-11-05 08:47:28 +00:00
|
|
|
|
let (args, note) = keep_to_chosen(&job, &history, &c.args);
|
|
|
|
|
|
c.args = args;
|
|
|
|
|
|
if let Some(n) = note {
|
|
|
|
|
|
moved.insert(c.id.clone(), n);
|
|
|
|
|
|
}
|
2026-11-05 07:00:31 +00:00
|
|
|
|
}
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
let msg = Message {
|
|
|
|
|
|
role: Role::Assistant,
|
2026-11-05 06:46:42 +00:00
|
|
|
|
text: crate::i18n::reply_text(&out.text),
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
thinking: out.thinking,
|
|
|
|
|
|
via: job.via.clone(),
|
|
|
|
|
|
tool_calls: out.tool_calls,
|
|
|
|
|
|
blocks: out.blocks,
|
|
|
|
|
|
at: crate::model::now_ms(),
|
|
|
|
|
|
..Default::default()
|
|
|
|
|
|
};
|
|
|
|
|
|
send(StreamEvent::StepDone(msg.clone()));
|
|
|
|
|
|
let calls = msg.tool_calls.clone();
|
|
|
|
|
|
history.push(msg);
|
|
|
|
|
|
if calls.is_empty() {
|
|
|
|
|
|
// Something typed while it answered keeps the same turn going.
|
|
|
|
|
|
if job.steer.lock().unwrap().is_empty() {
|
|
|
|
|
|
send(StreamEvent::Done);
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
2026-11-05 07:00:31 +00:00
|
|
|
|
// (Each call's computer was already kept to the chosen one, see keep_to_chosen.)
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
for call in calls {
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
// 鬼打牆: the same screen or browser step again and again (a click on the same
|
|
|
|
|
|
// empty spot ten times). The third time it runs with a warning; after that it isn't
|
|
|
|
|
|
// run at all, and the bot is told to change course.
|
|
|
|
|
|
let mut repeat_note = String::new();
|
|
|
|
|
|
if let Some(key) = step_key(&call.name, &call.args) {
|
|
|
|
|
|
let times = tried.iter().rev().take(10).filter(|k| **k == key).count() + 1;
|
|
|
|
|
|
tried.push(key);
|
|
|
|
|
|
if times >= 4 {
|
|
|
|
|
|
let text = format!("沒有執行:這一步你已經試了 {times} 次,畫面都沒有變成你要的樣子。不要再試同一步。改用別的方法:照截圖上的編號點、網頁改用 browser_read 拿連結直接 browser_open,或用 hand_over_computer 請使用者幫忙。");
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Failed));
|
|
|
|
|
|
let result = Message::tool_result(&call.id, &call.name, &text);
|
|
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
if times == 3 {
|
|
|
|
|
|
repeat_note = "⚠️ 這一步你已經做了 3 次。如果畫面還不是你要的,下一步一定要換方法,不要再做同一步。\n".into();
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
if appctl::is_app_tool(&call.name) {
|
|
|
|
|
|
// App actions run on the window's side, which also asks for deletions.
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running));
|
|
|
|
|
|
let (rtx, rrx) = channel();
|
|
|
|
|
|
send(StreamEvent::AppAction(AppCall { call_id: call.id.clone(), name: call.name.clone(), args: call.args.clone(), reply: rtx }));
|
|
|
|
|
|
let (output, ok) = loop {
|
|
|
|
|
|
if cancel.load(Ordering::Relaxed) {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
match rrx.recv_timeout(Duration::from_millis(200)) {
|
|
|
|
|
|
Ok(r) => break r,
|
|
|
|
|
|
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue,
|
|
|
|
|
|
Err(_) => break ("App 沒有回應".to_string(), false),
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed }));
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output)));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
// The person's own recording, shown back to the bot: nothing to ask.
|
|
|
|
|
|
if call.name == "view_demo_step" {
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running));
|
|
|
|
|
|
let (output, ok, img) = crate::teach::view_step(&job.demos, &call.args);
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed }));
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let mut result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output)));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
result.images.extend(img);
|
|
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
// Public web pages need no computer and no permission.
|
|
|
|
|
|
// …and neither does AnyBot's own browser (like Grok Bot's own browser).
|
2026-11-06 08:51:25 +00:00
|
|
|
|
let at = web::is_web_tool(&call.name).then(|| browser_place(&job, &call.name, &call.args));
|
|
|
|
|
|
if let Some((place, on)) = at.filter(|(place, _)| call.name == "web_fetch" || bots_browser(place, &call.name, &call.args)) {
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running));
|
2026-11-04 09:46:43 +00:00
|
|
|
|
// 裝置連結: the AnyBot browser of another computer (its own logins).
|
2026-11-05 08:47:28 +00:00
|
|
|
|
let (output, ok, img) = match place {
|
2026-11-05 07:00:31 +00:00
|
|
|
|
BrowserAt::Device(host) => match crate::remote::run_on_device(&host, &call.name, &call.args, &job.via_name) {
|
2026-11-04 09:46:43 +00:00
|
|
|
|
Ok(v) => (v["text"].as_str().unwrap_or("").to_string(), v["ok"].as_bool().unwrap_or(false), v["image"].as_str().map(String::from)),
|
|
|
|
|
|
Err(e) => (e, false, None),
|
|
|
|
|
|
},
|
2026-11-05 07:00:31 +00:00
|
|
|
|
BrowserAt::Box => crate::botbrowser::in_box(|| web::run_full(&call.name, &call.args, &cancel)),
|
|
|
|
|
|
BrowserAt::Here => web::run_full(&call.name, &call.args, &cancel),
|
2026-11-04 09:46:43 +00:00
|
|
|
|
};
|
2026-11-05 08:47:28 +00:00
|
|
|
|
// Which computer's browser it was, like every other step.
|
|
|
|
|
|
let output = if on.is_empty() { output } else { format!("[{on}] {output}") };
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed }));
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let mut result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output)));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
result.images.extend(img);
|
|
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
let is_web = web::is_web_tool(&call.name);
|
|
|
|
|
|
let is_screen = computer::is_computer_tool(&call.name);
|
|
|
|
|
|
let is_mac = crate::macapps::is_mac_tool(&call.name);
|
|
|
|
|
|
let what = if is_mac {
|
|
|
|
|
|
format!("{} {}", crate::macapps::label(&call.name), crate::macapps::describe(&call.name, &call.args))
|
|
|
|
|
|
} else if is_screen {
|
2026-11-05 10:25:46 +00:00
|
|
|
|
// A control by its number: name it from the app's list the bot last read.
|
|
|
|
|
|
let named = tools::args_of(&call.args)["ref"].as_str().map(str::trim).filter(|r| !r.is_empty()).and_then(|r| element_named(&history, r)).map(|e| format!("({e})")).unwrap_or_default();
|
|
|
|
|
|
format!("{} {}{named}", computer::label(&call.name), computer::describe(&call.name, &call.args))
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
} else if is_web {
|
|
|
|
|
|
web::describe(&call.name, &call.args)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
tools::describe(&call.name, &call.args)
|
|
|
|
|
|
};
|
|
|
|
|
|
// Ask the person right away (a pop-up), and wait for the answer.
|
|
|
|
|
|
let ask_person = |kind: ApprovalKind, reason: String, grant: String| -> ApprovalAnswer {
|
|
|
|
|
|
let (atx, arx) = channel();
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Waiting));
|
|
|
|
|
|
send(StreamEvent::NeedApproval(Approval { call_id: call.id.clone(), tool: call.name.clone(), reason, grant, reply: atx, kind }));
|
|
|
|
|
|
wait_answer(arx, &cancel)
|
|
|
|
|
|
};
|
2026-11-05 09:29:55 +00:00
|
|
|
|
// 需要處理 (the original's box hand-off): a step only the person can do — signing
|
|
|
|
|
|
// in, a captcha, a code from their phone. A card with the computer's live screen
|
|
|
|
|
|
// appears in the chat; they take over, do it, and say 「我完成了」 or 「跳過」. A
|
|
|
|
|
|
// message typed meanwhile answers it too.
|
|
|
|
|
|
if call.name == tools::HANDOFF_TOOL {
|
|
|
|
|
|
let instruction = tools::args_of(&call.args)["instruction"].as_str().unwrap_or("").trim().to_string();
|
|
|
|
|
|
let (output, status) = match screen_target(&job, &call.args) {
|
|
|
|
|
|
Err(e) => (e, ToolStatus::Failed),
|
|
|
|
|
|
Ok(_) if instruction.is_empty() => ("請在 instruction 寫清楚要使用者做什麼。".to_string(), ToolStatus::Failed),
|
|
|
|
|
|
Ok(t) => {
|
|
|
|
|
|
let (atx, arx) = channel();
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Waiting));
|
|
|
|
|
|
send(StreamEvent::NeedApproval(Approval { call_id: call.id.clone(), tool: call.name.clone(), reason: instruction.clone(), grant: t.name.clone(), reply: atx, kind: ApprovalKind::Handoff }));
|
|
|
|
|
|
let answer = loop {
|
|
|
|
|
|
if cancel.load(Ordering::Relaxed) {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
match arx.recv_timeout(Duration::from_millis(200)) {
|
|
|
|
|
|
Ok(a) => break Some(a),
|
|
|
|
|
|
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
|
|
|
|
|
|
if !job.steer.lock().unwrap().is_empty() {
|
|
|
|
|
|
break None;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
Err(_) => break Some(ApprovalAnswer::Reject),
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
match answer {
|
|
|
|
|
|
Some(ApprovalAnswer::Reject) => ("使用者選擇跳過這一步。不要再要求同一件事:沒有它就繼續其餘的工作,做不到的照實說。".to_string(), ToolStatus::Denied),
|
|
|
|
|
|
Some(_) => (format!("[{}] 使用者按了「我完成了」。先截圖確認畫面,再接著做。", t.name), ToolStatus::Done),
|
|
|
|
|
|
None => ("使用者沒有按按鈕,而是傳了訊息(就在下面)。照他說的做。".to_string(), ToolStatus::Done),
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), status));
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output)));
|
2026-11-05 09:29:55 +00:00
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
// 生活卡片: drawn by the window from the call itself.
|
|
|
|
|
|
if call.name == "show_card" {
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Done));
|
2026-11-05 08:47:28 +00:00
|
|
|
|
let result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, "卡片已顯示給使用者。"));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
// 表情回應: an emoji on the person's latest message (no words needed).
|
|
|
|
|
|
if call.name == "react" {
|
|
|
|
|
|
let emoji: String = tools::args_of(&call.args)["emoji"].as_str().unwrap_or("👍").chars().take(4).collect();
|
|
|
|
|
|
send(StreamEvent::React(emoji.clone()));
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Done));
|
2026-11-05 08:47:28 +00:00
|
|
|
|
let result = Message::tool_result(&call.id, &call.name, ¬ed(&moved, &call.id, &format!("已在使用者的訊息上按了 {emoji}。")));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
// 傳檔案: put a file the bot made into the chat as a card the person can open.
|
|
|
|
|
|
if call.name == "send_file" {
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running));
|
|
|
|
|
|
let path = tools::args_of(&call.args)["path"].as_str().unwrap_or("").to_string();
|
|
|
|
|
|
let local = match target_for(&job, &call.args) {
|
2026-11-05 07:48:20 +00:00
|
|
|
|
// Files ride to the chat only from this bot's own folder on the box.
|
|
|
|
|
|
Ok(tools::Target { place: tools::Place::Box { dir }, .. }) => match tools::box_path(std::path::Path::new(&dir), &path) {
|
|
|
|
|
|
Ok(p) => crate::cloudbox::copy_out(&p),
|
|
|
|
|
|
Err(e) => Err(e),
|
|
|
|
|
|
},
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
Ok(tools::Target { place: tools::Place::Sealed { root }, .. }) => Ok(tools::resolve(&root, &path)),
|
|
|
|
|
|
Ok(tools::Target { place: tools::Place::Local { workdir }, .. }) => Ok(tools::resolve(&workdir, &path)),
|
2026-11-05 08:47:28 +00:00
|
|
|
|
Ok(_) => Err(format!("這台電腦上的檔案不能直接傳,請先複製到「{}」。", job.roles.own)),
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
Err(e) => Err(e),
|
|
|
|
|
|
};
|
2026-11-05 07:48:20 +00:00
|
|
|
|
// Secrets (SSH keys, the app's own data folder, .env…) never ride into the chat,
|
|
|
|
|
|
// same rule as read_file/edit_file.
|
|
|
|
|
|
let local = local.and_then(|p| if tools::is_ignored(&p) || tools::is_ignored(std::path::Path::new(&path)) { Err("這個檔案是機密的,不能傳到聊天。".to_string()) } else { Ok(p) });
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
let (output, ok) = match local.and_then(|p| crate::attach::store(bot_id, &p)) {
|
|
|
|
|
|
Ok(a) => {
|
|
|
|
|
|
let out = format!("已把「{}」傳到聊天裡({}),使用者點一下就能預覽。", a.name, crate::attach::human(a.bytes));
|
|
|
|
|
|
send(StreamEvent::Attach(a));
|
|
|
|
|
|
(out, true)
|
|
|
|
|
|
}
|
|
|
|
|
|
Err(e) => (e, false),
|
|
|
|
|
|
};
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed }));
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output)));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
// 檔案預覽 / Bot 做的頁面: open what the bot made on the person's screen.
|
|
|
|
|
|
if call.name == "open_for_person" {
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running));
|
|
|
|
|
|
let v = tools::args_of(&call.args);
|
|
|
|
|
|
let path = v["path"].as_str().unwrap_or("").to_string();
|
|
|
|
|
|
let on_box = match target_for(&job, &call.args) {
|
|
|
|
|
|
Ok(t) => match t.place {
|
|
|
|
|
|
tools::Place::Box { dir } => Some(if path.starts_with('/') { path.clone() } else { format!("{dir}/{path}") }),
|
|
|
|
|
|
_ => None,
|
|
|
|
|
|
},
|
|
|
|
|
|
Err(_) => None,
|
|
|
|
|
|
};
|
2026-11-05 08:47:28 +00:00
|
|
|
|
// On a server nobody sits at its screen: a link opens in the browser of the
|
|
|
|
|
|
// computer the person is using; a file is sent into the chat instead.
|
2026-11-05 11:41:49 +00:00
|
|
|
|
let person_now = roles_now(&job).person;
|
|
|
|
|
|
let person_host = person_now.as_ref().and_then(|n| job.computers.iter().find(|t| &t.name == n)).and_then(|t| match &t.place {
|
2026-11-05 13:28:32 +00:00
|
|
|
|
tools::Place::Device { host, .. } => Some(host.clone()),
|
2026-11-05 08:47:28 +00:00
|
|
|
|
_ => None,
|
|
|
|
|
|
});
|
|
|
|
|
|
if job.roles.server {
|
|
|
|
|
|
let web = path.starts_with("http://") || path.starts_with("https://");
|
|
|
|
|
|
let (output, ok) = match (&person_host, web) {
|
|
|
|
|
|
(Some(host), true) => match crate::remote::run_on_device(host, "browser_open", &serde_json::json!({"url": path}).to_string(), &job.via_name) {
|
2026-11-05 11:41:49 +00:00
|
|
|
|
Ok(v) => (format!("[{}] {}", person_now.clone().unwrap_or_default(), v["text"].as_str().unwrap_or("")), v["ok"].as_bool().unwrap_or(false)),
|
2026-11-05 08:47:28 +00:00
|
|
|
|
Err(e) => (e, false),
|
|
|
|
|
|
},
|
|
|
|
|
|
(None, true) => ("不知道使用者現在在用哪台電腦,沒辦法打開給他看。把網址直接寫在回覆裡。".to_string(), false),
|
|
|
|
|
|
(_, false) => ("AnyBot 在伺服器上,檔案沒辦法直接在使用者的螢幕打開。改用 send_file 把檔案傳到聊天裡,使用者點一下就能看。".to_string(), false),
|
|
|
|
|
|
};
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed }));
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output)));
|
2026-11-05 08:47:28 +00:00
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
let local = match (&on_box, target_for(&job, &call.args)) {
|
|
|
|
|
|
(Some(p), _) => crate::cloudbox::copy_out(p),
|
|
|
|
|
|
(None, Ok(tools::Target { place: tools::Place::Sealed { root }, .. })) => Ok(tools::resolve(&root, &path)),
|
|
|
|
|
|
(None, Ok(tools::Target { place: tools::Place::Local { workdir }, .. })) => Ok(tools::resolve(&workdir, &path)),
|
|
|
|
|
|
(None, _) if path.starts_with("http://") || path.starts_with("https://") => Ok(std::path::PathBuf::from(&path)),
|
2026-11-05 08:47:28 +00:00
|
|
|
|
_ => Err(format!("這台電腦上的檔案不能直接打開給使用者看,請先複製到「{}」或這台電腦。", job.roles.own)),
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
};
|
2026-11-05 07:48:20 +00:00
|
|
|
|
// Secrets never open on the person's screen either.
|
|
|
|
|
|
let local = local.and_then(|p| if tools::is_ignored(&p) || tools::is_ignored(std::path::Path::new(&path)) { Err("這個檔案是機密的,不能打開。".to_string()) } else { Ok(p) });
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
let (output, ok) = match local {
|
|
|
|
|
|
Ok(p) if p.exists() || path.starts_with("http") => match crate::preview::system_open(&p).status() {
|
|
|
|
|
|
Ok(st) if st.success() => (format!("已在使用者的螢幕上打開 {}。", p.display()), true),
|
|
|
|
|
|
_ => (format!("打不開 {}", p.display()), false),
|
|
|
|
|
|
},
|
|
|
|
|
|
Ok(p) => (format!("找不到 {}", p.display()), false),
|
|
|
|
|
|
Err(e) => (e, false),
|
|
|
|
|
|
};
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), if ok { ToolStatus::Done } else { ToolStatus::Failed }));
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output)));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
// 打電話、傳簡訊、在 X 發文: speaks for the person, so asked every single time
|
|
|
|
|
|
// (like the original's send tools; no "always").
|
|
|
|
|
|
if crate::phone::is_phone_tool(&call.name) {
|
|
|
|
|
|
let what = crate::phone::describe(&call.name, &call.args);
|
|
|
|
|
|
let answer = ask_person(ApprovalKind::Action, what, String::new());
|
|
|
|
|
|
if cancel.load(Ordering::Relaxed) {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
let (output, status) = if answer == ApprovalAnswer::Reject {
|
|
|
|
|
|
("使用者沒有允許。請不要再試,改問使用者。".to_string(), ToolStatus::Denied)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running));
|
|
|
|
|
|
match crate::phone::run(&call.name, &call.args) {
|
|
|
|
|
|
(t, true) => (t, ToolStatus::Done),
|
|
|
|
|
|
(t, false) => (t, ToolStatus::Failed),
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), status));
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output)));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
// 發布網站: put a page or folder the bot made on the web (asked once per site name).
|
|
|
|
|
|
if call.name == "publish_site" {
|
|
|
|
|
|
let v = tools::args_of(&call.args);
|
|
|
|
|
|
let path = v["path"].as_str().unwrap_or("").to_string();
|
|
|
|
|
|
let name = v["name"].as_str().unwrap_or("").trim().to_lowercase();
|
|
|
|
|
|
let grant = format!("publish:{name}");
|
|
|
|
|
|
let mut allowed = grants.contains(&grant);
|
|
|
|
|
|
if !allowed {
|
|
|
|
|
|
match ask_person(ApprovalKind::Action, format!("把 {path} 發布成公開網站「{name}」(任何知道網址的人都看得到)"), grant.clone()) {
|
|
|
|
|
|
ApprovalAnswer::Once => allowed = true,
|
|
|
|
|
|
ApprovalAnswer::Always(g) => {
|
|
|
|
|
|
if !g.is_empty() && !grants.contains(&g) {
|
|
|
|
|
|
grants.push(g);
|
|
|
|
|
|
}
|
|
|
|
|
|
allowed = true;
|
|
|
|
|
|
}
|
|
|
|
|
|
ApprovalAnswer::Reject => {}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
if cancel.load(Ordering::Relaxed) {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
let (output, status) = if !allowed {
|
|
|
|
|
|
("使用者沒有允許發布。".to_string(), ToolStatus::Denied)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running));
|
|
|
|
|
|
let local = match target_for(&job, &call.args) {
|
|
|
|
|
|
Ok(tools::Target { place: tools::Place::Box { dir }, .. }) => crate::cloudbox::copy_out(&if path.starts_with('/') { path.clone() } else { format!("{dir}/{path}") }),
|
|
|
|
|
|
Ok(tools::Target { place: tools::Place::Sealed { root }, .. }) => Ok(tools::resolve(&root, &path)),
|
|
|
|
|
|
Ok(tools::Target { place: tools::Place::Local { workdir }, .. }) => Ok(tools::resolve(&workdir, &path)),
|
2026-11-05 08:47:28 +00:00
|
|
|
|
Ok(_) => Err(format!("這台電腦上的檔案不能直接發布,請先複製到「{}」。", job.roles.own)),
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
Err(e) => Err(e),
|
|
|
|
|
|
};
|
|
|
|
|
|
match local.and_then(|p| crate::remote::publish_site(&name, &p)) {
|
|
|
|
|
|
Ok(url) => (format!("已發布:{url}"), ToolStatus::Done),
|
|
|
|
|
|
Err(e) => (e, ToolStatus::Failed),
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), status));
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output)));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
// 保險庫: the bot sees only metadata; filling asks the person first. A login on a
|
|
|
|
|
|
// site it wasn't saved for warns about look-alike sites and can't be "always";
|
|
|
|
|
|
// a card is asked every time (like the original's 允許一次 / 拒絕).
|
|
|
|
|
|
if call.name == crate::vault::LIST_TOOL {
|
|
|
|
|
|
let output = crate::vault::list_for_bot(&call.args);
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Done));
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output)));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
if call.name == crate::vault::TOOL || call.name == crate::vault::CARD_TOOL || call.name == crate::vault::CONTACT_TOOL {
|
|
|
|
|
|
enum Plan {
|
|
|
|
|
|
Login(crate::vault::LoginPlan),
|
|
|
|
|
|
Card,
|
|
|
|
|
|
Contact,
|
|
|
|
|
|
}
|
|
|
|
|
|
let planned: Result<(Plan, String, String), String> = if call.name == crate::vault::TOOL {
|
|
|
|
|
|
crate::vault::plan_login(&call.args).map(|pl| {
|
|
|
|
|
|
let q = pl.question();
|
|
|
|
|
|
let grant = if pl.matched { format!("login:{}", pl.host) } else { String::new() };
|
|
|
|
|
|
(Plan::Login(pl), q, grant)
|
|
|
|
|
|
})
|
|
|
|
|
|
} else if call.name == crate::vault::CARD_TOOL {
|
|
|
|
|
|
crate::vault::plan_card(&call.args).map(|(host, c)| (Plan::Card, format!("用保險庫的卡片({} •••• {})在 {host} 填入付款資料(Bot 看不到卡號和安全碼)", c.display_title(), c.last4), String::new()))
|
|
|
|
|
|
} else {
|
|
|
|
|
|
crate::vault::plan_contact(&call.args).map(|(host, c)| (Plan::Contact, format!("把保險庫的聯絡資訊「{}」填進 {host} 的表單", c.display_title()), format!("contact:{host}")))
|
|
|
|
|
|
};
|
|
|
|
|
|
let (output, status) = match planned {
|
|
|
|
|
|
Err(e) => (e, ToolStatus::Failed),
|
|
|
|
|
|
Ok((plan, question, grant)) => {
|
|
|
|
|
|
let mut allowed = !grant.is_empty() && grants.contains(&grant);
|
|
|
|
|
|
if !allowed {
|
|
|
|
|
|
match ask_person(ApprovalKind::Action, question, grant.clone()) {
|
|
|
|
|
|
ApprovalAnswer::Once => allowed = true,
|
|
|
|
|
|
ApprovalAnswer::Always(g) => {
|
|
|
|
|
|
if !g.is_empty() && !grants.contains(&g) {
|
|
|
|
|
|
grants.push(g);
|
|
|
|
|
|
}
|
|
|
|
|
|
allowed = true;
|
|
|
|
|
|
}
|
|
|
|
|
|
ApprovalAnswer::Reject => {}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
if cancel.load(Ordering::Relaxed) {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
if !allowed {
|
|
|
|
|
|
("使用者沒有允許使用保險庫的這個項目。請改問使用者。".to_string(), ToolStatus::Denied)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running));
|
|
|
|
|
|
let r = match plan {
|
|
|
|
|
|
Plan::Login(pl) => crate::vault::fill_login(&pl, tools::args_of(&call.args)["submit"].as_bool().unwrap_or(false)),
|
|
|
|
|
|
Plan::Card => crate::vault::fill_card(&call.args),
|
|
|
|
|
|
Plan::Contact => crate::vault::fill_contact(&call.args),
|
|
|
|
|
|
};
|
|
|
|
|
|
match r {
|
|
|
|
|
|
Ok(t) => (t, ToolStatus::Done),
|
|
|
|
|
|
Err(e) => (e, ToolStatus::Failed),
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), status));
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output)));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
// MCP connectors: asked once per server (or always allowed), like the original's
|
|
|
|
|
|
// MCP allow-list; direct mode doesn't ask.
|
|
|
|
|
|
if crate::mcp::is_mcp_tool(&call.name) {
|
|
|
|
|
|
let (server, tool) = crate::mcp::resolve(&call.name).unwrap_or_default();
|
|
|
|
|
|
let grant = format!("mcp:{server}");
|
|
|
|
|
|
let args_short: String = call.args.chars().take(160).collect();
|
|
|
|
|
|
let mut allowed = job.direct.load(Ordering::Relaxed) || grants.contains(&grant);
|
|
|
|
|
|
if !allowed {
|
|
|
|
|
|
match ask_person(ApprovalKind::Action, format!("使用連接器「{server}」的工具:{tool} {args_short}"), grant.clone()) {
|
|
|
|
|
|
ApprovalAnswer::Once => allowed = true,
|
|
|
|
|
|
ApprovalAnswer::Always(g) => {
|
|
|
|
|
|
if !g.is_empty() && !grants.contains(&g) {
|
|
|
|
|
|
grants.push(g);
|
|
|
|
|
|
}
|
|
|
|
|
|
allowed = true;
|
|
|
|
|
|
}
|
|
|
|
|
|
ApprovalAnswer::Reject => {}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
if cancel.load(Ordering::Relaxed) {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
let (output, status, img) = if allowed {
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running));
|
|
|
|
|
|
let (o, ok, img) = crate::mcp::call(&call.name, &call.args);
|
|
|
|
|
|
(o, if ok { ToolStatus::Done } else { ToolStatus::Failed }, img)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
("使用者拒絕了這個操作。請不要再試同樣的操作,改問使用者想怎麼做。".to_string(), ToolStatus::Denied, None)
|
|
|
|
|
|
};
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), status));
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let mut result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output)));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
result.images.extend(img);
|
|
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
2026-11-05 08:47:28 +00:00
|
|
|
|
// The person's own apps and everyday browser are on the computer they are using:
|
|
|
|
|
|
// the one they name, else the one they typed on (else this one, when it is theirs).
|
|
|
|
|
|
let person_target = || -> Result<tools::Target, String> {
|
|
|
|
|
|
if names_a_computer(&call.args) {
|
|
|
|
|
|
return target_for(&job, &call.args);
|
|
|
|
|
|
}
|
2026-11-05 11:41:49 +00:00
|
|
|
|
roles_now(&job)
|
2026-11-05 08:47:28 +00:00
|
|
|
|
.person
|
2026-11-05 11:41:49 +00:00
|
|
|
|
.and_then(|n| job.computers.iter().find(|t| t.name == n))
|
2026-11-05 08:47:28 +00:00
|
|
|
|
.cloned()
|
|
|
|
|
|
.ok_or_else(|| format!("不知道使用者現在在用哪台電腦。請問他要在哪台做(可以用:{})。", job.computers.iter().map(|t| t.name.as_str()).collect::<Vec<_>>().join("、")))
|
|
|
|
|
|
};
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
let target = if is_screen {
|
|
|
|
|
|
screen_target(&job, &call.args)
|
2026-11-05 08:47:28 +00:00
|
|
|
|
} else if is_mac || is_web {
|
|
|
|
|
|
// (Other browser steps went their own way above; this is the person's own Chrome.)
|
|
|
|
|
|
person_target()
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
} else {
|
|
|
|
|
|
target_for(&job, &call.args)
|
|
|
|
|
|
};
|
|
|
|
|
|
let on_box = matches!(target.as_ref().map(|t| &t.place), Ok(tools::Place::Box { .. }));
|
|
|
|
|
|
let on_device = match target.as_ref().map(|t| &t.place) {
|
2026-11-05 13:28:32 +00:00
|
|
|
|
Ok(tools::Place::Device { host, .. }) => Some(host.clone()),
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
_ => None,
|
|
|
|
|
|
};
|
|
|
|
|
|
// Permission to use computers at all, and this Mac in particular, is asked for
|
|
|
|
|
|
// when it's needed instead of the step failing quietly.
|
|
|
|
|
|
let mut gate: Result<(), String> = Ok(());
|
2026-11-04 23:56:19 +00:00
|
|
|
|
// The person just said yes to this very step in a switch's pop-up (it shows the
|
|
|
|
|
|
// step): no second pop-up for it.
|
|
|
|
|
|
let mut said_yes = false;
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
// 直接操作: like Grok Bot after its one-time setup ("讓 Grok Bot 在你的電腦上開啟檔案和
|
|
|
|
|
|
// 執行任務。自動審核仍會先檢查所有操作", ENHi8c), nothing is asked except what the
|
|
|
|
|
|
// safety checks and the person's "一律詢問" rules stop.
|
|
|
|
|
|
let direct = job.direct.load(Ordering::Relaxed);
|
|
|
|
|
|
if direct {
|
|
|
|
|
|
tools_on = true;
|
|
|
|
|
|
local_on = true;
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Ok(t) = &target {
|
|
|
|
|
|
if !tools_on {
|
|
|
|
|
|
let reason = format!("讓 {} 使用電腦工具:{}\n(在 {} 上)", job.via_name, what, t.name);
|
|
|
|
|
|
match ask_person(ApprovalKind::EnableTools, reason, "tools:on".into()) {
|
2026-11-04 23:56:19 +00:00
|
|
|
|
ApprovalAnswer::Once => said_yes = true,
|
|
|
|
|
|
ApprovalAnswer::Always(_) => {
|
|
|
|
|
|
tools_on = true;
|
|
|
|
|
|
said_yes = true;
|
|
|
|
|
|
}
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
ApprovalAnswer::Reject => gate = Err("使用者沒有允許使用電腦工具。請改用文字回答,或問使用者想怎麼做。".into()),
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
// An admin lock (policy) refuses outright instead of asking.
|
|
|
|
|
|
if gate.is_ok() && matches!(t.place, tools::Place::Local { .. }) && job.local_locked {
|
2026-11-05 08:47:28 +00:00
|
|
|
|
gate = Err(format!("「在這台電腦上執行」設成了永不允許(或被管理員關閉),不能在「{}」上執行。請改用「{}」。", t.name, job.roles.own));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
}
|
|
|
|
|
|
if gate.is_ok() && matches!(t.place, tools::Place::Local { .. }) && !local_on {
|
2026-11-05 08:47:28 +00:00
|
|
|
|
let reason = format!("在「{}」上執行任務,而不只是在它自己的電腦上:{}{what}", t.name, if is_web { format!("{} ", web::label(&call.name)) } else { String::new() });
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
match ask_person(ApprovalKind::EnableThisComputer, reason, "local:on".into()) {
|
2026-11-04 23:56:19 +00:00
|
|
|
|
ApprovalAnswer::Once => said_yes = true,
|
|
|
|
|
|
ApprovalAnswer::Always(_) => {
|
|
|
|
|
|
local_on = true;
|
|
|
|
|
|
said_yes = true;
|
|
|
|
|
|
}
|
2026-11-05 08:47:28 +00:00
|
|
|
|
ApprovalAnswer::Reject => gate = Err(format!("使用者沒有允許在「{}」上執行。可以改用「{}」,或問使用者。", t.name, job.roles.own)),
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
if cancel.load(Ordering::Relaxed) {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
let decision = match (&target, &gate) {
|
|
|
|
|
|
// Looking at the screen and using the mouse/keyboard each ask once (or always).
|
|
|
|
|
|
// The Mac's own apps: reading asks once per app, writing once per app (or each
|
|
|
|
|
|
// time), sending every single time (the original never keeps a grant for it).
|
|
|
|
|
|
(Ok(_), Ok(())) if is_mac => {
|
|
|
|
|
|
let app = crate::macapps::app_of(&call.name);
|
|
|
|
|
|
if crate::macapps::is_send(&call.name) && call.name == crate::macapps::SEND_MESSAGE && grants.iter().any(|g| g == "messages:no-ask") {
|
|
|
|
|
|
Decision::Allow
|
|
|
|
|
|
} else if crate::macapps::is_send(&call.name) {
|
|
|
|
|
|
Decision::Ask { reason: format!("用你的「{app}」{what}"), grant: String::new() }
|
|
|
|
|
|
} else if crate::macapps::is_write(&call.name) {
|
|
|
|
|
|
let grant = format!("macapp-write:{app}");
|
|
|
|
|
|
if grants.contains(&grant) { Decision::Allow } else { Decision::Ask { reason: format!("讓 {} 修改你的「{app}」:{what}", job.via_name), grant } }
|
|
|
|
|
|
} else {
|
|
|
|
|
|
let grant = format!("macapp:{app}");
|
|
|
|
|
|
if grants.contains(&grant) || grants.contains(&format!("macapp-write:{app}")) { Decision::Allow } else { Decision::Ask { reason: format!("讓 {} 讀取你的「{app}」:{what}", job.via_name), grant } }
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
// Its own desktop: nothing to ask, like the original's box.
|
|
|
|
|
|
(Ok(_), Ok(())) if is_screen && on_box => Decision::Allow,
|
|
|
|
|
|
(Ok(_), Ok(())) if is_screen && call.name == "computer_use_check_permissions" => Decision::Allow,
|
|
|
|
|
|
// 裝置連結: another of the person's computers; seeing and operating it ask once each.
|
|
|
|
|
|
(Ok(t), Ok(())) if is_screen && on_device.is_some() => {
|
|
|
|
|
|
let grant = if call.name == "computer_use_screenshot" { format!("device-see:{}", t.name) } else { format!("device-control:{}", t.name) };
|
|
|
|
|
|
if grants.contains(&grant) {
|
|
|
|
|
|
Decision::Allow
|
|
|
|
|
|
} else {
|
|
|
|
|
|
Decision::Ask { reason: format!("{}「{}」的螢幕:{what}", if call.name == "computer_use_screenshot" { "看" } else { "操作" }, t.name), grant }
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
(Ok(_), Ok(())) if is_screen => {
|
|
|
|
|
|
if call.name == "computer_use_screenshot" {
|
|
|
|
|
|
if grants.iter().any(|g| g == "computer:see") {
|
|
|
|
|
|
Decision::Allow
|
|
|
|
|
|
} else {
|
|
|
|
|
|
Decision::Ask { reason: format!("看你的螢幕畫面:{what}"), grant: "computer:see".into() }
|
|
|
|
|
|
}
|
|
|
|
|
|
} else {
|
|
|
|
|
|
// Per app, like Grok Bot: 「Grok Bot 可以看到 {app},並在其中點擊和輸入」.
|
|
|
|
|
|
let app = computer::target_app(&call.name, &call.args).unwrap_or_else(|| "目前的 App".into());
|
|
|
|
|
|
let grant = format!("app:{app}");
|
|
|
|
|
|
if grants.iter().any(|g| g == "computer:control" || *g == grant) {
|
|
|
|
|
|
Decision::Allow
|
|
|
|
|
|
} else {
|
|
|
|
|
|
Decision::Ask { reason: format!("讓 {} 看到「{app}」,並在其中點擊和輸入:{what}", job.via_name), grant }
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
// Opening a link is visible and harmless; reading a logged-in tab asks first.
|
|
|
|
|
|
(Ok(_), Ok(())) if is_web => {
|
|
|
|
|
|
if call.name == "browser_read" && web::read_needs_permission(&call.args) && !grants.iter().any(|g| g == "browser:read") {
|
|
|
|
|
|
Decision::Ask { reason: "讀取你瀏覽器目前分頁的內容(包含已登入的頁面):目前的分頁".into(), grant: "browser:read".into() }
|
|
|
|
|
|
} else {
|
|
|
|
|
|
Decision::Allow
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
(Ok(t), Ok(())) => tools::decide_on(&call.name, &call.args, t, &grants),
|
|
|
|
|
|
(Err(e), _) => Decision::Deny(e.clone()),
|
|
|
|
|
|
// The person said no to the pop-up: nothing more to ask.
|
|
|
|
|
|
(_, Err(_)) => Decision::Allow,
|
|
|
|
|
|
};
|
|
|
|
|
|
// 自動審核: when the rules say yes, continue without asking (JOecNC);
|
|
|
|
|
|
// when an "always ask" rule matches, stop and ask (Pca/iV).
|
|
|
|
|
|
let mut review_note = String::new();
|
|
|
|
|
|
let direct = job.direct.load(Ordering::Relaxed);
|
|
|
|
|
|
let decision = match (decision, &job.review) {
|
|
|
|
|
|
// Direct mode: go ahead, unless an "always ask" rule matches.
|
|
|
|
|
|
(Decision::Ask { reason, grant }, review) if direct => {
|
|
|
|
|
|
let verdict = match review {
|
|
|
|
|
|
Some(rv) if !rv.ask.is_empty() => review_action(&job, &Review { allow: vec![], ask: rv.ask.clone() }, &reason, &cancel),
|
|
|
|
|
|
_ => ReviewVerdict::NoMatch,
|
|
|
|
|
|
};
|
|
|
|
|
|
match verdict {
|
|
|
|
|
|
ReviewVerdict::Ask(rule) => Decision::Ask { reason: format!("{reason}(符合你的「一律詢問」規則:{rule})"), grant: String::new() },
|
|
|
|
|
|
_ => {
|
|
|
|
|
|
let _ = grant;
|
|
|
|
|
|
review_note = "(直接操作模式:已自動允許)\n".into();
|
|
|
|
|
|
Decision::Allow
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
(Decision::Ask { reason, grant }, Some(rv)) => match review_action(&job, rv, &reason, &cancel) {
|
|
|
|
|
|
ReviewVerdict::Allow(rule) => {
|
|
|
|
|
|
review_note = format!("(自動審核:符合「{rule}」,直接允許)\n");
|
|
|
|
|
|
Decision::Allow
|
|
|
|
|
|
}
|
|
|
|
|
|
ReviewVerdict::Ask(rule) => Decision::Ask { reason: format!("{reason}(符合你的「一律詢問」規則:{rule})"), grant: String::new() },
|
|
|
|
|
|
ReviewVerdict::NoMatch => Decision::Ask { reason, grant },
|
|
|
|
|
|
},
|
|
|
|
|
|
(d, _) => d,
|
|
|
|
|
|
};
|
|
|
|
|
|
// Sending mail or messages is approved one by one, even in direct mode.
|
|
|
|
|
|
let decision = if is_mac && crate::macapps::is_send(&call.name) && gate.is_ok() && target.is_ok() {
|
|
|
|
|
|
review_note.clear();
|
|
|
|
|
|
Decision::Ask { reason: format!("用你的「{}」{what}(每一封都會先問你)", crate::macapps::app_of(&call.name)), grant: String::new() }
|
|
|
|
|
|
} else {
|
|
|
|
|
|
decision
|
|
|
|
|
|
};
|
|
|
|
|
|
let approved = match decision {
|
|
|
|
|
|
_ if gate.is_err() => Err((gate.clone().unwrap_err(), ToolStatus::Denied)),
|
|
|
|
|
|
Decision::Allow => Ok(()),
|
2026-11-04 23:56:19 +00:00
|
|
|
|
// Sending (Messages, mail…) still asks every time, even right after a switch.
|
|
|
|
|
|
Decision::Ask { .. } if said_yes && !(is_mac && crate::macapps::is_send(&call.name)) => Ok(()),
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
Decision::Deny(r) => Err((format!("被安全規則擋下:{r}"), ToolStatus::Denied)),
|
|
|
|
|
|
Decision::Ask { reason, grant } => {
|
|
|
|
|
|
let machine = target.as_ref().map(|t| t.name.clone()).unwrap_or_default();
|
|
|
|
|
|
let reason = format!("{reason}\n(在 {machine} 上)");
|
|
|
|
|
|
match ask_person(ApprovalKind::Action, reason, grant) {
|
|
|
|
|
|
ApprovalAnswer::Once => Ok(()),
|
|
|
|
|
|
ApprovalAnswer::Always(g) => {
|
|
|
|
|
|
if !g.is_empty() && !grants.contains(&g) {
|
|
|
|
|
|
grants.push(g);
|
|
|
|
|
|
}
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
}
|
|
|
|
|
|
ApprovalAnswer::Reject => Err(("使用者拒絕了這個操作。請不要再試同樣的操作,改問使用者想怎麼做。".into(), ToolStatus::Denied)),
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
if cancel.load(Ordering::Relaxed) {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
let mut image = None;
|
|
|
|
|
|
let (output, status) = match (approved, &target) {
|
|
|
|
|
|
(Ok(()), Ok(t)) => {
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Running));
|
|
|
|
|
|
let ids = tools::RunIds { conversation: bot_id.to_string(), request: call.id.clone() };
|
2026-11-05 08:47:28 +00:00
|
|
|
|
let (out, ok) = if is_mac && on_device.is_none() {
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
crate::macapps::run(&call.name, &call.args)
|
2026-11-05 08:47:28 +00:00
|
|
|
|
} else if let (true, Some(host)) = (is_screen || is_web || is_mac, on_device.as_ref()) {
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
match crate::remote::run_on_device(host, &call.name, &call.args, &job.via_name) {
|
|
|
|
|
|
Ok(v) => {
|
|
|
|
|
|
image = v["image"].as_str().map(String::from);
|
|
|
|
|
|
(v["text"].as_str().unwrap_or("").to_string(), v["ok"].as_bool().unwrap_or(false))
|
|
|
|
|
|
}
|
|
|
|
|
|
Err(e) => (e, false),
|
|
|
|
|
|
}
|
|
|
|
|
|
} else if is_screen && on_box {
|
|
|
|
|
|
match crate::cloudbox::computer_run(&call.name, &tools::args_of(&call.args)) {
|
|
|
|
|
|
Ok((out, img)) => {
|
|
|
|
|
|
image = img;
|
|
|
|
|
|
(out, true)
|
|
|
|
|
|
}
|
|
|
|
|
|
Err(e) => (e, false),
|
|
|
|
|
|
}
|
|
|
|
|
|
} else if is_screen {
|
|
|
|
|
|
let (out, ok, img) = computer::run_by(&call.name, &call.args, &job.via_name);
|
|
|
|
|
|
image = img;
|
|
|
|
|
|
(out, ok)
|
|
|
|
|
|
} else if is_web {
|
|
|
|
|
|
web::run(&call.name, &call.args, &cancel)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
tools::run_on(&call.name, &call.args, t, &cancel, &ids)
|
|
|
|
|
|
};
|
|
|
|
|
|
(format!("{review_note}[{}] {out}", t.name), if ok { ToolStatus::Done } else { ToolStatus::Failed })
|
|
|
|
|
|
}
|
|
|
|
|
|
(Ok(()), Err(e)) => (e.clone(), ToolStatus::Failed),
|
|
|
|
|
|
(Err(x), _) => x,
|
|
|
|
|
|
};
|
|
|
|
|
|
send(StreamEvent::ToolStatus(call.id.clone(), status));
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let mut result = Message::tool_result(&call.id, &call.name, &format!("{repeat_note}{}", noted(&moved, &call.id, &output)));
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
result.images.extend(image);
|
|
|
|
|
|
send(StreamEvent::ToolResult(result.clone()));
|
|
|
|
|
|
history.push(result);
|
|
|
|
|
|
if cancel.load(Ordering::Relaxed) {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
send(StreamEvent::NextStep(job.via.clone()));
|
|
|
|
|
|
send(StreamEvent::StepDone(Message::assistant(&format!("(這次已經用了 {MAX_STEPS} 個步驟,先停在這裡。要繼續的話,回覆「繼續」。)"), &job.via)));
|
|
|
|
|
|
send(StreamEvent::Done);
|
|
|
|
|
|
});
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
|
|
mod tests {
|
|
|
|
|
|
use super::*;
|
|
|
|
|
|
use crate::model::ToolCall;
|
|
|
|
|
|
use crate::providers::ProviderKind;
|
|
|
|
|
|
|
2026-11-05 08:47:28 +00:00
|
|
|
|
|
|
|
|
|
|
/// A job with nothing in it, for tests that only need a few fields.
|
|
|
|
|
|
fn test_job() -> Job {
|
|
|
|
|
|
Job {
|
|
|
|
|
|
req: ChatRequest { kind: ProviderKind::Custom, base_url: String::new(), api_key: String::new(), model: String::new(), system: String::new(), history: vec![], tools: vec![], session: String::new() },
|
|
|
|
|
|
computers: vec![],
|
|
|
|
|
|
default_computer: String::new(),
|
|
|
|
|
|
roles: tools::Roles::default(),
|
|
|
|
|
|
local_execution: false,
|
|
|
|
|
|
grants: vec![],
|
|
|
|
|
|
context_tokens: DEFAULT_CONTEXT_TOKENS,
|
|
|
|
|
|
via: String::new(),
|
|
|
|
|
|
via_name: String::new(),
|
|
|
|
|
|
tools_enabled: true,
|
|
|
|
|
|
direct: Arc::new(AtomicBool::new(false)),
|
|
|
|
|
|
local_locked: false,
|
|
|
|
|
|
steer: Arc::new(Mutex::new(vec![])),
|
|
|
|
|
|
review: None,
|
|
|
|
|
|
demos: vec![],
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-05 07:48:20 +00:00
|
|
|
|
#[test]
|
|
|
|
|
|
fn computer_words_need_word_boundaries() {
|
|
|
|
|
|
// "mac" as its own word names this computer; inside longer words (macOS, a device
|
|
|
|
|
|
// named Ann inside "anniversary") it must not — that is what made the guard fire on
|
|
|
|
|
|
// text the person never aimed at any computer.
|
|
|
|
|
|
assert!(finds("用我的 mac 執行", "mac"));
|
|
|
|
|
|
assert!(finds("mac 上開檔案", "mac"));
|
|
|
|
|
|
assert!(!finds("在 macOS 上開檔案", "mac"));
|
|
|
|
|
|
assert!(!finds("telegram 傳給我", "mac"));
|
|
|
|
|
|
assert!(finds("我的電腦好慢", "我的電腦"));
|
|
|
|
|
|
assert!(!finds("你的虛擬電腦很吵", "bot 的電腦"));
|
|
|
|
|
|
assert!(finds("在 bot 的電腦跑", "bot 的電腦"));
|
2026-11-05 08:47:28 +00:00
|
|
|
|
let roles = tools::Roles { own: "Master 的電腦".into(), person: Some("王小明的Mac mini".into()), server: true };
|
2026-11-05 13:28:32 +00:00
|
|
|
|
let mac = tools::Target { name: "王小明的Mac mini".into(), place: tools::Place::Device { host: "m".into(), dir: String::new() } };
|
2026-11-05 08:47:28 +00:00
|
|
|
|
assert!(mentions("幫我在 Mac 截圖", &mac, &roles));
|
|
|
|
|
|
assert!(!mentions("在 macOS 的登入機制下,幫 Bot 的電腦截圖", &mac, &roles));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-06 08:51:25 +00:00
|
|
|
|
#[test]
|
|
|
|
|
|
fn long_turns_check_in_instead_of_stopping() {
|
|
|
|
|
|
let mut h = vec![Message::user("修好它"), Message::tool_result("t1", "shell", "ok")];
|
|
|
|
|
|
check_in(&mut h, CHECK_IN_STEPS - 1);
|
|
|
|
|
|
assert_eq!(h[1].text, "ok");
|
|
|
|
|
|
check_in(&mut h, CHECK_IN_STEPS);
|
|
|
|
|
|
assert!(h[1].text.starts_with("ok") && h[1].text.contains(&format!("已經做了 {CHECK_IN_STEPS} 步")));
|
|
|
|
|
|
// Only a tool result carries it (a steer the person typed is left as they wrote it).
|
|
|
|
|
|
let mut h = vec![Message::user("改用別台")];
|
|
|
|
|
|
check_in(&mut h, CHECK_IN_STEPS * 2);
|
|
|
|
|
|
assert_eq!(h[0].text, "改用別台");
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn the_boxs_browser_is_always_the_bots_own() {
|
|
|
|
|
|
// A cloud brain has no Chrome of its own; browser steps for the box still go to the box.
|
|
|
|
|
|
for args in [r#"{}"#, r#"{"url":"https://www.threads.com"}"#, r#"{"profile":"你的瀏覽器"}"#] {
|
|
|
|
|
|
assert!(bots_browser(&BrowserAt::Box, "browser_open", args));
|
|
|
|
|
|
assert!(bots_browser(&BrowserAt::Box, "browser_read", args));
|
|
|
|
|
|
assert!(crate::botbrowser::in_box(|| !web::uses_person_browser(args)));
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// A made-up model on a local port that answers each request with the next reply: a tool
|
|
|
|
|
|
/// call (name, JSON arguments) or, for an empty name, words.
|
|
|
|
|
|
fn fake_model(replies: Vec<(&'static str, serde_json::Value)>) -> u16 {
|
|
|
|
|
|
use std::io::{Read, Write};
|
|
|
|
|
|
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
|
|
|
|
|
let port = l.local_addr().unwrap().port();
|
|
|
|
|
|
std::thread::spawn(move || {
|
|
|
|
|
|
for (i, (name, v)) in replies.into_iter().enumerate() {
|
|
|
|
|
|
let delta = if name.is_empty() {
|
|
|
|
|
|
serde_json::json!({"content": v.as_str().unwrap_or("")})
|
|
|
|
|
|
} else {
|
|
|
|
|
|
serde_json::json!({"tool_calls": [{"index": 0, "id": format!("c{i}"), "function": {"name": name, "arguments": v.to_string()}}]})
|
|
|
|
|
|
};
|
|
|
|
|
|
let body = format!("data: {}\n\ndata: [DONE]\n\n", serde_json::json!({"choices": [{"delta": delta}]}));
|
|
|
|
|
|
let (mut s, _) = l.accept().unwrap();
|
|
|
|
|
|
let mut req = Vec::new();
|
|
|
|
|
|
let mut buf = [0u8; 8192];
|
|
|
|
|
|
loop {
|
|
|
|
|
|
let n = s.read(&mut buf).unwrap_or(0);
|
|
|
|
|
|
if n == 0 {
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
req.extend_from_slice(&buf[..n]);
|
|
|
|
|
|
let t = String::from_utf8_lossy(&req).to_string();
|
|
|
|
|
|
if let Some(end) = t.find("\r\n\r\n") {
|
|
|
|
|
|
let len = t[..end].lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::<usize>().unwrap_or(0))).unwrap_or(0);
|
|
|
|
|
|
if req.len() >= end + 4 + len {
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n");
|
|
|
|
|
|
let _ = s.write_all(body.as_bytes());
|
|
|
|
|
|
}
|
|
|
|
|
|
});
|
|
|
|
|
|
port
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// A brain with no Chrome of its own — a cloud brain — browses on its own box: the bot's
|
|
|
|
|
|
/// browser_open and browser_read for its computer reach the box's Chromium, not the
|
|
|
|
|
|
/// person's browser on the server. Run where no Chrome-family browser is installed, against
|
|
|
|
|
|
/// a box: `ANYBOT_DATA_DIR=<scratch> cargo test --release -- --ignored
|
|
|
|
|
|
/// a_brain_without_chrome_browses_on_its_box --nocapture`.
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
#[ignore]
|
|
|
|
|
|
fn a_brain_without_chrome_browses_on_its_box() {
|
|
|
|
|
|
let own = crate::model::BOT_COMPUTER_NAME;
|
|
|
|
|
|
let page = "mkdir -p /tmp/anybot-page && printf '<meta charset=utf-8><title>box page</title><h1>盒子裡的測試頁 42</h1>' > /tmp/anybot-page/index.html && (cd /tmp/anybot-page && setsid python3 -m http.server 8766 --bind 127.0.0.1 >/dev/null 2>&1 &) ; sleep 1; echo served";
|
|
|
|
|
|
let port = fake_model(vec![
|
|
|
|
|
|
("shell", serde_json::json!({"command": page})),
|
|
|
|
|
|
("browser_open", serde_json::json!({"url": "http://127.0.0.1:8766/"})),
|
|
|
|
|
|
("browser_read", serde_json::json!({})),
|
|
|
|
|
|
("", serde_json::json!("看到了")),
|
|
|
|
|
|
]);
|
|
|
|
|
|
let job = Job {
|
|
|
|
|
|
req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("打開盒子裡的測試頁")], tools: tools::specs(), session: String::new() },
|
|
|
|
|
|
computers: vec![tools::Target { name: own.into(), place: tools::Place::Box { dir: crate::cloudbox::bot_dir(1) } }],
|
|
|
|
|
|
default_computer: own.into(),
|
|
|
|
|
|
roles: tools::Roles { own: own.into(), person: None, server: true },
|
|
|
|
|
|
local_execution: false,
|
|
|
|
|
|
direct: Arc::new(AtomicBool::new(true)),
|
|
|
|
|
|
..test_job()
|
|
|
|
|
|
};
|
|
|
|
|
|
let (tx, rx) = channel();
|
|
|
|
|
|
spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default());
|
|
|
|
|
|
let mut results = vec![];
|
|
|
|
|
|
loop {
|
|
|
|
|
|
match rx.recv_timeout(Duration::from_secs(120)).expect("loop stalled") {
|
|
|
|
|
|
(_, StreamEvent::ToolResult(m)) => {
|
|
|
|
|
|
println!("{}: {}", m.tool_name, m.text);
|
|
|
|
|
|
results.push(m.text)
|
|
|
|
|
|
}
|
|
|
|
|
|
(_, StreamEvent::Done) => break,
|
|
|
|
|
|
(_, StreamEvent::Error(e)) => panic!("{e}"),
|
|
|
|
|
|
_ => {}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
println!("this computer has a Chrome-family browser: {}", crate::botbrowser::engine().is_some());
|
|
|
|
|
|
assert!(results[0].contains("served"), "{results:?}");
|
|
|
|
|
|
assert!(results[1].starts_with(&format!("[{own}]")) && results[1].contains("它桌面上的 Chromium"), "{}", results[1]);
|
|
|
|
|
|
assert!(results[2].contains("盒子裡的測試頁 42"), "{}", results[2]);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
#[test]
|
|
|
|
|
|
fn the_same_miss_counts_as_the_same_step() {
|
|
|
|
|
|
// Clicks a few pixels apart on the same strip are one step; another computer or target isn't.
|
|
|
|
|
|
assert_eq!(step_key("computer_use_click", r#"{"x":70,"y":355,"computer":"M"}"#), step_key("computer_use_click", r#"{"x":66,"y":358,"computer":"M"}"#));
|
|
|
|
|
|
assert_ne!(step_key("computer_use_click", r#"{"x":70,"y":355}"#), step_key("computer_use_click", r#"{"x":70,"y":290}"#));
|
|
|
|
|
|
assert_ne!(step_key("computer_use_click", r#"{"ref":"3"}"#), step_key("computer_use_click", r#"{"ref":"4"}"#));
|
|
|
|
|
|
// Looking is never a repeat; other tools aren't counted.
|
|
|
|
|
|
assert_eq!(step_key("computer_use_screenshot", "{}"), None);
|
|
|
|
|
|
assert_eq!(step_key("shell", r#"{"command":"ls"}"#), None);
|
|
|
|
|
|
assert!(step_key("browser_click", r#"{"ref":"5"}"#).is_some());
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-05 10:25:46 +00:00
|
|
|
|
#[test]
|
|
|
|
|
|
fn a_click_by_number_is_named_in_the_question() {
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
let list = Message::tool_result("c1", "computer_use_screenshot", "[王小明的Mac mini] 螢幕截圖(1024×576)。\n「計算機」上可以操作的東西(圖上同樣的編號,用 ref 點):\n 文字「0」\n[4] 按鈕「全部清除」\n[7] 按鈕「7」");
|
|
|
|
|
|
assert_eq!(element_named(&[list.clone()], "7").as_deref(), Some("按鈕「7」"));
|
2026-11-05 10:25:46 +00:00
|
|
|
|
assert_eq!(element_named(&[list.clone()], "e7").as_deref(), Some("按鈕「7」"));
|
Bots click by number: every control on the screen is boxed and numbered, on every computer
A bot asked to open a Threads profile clicked an empty strip ten times: it guessed coordinates
from a picture, and the web tools' description told it they didn't work on its own computer.
- marks.rs: the picture a bot gets has every button, link, field and menu item boxed and
numbered (a tiny built-in digit font, pure Rust), with the same numbers in a list. click /
typing / scroll / mouse_move take the number (ref) and land on the control's centre.
- Where the controls come from: macOS AX and Linux AT-SPI through agent-desktop on a person's
computer (desktop::elements); AT-SPI in the bots' own computer (box/a11y, after XBot's
box-a11y), web pages included. On a Mac a control whose spot another app's window covers gets
no number (topmost window at its centre must be the app's; the Dock's invisible full-screen
window doesn't count). open_app waits until the app is really in front.
- No going round in circles: a step that changed nothing says so; the same step (clicks within
16 px) a third time in a turn comes with a warning, from the fourth time it isn't run.
- The box (version 4): one TigerVNC server instead of Xvfb + x11vnc, the AT-SPI bus and registry
at boot, one browser launcher (box/browser, after XBot's box-chrome: same profile and CDP,
renderer accessibility, a launch lock, stale profile locks cleared). One docker exec per look
(list + picture, ~0.4 s). A brain updates an older box by itself when it has been idle a
minute (files and logins stay); a new box is never made from an older image.
- Gone: computer_use_read_app (the list is in every picture), computer_use_mouse_button, the
per-call picture for the phone (it gets a plain one), the stale browser_read note; the prompt
says web work goes through browser_* by number, and never to read browser cookie or history
files.
Tests: marks (numbering, drawing, fingerprints), box_numbers_are_exact on a box from the new
image (a Threads-like page: the icon links get numbers, Profile clicked by number, Chinese typed
into the numbered field, a click on nothing called out), box_screen_is_exact, VNC e2e 11/11 on
TigerVNC, Calculator 7×8=56 by numbers through the signed app on this Mac, remote_e2e 28/28,
153 unit tests; Linux build + 157 tests + x11_live.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 16:47:01 +00:00
|
|
|
|
assert_eq!(element_named(&[list], "9"), None);
|
|
|
|
|
|
assert_eq!(computer::describe("computer_use_click", r#"{"ref":"7"}"#), "點 #7");
|
2026-11-05 10:25:46 +00:00
|
|
|
|
assert_eq!(computer::describe("computer_use_click", r#"{"x":10,"y":20}"#), "點 (10, 20)");
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-05 08:47:28 +00:00
|
|
|
|
#[test]
|
|
|
|
|
|
fn every_computer_answers_to_its_own_name() {
|
|
|
|
|
|
// One name per computer: the bot's own goes by the bot's name, the others by theirs, and
|
|
|
|
|
|
// the everyday words point at the right one (「我的電腦」 = where the person typed).
|
|
|
|
|
|
let roles = tools::Roles { own: "Master 的電腦".into(), person: Some("王小明的Mac mini".into()), server: true };
|
|
|
|
|
|
let own = tools::Target { name: "Master 的電腦".into(), place: tools::Place::Box { dir: String::new() } };
|
2026-11-05 13:28:32 +00:00
|
|
|
|
let mac = tools::Target { name: "王小明的Mac mini".into(), place: tools::Place::Device { host: "m".into(), dir: String::new() } };
|
|
|
|
|
|
let desk = tools::Target { name: "ai".into(), place: tools::Place::Device { host: "a".into(), dir: String::new() } };
|
2026-11-05 08:47:28 +00:00
|
|
|
|
let server = tools::Target { name: "雲端大腦".into(), place: tools::Place::Local { workdir: Default::default() } };
|
|
|
|
|
|
assert!(mentions("在 Master 的電腦打開 Threads", &own, &roles));
|
|
|
|
|
|
assert!(mentions("用master的電腦", &own, &roles));
|
|
|
|
|
|
assert!(mentions("你都用 Bot的電腦", &own, &roles));
|
|
|
|
|
|
assert!(mentions("在 Mac mini 上開", &mac, &roles));
|
|
|
|
|
|
assert!(mentions("截圖我的電腦", &mac, &roles));
|
|
|
|
|
|
assert!(!mentions("截圖我的電腦", &desk, &roles));
|
|
|
|
|
|
assert!(!mentions("截圖我的電腦", &own, &roles));
|
|
|
|
|
|
assert!(mentions("在 ai 上跑", &desk, &roles));
|
|
|
|
|
|
assert!(!mentions("用 AI 幫我想標題", &mac, &roles));
|
|
|
|
|
|
assert!(mentions("在伺服器上看看磁碟", &server, &roles));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn a_step_moved_back_to_the_chosen_computer_says_so() {
|
|
|
|
|
|
let roles = tools::Roles { own: "Master 的電腦".into(), person: Some("王小明的Mac mini".into()), server: true };
|
|
|
|
|
|
let job = Job {
|
|
|
|
|
|
computers: vec![
|
|
|
|
|
|
tools::Target { name: "Master 的電腦".into(), place: tools::Place::Box { dir: String::new() } },
|
2026-11-05 13:28:32 +00:00
|
|
|
|
tools::Target { name: "王小明的Mac mini".into(), place: tools::Place::Device { host: "m".into(), dir: String::new() } },
|
2026-11-05 08:47:28 +00:00
|
|
|
|
],
|
|
|
|
|
|
default_computer: "Master 的電腦".into(),
|
|
|
|
|
|
roles,
|
|
|
|
|
|
..test_job()
|
|
|
|
|
|
};
|
|
|
|
|
|
// Not asked for: back to the chosen one, with a word for the bot.
|
|
|
|
|
|
let (args, note) = keep_to_chosen(&job, &[Message::user("幫我截圖")], r#"{"computer":"王小明的Mac mini"}"#);
|
|
|
|
|
|
assert!(args.contains("Master 的電腦") && note.unwrap().contains("王小明的Mac mini"));
|
|
|
|
|
|
// Named by the person: it stays.
|
|
|
|
|
|
let (args, note) = keep_to_chosen(&job, &[Message::user("看一下我的電腦畫面")], r#"{"computer":"王小明的Mac mini"}"#);
|
|
|
|
|
|
assert!(args.contains("王小明的Mac mini") && note.is_none());
|
2026-11-05 07:48:20 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
#[test]
|
|
|
|
|
|
fn cut_keeps_tool_pairs_and_recent_messages() {
|
|
|
|
|
|
let mut h = vec![];
|
|
|
|
|
|
for i in 0..6 {
|
|
|
|
|
|
h.push(Message::user(&format!("q{i}")));
|
|
|
|
|
|
let mut a = Message::assistant("", "");
|
|
|
|
|
|
a.tool_calls = vec![ToolCall { id: format!("c{i}"), name: "list_dir".into(), args: "{}".into(), status: ToolStatus::Done }];
|
|
|
|
|
|
h.push(a);
|
|
|
|
|
|
h.push(Message::tool_result(&format!("c{i}"), "list_dir", "x"));
|
|
|
|
|
|
}
|
|
|
|
|
|
let cut = compaction_cut(&h).unwrap();
|
|
|
|
|
|
assert_eq!(h[cut].role, Role::User);
|
|
|
|
|
|
assert!(h.len() - cut >= KEEP_RECENT_MESSAGES);
|
|
|
|
|
|
assert!(compaction_cut(&h[..5]).is_none());
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn tokens_count_chinese_heavier() {
|
|
|
|
|
|
assert_eq!(estimate_tokens("abcdefgh"), 2);
|
|
|
|
|
|
assert_eq!(estimate_tokens("你好"), 2);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Full loop against a fake provider: the model asks to list a folder, gets the result,
|
|
|
|
|
|
/// then answers in words.
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn loop_runs_tool_then_answers() {
|
|
|
|
|
|
use std::io::{Read, Write};
|
|
|
|
|
|
use std::net::TcpListener;
|
|
|
|
|
|
let l = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
|
|
|
|
let port = l.local_addr().unwrap().port();
|
|
|
|
|
|
std::thread::spawn(move || {
|
|
|
|
|
|
let replies = [
|
|
|
|
|
|
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"c1\",\"function\":{\"name\":\"list_dir\",\"arguments\":\"{}\"}}]}}]}\n\ndata: [DONE]\n\n",
|
|
|
|
|
|
"data: {\"choices\":[{\"delta\":{\"content\":\"裡面有 a.txt\"}}]}\n\ndata: [DONE]\n\n",
|
|
|
|
|
|
];
|
|
|
|
|
|
for body in replies {
|
|
|
|
|
|
let (mut s, _) = l.accept().unwrap();
|
|
|
|
|
|
let mut req = Vec::new();
|
|
|
|
|
|
let mut buf = [0u8; 8192];
|
|
|
|
|
|
loop {
|
|
|
|
|
|
let n = s.read(&mut buf).unwrap_or(0);
|
|
|
|
|
|
if n == 0 {
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
req.extend_from_slice(&buf[..n]);
|
|
|
|
|
|
let t = String::from_utf8_lossy(&req).to_string();
|
|
|
|
|
|
if let Some(end) = t.find("\r\n\r\n") {
|
|
|
|
|
|
let len = t[..end].lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::<usize>().unwrap_or(0))).unwrap_or(0);
|
|
|
|
|
|
if req.len() >= end + 4 + len {
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n");
|
|
|
|
|
|
let _ = s.write_all(body.as_bytes());
|
|
|
|
|
|
}
|
|
|
|
|
|
});
|
|
|
|
|
|
let w = std::env::temp_dir().join(format!("anybot-loop-{}", std::process::id()));
|
|
|
|
|
|
std::fs::create_dir_all(&w).unwrap();
|
|
|
|
|
|
std::fs::write(w.join("a.txt"), "hi").unwrap();
|
|
|
|
|
|
let job = Job {
|
|
|
|
|
|
req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("有什麼檔案?")], tools: tools::specs(), session: String::new() },
|
|
|
|
|
|
computers: vec![tools::Target { name: crate::model::THIS_COMPUTER_NAME.into(), place: tools::Place::Local { workdir: w } }],
|
|
|
|
|
|
default_computer: crate::model::THIS_COMPUTER_NAME.into(),
|
2026-11-05 08:47:28 +00:00
|
|
|
|
roles: tools::Roles { own: crate::model::BOT_COMPUTER_NAME.into(), person: Some(crate::model::THIS_COMPUTER_NAME.into()), server: false },
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
local_execution: true,
|
|
|
|
|
|
grants: vec![],
|
|
|
|
|
|
context_tokens: DEFAULT_CONTEXT_TOKENS,
|
|
|
|
|
|
via: "test".into(),
|
|
|
|
|
|
via_name: "Bot".into(),
|
|
|
|
|
|
tools_enabled: true,
|
|
|
|
|
|
direct: Arc::new(AtomicBool::new(false)),
|
|
|
|
|
|
local_locked: false,
|
|
|
|
|
|
steer: Arc::new(Mutex::new(vec![])),
|
|
|
|
|
|
review: None,
|
|
|
|
|
|
demos: vec![],
|
|
|
|
|
|
};
|
|
|
|
|
|
let (tx, rx) = channel();
|
|
|
|
|
|
let cancel = Arc::new(AtomicBool::new(false));
|
|
|
|
|
|
// Run the loop body on this thread's terms: spawn needs an egui context, so drive it directly.
|
|
|
|
|
|
let ctx = eframe::egui::Context::default();
|
|
|
|
|
|
spawn(job, 1, tx, cancel, ctx);
|
|
|
|
|
|
let mut texts = vec![];
|
|
|
|
|
|
let mut results = vec![];
|
|
|
|
|
|
loop {
|
|
|
|
|
|
match rx.recv_timeout(Duration::from_secs(10)).expect("loop stalled") {
|
|
|
|
|
|
(_, StreamEvent::ToolResult(m)) => results.push(m.text),
|
|
|
|
|
|
(_, StreamEvent::StepDone(m)) => texts.push(m.text),
|
|
|
|
|
|
(_, StreamEvent::Done) => break,
|
|
|
|
|
|
(_, StreamEvent::Error(e)) => panic!("{e}"),
|
|
|
|
|
|
_ => {}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
assert!(results[0].contains("a.txt"), "{results:?}");
|
|
|
|
|
|
assert_eq!(texts.last().unwrap(), "裡面有 a.txt");
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// A message sent while the bot works steers the same turn: the model sees it at the
|
|
|
|
|
|
/// next step and the turn only ends (one Done) after that.
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn mid_turn_message_steers_the_same_turn() {
|
|
|
|
|
|
use std::io::{Read, Write};
|
|
|
|
|
|
use std::net::TcpListener;
|
|
|
|
|
|
let l = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
|
|
|
|
let port = l.local_addr().unwrap().port();
|
|
|
|
|
|
let seen = Arc::new(Mutex::new(String::new()));
|
|
|
|
|
|
let seen2 = seen.clone();
|
|
|
|
|
|
std::thread::spawn(move || {
|
|
|
|
|
|
for body in ["data: {\"choices\":[{\"delta\":{\"content\":\"好\"}}]}\n\ndata: [DONE]\n\n", "data: {\"choices\":[{\"delta\":{\"content\":\"收到補充\"}}]}\n\ndata: [DONE]\n\n"] {
|
|
|
|
|
|
let (mut s, _) = l.accept().unwrap();
|
|
|
|
|
|
let mut req = Vec::new();
|
|
|
|
|
|
let mut buf = [0u8; 8192];
|
|
|
|
|
|
loop {
|
|
|
|
|
|
let n = s.read(&mut buf).unwrap_or(0);
|
|
|
|
|
|
if n == 0 {
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
req.extend_from_slice(&buf[..n]);
|
|
|
|
|
|
let t = String::from_utf8_lossy(&req).to_string();
|
|
|
|
|
|
if let Some(end) = t.find("\r\n\r\n") {
|
|
|
|
|
|
let len = t[..end].lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::<usize>().unwrap_or(0))).unwrap_or(0);
|
|
|
|
|
|
if req.len() >= end + 4 + len {
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
*seen2.lock().unwrap() = String::from_utf8_lossy(&req).to_string();
|
|
|
|
|
|
let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n");
|
|
|
|
|
|
let _ = s.write_all(body.as_bytes());
|
|
|
|
|
|
}
|
|
|
|
|
|
});
|
|
|
|
|
|
let steer = Arc::new(Mutex::new(vec!["順便用英文".to_string()]));
|
|
|
|
|
|
let job = Job {
|
|
|
|
|
|
req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("打招呼")], tools: vec![], session: String::new() },
|
|
|
|
|
|
computers: vec![],
|
|
|
|
|
|
default_computer: crate::model::BOT_COMPUTER_NAME.into(),
|
2026-11-05 08:47:28 +00:00
|
|
|
|
roles: tools::Roles { own: crate::model::BOT_COMPUTER_NAME.into(), person: Some(crate::model::THIS_COMPUTER_NAME.into()), server: false },
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
local_execution: false,
|
|
|
|
|
|
grants: vec![],
|
|
|
|
|
|
context_tokens: DEFAULT_CONTEXT_TOKENS,
|
|
|
|
|
|
via: "test".into(),
|
|
|
|
|
|
via_name: "Bot".into(),
|
|
|
|
|
|
tools_enabled: true,
|
|
|
|
|
|
direct: Arc::new(AtomicBool::new(false)),
|
|
|
|
|
|
local_locked: false,
|
|
|
|
|
|
steer,
|
|
|
|
|
|
review: None,
|
|
|
|
|
|
demos: vec![],
|
|
|
|
|
|
};
|
|
|
|
|
|
let (tx, rx) = channel();
|
|
|
|
|
|
spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default());
|
|
|
|
|
|
let (mut steered, mut done, mut texts) = (0, 0, vec![]);
|
|
|
|
|
|
loop {
|
|
|
|
|
|
match rx.recv_timeout(Duration::from_secs(10)).expect("loop stalled") {
|
|
|
|
|
|
(_, StreamEvent::Steered(m)) => {
|
|
|
|
|
|
assert!(m.steer);
|
|
|
|
|
|
steered += 1;
|
|
|
|
|
|
}
|
|
|
|
|
|
(_, StreamEvent::StepDone(m)) => texts.push(m.text),
|
|
|
|
|
|
(_, StreamEvent::Done) => {
|
|
|
|
|
|
done += 1;
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
(_, StreamEvent::Error(e)) => panic!("{e}"),
|
|
|
|
|
|
_ => {}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
assert_eq!((steered, done), (1, 1));
|
|
|
|
|
|
assert_eq!(texts, vec!["好", "收到補充"]);
|
|
|
|
|
|
assert!(seen.lock().unwrap().contains("順便用英文"));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Tools off and this Mac not allowed: the step pops up two requests (use tools, use this
|
|
|
|
|
|
/// computer) instead of failing, and runs once both are allowed.
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn missing_permission_pops_up_and_continues() {
|
|
|
|
|
|
use std::io::{Read, Write};
|
|
|
|
|
|
use std::net::TcpListener;
|
|
|
|
|
|
let l = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
|
|
|
|
let port = l.local_addr().unwrap().port();
|
|
|
|
|
|
std::thread::spawn(move || {
|
|
|
|
|
|
let replies = [
|
|
|
|
|
|
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"c1\",\"function\":{\"name\":\"list_dir\",\"arguments\":\"{\\\"computer\\\":\\\"你的電腦\\\"}\"}}]}}]}\n\ndata: [DONE]\n\n",
|
|
|
|
|
|
"data: {\"choices\":[{\"delta\":{\"content\":\"看完了\"}}]}\n\ndata: [DONE]\n\n",
|
|
|
|
|
|
];
|
|
|
|
|
|
for body in replies {
|
|
|
|
|
|
let (mut s, _) = l.accept().unwrap();
|
|
|
|
|
|
let mut req = Vec::new();
|
|
|
|
|
|
let mut buf = [0u8; 8192];
|
|
|
|
|
|
loop {
|
|
|
|
|
|
let n = s.read(&mut buf).unwrap_or(0);
|
|
|
|
|
|
if n == 0 {
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
req.extend_from_slice(&buf[..n]);
|
|
|
|
|
|
let t = String::from_utf8_lossy(&req).to_string();
|
|
|
|
|
|
if let Some(end) = t.find("\r\n\r\n") {
|
|
|
|
|
|
let len = t[..end].lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::<usize>().unwrap_or(0))).unwrap_or(0);
|
|
|
|
|
|
if req.len() >= end + 4 + len {
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n");
|
|
|
|
|
|
let _ = s.write_all(body.as_bytes());
|
|
|
|
|
|
}
|
|
|
|
|
|
});
|
|
|
|
|
|
let w = std::env::temp_dir().join(format!("anybot-perm-{}", std::process::id()));
|
|
|
|
|
|
std::fs::create_dir_all(&w).unwrap();
|
|
|
|
|
|
std::fs::write(w.join("note.txt"), "hi").unwrap();
|
|
|
|
|
|
let job = Job {
|
|
|
|
|
|
req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("看看我的電腦")], tools: tools::specs(), session: String::new() },
|
|
|
|
|
|
computers: vec![tools::Target { name: crate::model::THIS_COMPUTER_NAME.into(), place: tools::Place::Local { workdir: w } }],
|
|
|
|
|
|
default_computer: crate::model::THIS_COMPUTER_NAME.into(),
|
2026-11-05 08:47:28 +00:00
|
|
|
|
roles: tools::Roles { own: crate::model::BOT_COMPUTER_NAME.into(), person: Some(crate::model::THIS_COMPUTER_NAME.into()), server: false },
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
local_execution: false,
|
|
|
|
|
|
tools_enabled: false,
|
|
|
|
|
|
direct: Arc::new(AtomicBool::new(false)),
|
|
|
|
|
|
local_locked: false,
|
|
|
|
|
|
grants: vec![],
|
|
|
|
|
|
context_tokens: DEFAULT_CONTEXT_TOKENS,
|
|
|
|
|
|
via: "test".into(),
|
|
|
|
|
|
via_name: "Bot".into(),
|
|
|
|
|
|
steer: Arc::new(Mutex::new(vec![])),
|
|
|
|
|
|
review: None,
|
|
|
|
|
|
demos: vec![],
|
|
|
|
|
|
};
|
|
|
|
|
|
let (tx, rx) = channel();
|
|
|
|
|
|
spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default());
|
|
|
|
|
|
let mut kinds = vec![];
|
|
|
|
|
|
let mut result = String::new();
|
|
|
|
|
|
loop {
|
|
|
|
|
|
match rx.recv_timeout(Duration::from_secs(10)).expect("loop stalled") {
|
|
|
|
|
|
(_, StreamEvent::NeedApproval(a)) => {
|
|
|
|
|
|
kinds.push(a.kind);
|
|
|
|
|
|
let _ = a.reply.send(ApprovalAnswer::Once);
|
|
|
|
|
|
}
|
|
|
|
|
|
(_, StreamEvent::ToolResult(m)) => result = m.text,
|
|
|
|
|
|
(_, StreamEvent::Done) => break,
|
|
|
|
|
|
(_, StreamEvent::Error(e)) => panic!("{e}"),
|
|
|
|
|
|
_ => {}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
assert_eq!(kinds, vec![ApprovalKind::EnableTools, ApprovalKind::EnableThisComputer]);
|
|
|
|
|
|
assert!(result.contains("note.txt"), "{result}");
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-05 09:29:55 +00:00
|
|
|
|
/// 需要處理: the model hands a step to the person; the card's request carries the instruction
|
|
|
|
|
|
/// and the computer, 「我完成了」 tells the bot to carry on, 「跳過」 that the step is skipped.
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn a_hand_off_waits_for_the_person_and_says_what_they_chose() {
|
|
|
|
|
|
use std::io::{Read, Write};
|
|
|
|
|
|
use std::net::TcpListener;
|
|
|
|
|
|
for (answer, expect, status) in [(ApprovalAnswer::Once, "我完成了", ToolStatus::Done), (ApprovalAnswer::Reject, "跳過", ToolStatus::Denied)] {
|
|
|
|
|
|
let l = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
|
|
|
|
let port = l.local_addr().unwrap().port();
|
|
|
|
|
|
std::thread::spawn(move || {
|
|
|
|
|
|
let replies = [
|
|
|
|
|
|
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"h1\",\"function\":{\"name\":\"hand_over_computer\",\"arguments\":\"{\\\"instruction\\\":\\\"請登入 Threads\\\"}\"}}]}}]}\n\ndata: [DONE]\n\n",
|
|
|
|
|
|
"data: {\"choices\":[{\"delta\":{\"content\":\"好\"}}]}\n\ndata: [DONE]\n\n",
|
|
|
|
|
|
];
|
|
|
|
|
|
for body in replies {
|
|
|
|
|
|
let (mut s, _) = l.accept().unwrap();
|
|
|
|
|
|
let mut req = Vec::new();
|
|
|
|
|
|
let mut buf = [0u8; 8192];
|
|
|
|
|
|
loop {
|
|
|
|
|
|
let n = s.read(&mut buf).unwrap_or(0);
|
|
|
|
|
|
if n == 0 {
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
req.extend_from_slice(&buf[..n]);
|
|
|
|
|
|
let t = String::from_utf8_lossy(&req).to_string();
|
|
|
|
|
|
if let Some(end) = t.find("\r\n\r\n") {
|
|
|
|
|
|
let len = t[..end].lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::<usize>().unwrap_or(0))).unwrap_or(0);
|
|
|
|
|
|
if req.len() >= end + 4 + len {
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n");
|
|
|
|
|
|
let _ = s.write_all(body.as_bytes());
|
|
|
|
|
|
}
|
|
|
|
|
|
});
|
|
|
|
|
|
let job = Job {
|
|
|
|
|
|
req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("幫我登入 Threads")], tools: vec![], session: String::new() },
|
2026-11-05 13:28:32 +00:00
|
|
|
|
computers: vec![tools::Target { name: "王小明的Mac mini".into(), place: tools::Place::Device { host: "m".into(), dir: String::new() } }],
|
2026-11-05 09:29:55 +00:00
|
|
|
|
default_computer: "王小明的Mac mini".into(),
|
|
|
|
|
|
roles: tools::Roles { own: "Master 的電腦".into(), person: Some("王小明的Mac mini".into()), server: true },
|
|
|
|
|
|
..test_job()
|
|
|
|
|
|
};
|
|
|
|
|
|
let (tx, rx) = channel();
|
|
|
|
|
|
spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default());
|
|
|
|
|
|
let (mut asked, mut result, mut last) = (None, String::new(), None);
|
|
|
|
|
|
loop {
|
|
|
|
|
|
match rx.recv_timeout(Duration::from_secs(10)).expect("loop stalled") {
|
|
|
|
|
|
(_, StreamEvent::NeedApproval(a)) => {
|
|
|
|
|
|
asked = Some((a.kind, a.reason.clone(), a.grant.clone()));
|
|
|
|
|
|
let _ = a.reply.send(answer.clone());
|
|
|
|
|
|
}
|
|
|
|
|
|
(_, StreamEvent::ToolStatus(_, st)) => last = Some(st),
|
|
|
|
|
|
(_, StreamEvent::ToolResult(m)) => result = m.text,
|
|
|
|
|
|
(_, StreamEvent::Done) => break,
|
|
|
|
|
|
(_, StreamEvent::Error(e)) => panic!("{e}"),
|
|
|
|
|
|
_ => {}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
assert_eq!(asked, Some((ApprovalKind::Handoff, "請登入 Threads".to_string(), "王小明的Mac mini".to_string())));
|
|
|
|
|
|
assert!(result.contains(expect), "{result}");
|
|
|
|
|
|
assert_eq!(last, Some(status));
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
/// 直接操作: tools off, this Mac off, a command that changes files — and still no question.
|
|
|
|
|
|
/// Dangerous commands stay blocked.
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn direct_mode_asks_nothing_but_still_blocks_danger() {
|
|
|
|
|
|
use std::io::{Read, Write};
|
|
|
|
|
|
use std::net::TcpListener;
|
|
|
|
|
|
let l = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
|
|
|
|
let port = l.local_addr().unwrap().port();
|
|
|
|
|
|
std::thread::spawn(move || {
|
|
|
|
|
|
let replies = [
|
|
|
|
|
|
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"c1\",\"function\":{\"name\":\"shell\",\"arguments\":\"{\\\"command\\\":\\\"echo hi > made.txt\\\",\\\"computer\\\":\\\"你的電腦\\\"}\"}},{\"index\":1,\"id\":\"c2\",\"function\":{\"name\":\"shell\",\"arguments\":\"{\\\"command\\\":\\\"sudo ls\\\",\\\"computer\\\":\\\"你的電腦\\\"}\"}}]}}]}\n\ndata: [DONE]\n\n",
|
|
|
|
|
|
"data: {\"choices\":[{\"delta\":{\"content\":\"好了\"}}]}\n\ndata: [DONE]\n\n",
|
|
|
|
|
|
];
|
|
|
|
|
|
for body in replies {
|
|
|
|
|
|
let (mut s, _) = l.accept().unwrap();
|
|
|
|
|
|
let mut req = Vec::new();
|
|
|
|
|
|
let mut buf = [0u8; 8192];
|
|
|
|
|
|
loop {
|
|
|
|
|
|
let n = s.read(&mut buf).unwrap_or(0);
|
|
|
|
|
|
if n == 0 {
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
req.extend_from_slice(&buf[..n]);
|
|
|
|
|
|
let t = String::from_utf8_lossy(&req).to_string();
|
|
|
|
|
|
if let Some(end) = t.find("\r\n\r\n") {
|
|
|
|
|
|
let len = t[..end].lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::<usize>().unwrap_or(0))).unwrap_or(0);
|
|
|
|
|
|
if req.len() >= end + 4 + len {
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n");
|
|
|
|
|
|
let _ = s.write_all(body.as_bytes());
|
|
|
|
|
|
}
|
|
|
|
|
|
});
|
|
|
|
|
|
let w = std::env::temp_dir().join(format!("anybot-direct-{}", std::process::id()));
|
|
|
|
|
|
std::fs::create_dir_all(&w).unwrap();
|
|
|
|
|
|
let job = Job {
|
|
|
|
|
|
req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("做一個檔案")], tools: tools::specs(), session: String::new() },
|
|
|
|
|
|
computers: vec![tools::Target { name: crate::model::THIS_COMPUTER_NAME.into(), place: tools::Place::Local { workdir: w.clone() } }],
|
|
|
|
|
|
default_computer: crate::model::THIS_COMPUTER_NAME.into(),
|
2026-11-05 08:47:28 +00:00
|
|
|
|
roles: tools::Roles { own: crate::model::BOT_COMPUTER_NAME.into(), person: Some(crate::model::THIS_COMPUTER_NAME.into()), server: false },
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
local_execution: false,
|
|
|
|
|
|
tools_enabled: false,
|
|
|
|
|
|
direct: Arc::new(AtomicBool::new(true)),
|
|
|
|
|
|
local_locked: false,
|
|
|
|
|
|
grants: vec![],
|
|
|
|
|
|
context_tokens: DEFAULT_CONTEXT_TOKENS,
|
|
|
|
|
|
via: "test".into(),
|
|
|
|
|
|
via_name: "Bot".into(),
|
|
|
|
|
|
steer: Arc::new(Mutex::new(vec![])),
|
|
|
|
|
|
review: None,
|
|
|
|
|
|
demos: vec![],
|
|
|
|
|
|
};
|
|
|
|
|
|
let (tx, rx) = channel();
|
|
|
|
|
|
spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default());
|
|
|
|
|
|
let mut asked = 0;
|
|
|
|
|
|
let mut results = vec![];
|
|
|
|
|
|
loop {
|
|
|
|
|
|
match rx.recv_timeout(Duration::from_secs(10)).expect("loop stalled") {
|
|
|
|
|
|
(_, StreamEvent::NeedApproval(a)) => {
|
|
|
|
|
|
asked += 1;
|
|
|
|
|
|
let _ = a.reply.send(ApprovalAnswer::Reject);
|
|
|
|
|
|
}
|
|
|
|
|
|
(_, StreamEvent::ToolResult(m)) => results.push(m.text),
|
|
|
|
|
|
(_, StreamEvent::Done) => break,
|
|
|
|
|
|
(_, StreamEvent::Error(e)) => panic!("{e}"),
|
|
|
|
|
|
_ => {}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
assert_eq!(asked, 0, "{results:?}");
|
|
|
|
|
|
assert!(w.join("made.txt").exists());
|
|
|
|
|
|
assert!(results[1].contains("一律不允許"), "{results:?}");
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|