AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
// The switchable AI providers and how to talk to each one.
// Every provider streams its reply piece by piece over Server-Sent Events.
use serde ::{ Deserialize , Serialize } ;
use serde_json ::{ json , Value } ;
use std ::io ::{ BufRead , BufReader } ;
use std ::sync ::atomic ::{ AtomicBool , Ordering } ;
use std ::sync ::mpsc ::Sender ;
use std ::time ::Duration ;
use crate ::model ::{ Message , Role , ToolCall , ToolStatus } ;
#[ derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Debug, Hash) ]
pub enum ProviderKind {
Xai ,
OpenAi ,
Anthropic ,
Gemini ,
OpenRouter ,
OpenCodeGo ,
Ollama ,
Custom ,
}
pub const ALL_PROVIDERS : [ ProviderKind ; 8 ] = [
ProviderKind ::Xai ,
ProviderKind ::OpenAi ,
ProviderKind ::Anthropic ,
ProviderKind ::Gemini ,
ProviderKind ::OpenRouter ,
ProviderKind ::OpenCodeGo ,
ProviderKind ::Ollama ,
ProviderKind ::Custom ,
] ;
/// How long to wait for the first bytes before giving up
/// (the original shows "模型未能及時開始回應" in that case; its deadline is server-side: estimated).
pub const FIRST_RESPONSE_TIMEOUT_SECS : u64 = 120 ;
/// Connect deadline: main-core.cjs @1082927, `$D=15e3` (gateway-sse-connect).
pub const CONNECT_TIMEOUT_MS : u64 = 15_000 ;
/// Mid-stream silence that counts as a stall: main-core.cjs @1082927, `VD=35e3` (gateway-sse-stall idleMs).
pub const STREAM_IDLE_STALL_MS : u64 = 35_000 ;
/// Retry policy: main-core.cjs @1082927, `D3={maxAttempts:3,initialDelayMs:1e3,maxDelayMs:4e3,backoffFactor:2}`.
pub const RETRY_MAX_ATTEMPTS : u32 = 3 ;
pub const RETRY_INITIAL_DELAY_MS : u64 = 1_000 ;
pub const RETRY_MAX_DELAY_MS : u64 = 4_000 ;
pub const RETRY_BACKOFF_FACTOR : u64 = 2 ;
/// Longest server-paced Retry-After we honour (SAND-E0408 backendCapacityDeferred): estimated cap.
pub const RETRY_AFTER_CAP_MS : u64 = 30_000 ;
pub const MAX_OUTPUT_TOKENS_ANTHROPIC : u64 = 64000 ;
pub const ANTHROPIC_VERSION : & str = " 2026-10-21 " ;
/// OpenCode Go (opencode.ai/docs/go): one subscription, many open models. Each model is served
/// in its own API style — the public models list says which (`chat-completions` → {base}/openai/v1,
/// `messages` → {base}/anthropic/v1); every request carries a stable `x-opencode-session`.
pub const OPENCODE_GO_BASE : & str = " https://opencode.ai/inference/go " ;
impl ProviderKind {
pub fn key ( self ) -> & 'static str {
match self {
ProviderKind ::Xai = > " xai " ,
ProviderKind ::OpenAi = > " openai " ,
ProviderKind ::Anthropic = > " anthropic " ,
ProviderKind ::Gemini = > " gemini " ,
ProviderKind ::OpenRouter = > " openrouter " ,
ProviderKind ::OpenCodeGo = > " opencode-go " ,
ProviderKind ::Ollama = > " ollama " ,
ProviderKind ::Custom = > " custom " ,
}
}
pub fn label ( self ) -> & 'static str {
match self {
ProviderKind ::Xai = > " xAI Grok " ,
ProviderKind ::OpenAi = > " OpenAI " ,
ProviderKind ::Anthropic = > " Claude " ,
ProviderKind ::Gemini = > " Google Gemini " ,
ProviderKind ::OpenRouter = > " OpenRouter " ,
ProviderKind ::OpenCodeGo = > " OpenCode Go " ,
ProviderKind ::Ollama = > " Ollama( 本機) " ,
ProviderKind ::Custom = > " 自訂( OpenAI 相容) " ,
}
}
pub fn default_base_url ( self ) -> & 'static str {
match self {
ProviderKind ::Xai = > " https://api.x.ai/v1 " ,
ProviderKind ::OpenAi = > " https://api.openai.com/v1 " ,
ProviderKind ::Anthropic = > " https://api.anthropic.com/v1 " ,
ProviderKind ::Gemini = > " https://generativelanguage.googleapis.com/v1beta/openai " ,
ProviderKind ::OpenRouter = > " https://openrouter.ai/api/v1 " ,
// The models list is {base}/v1/models; chat goes to {base}/openai/v1 or {base}/anthropic/v1.
ProviderKind ::OpenCodeGo = > OPENCODE_GO_BASE ,
ProviderKind ::Ollama = > " http://localhost:11434/v1 " ,
ProviderKind ::Custom = > " http://localhost:8080/v1 " ,
}
}
pub fn suggested_model ( self ) -> & 'static str {
match self {
ProviderKind ::Xai = > " grok-4 " ,
ProviderKind ::OpenAi = > " gpt-5 " ,
ProviderKind ::Anthropic = > " claude-opus-5-5 " ,
ProviderKind ::Gemini = > " gemini-2.5-flash " ,
ProviderKind ::OpenRouter = > " openrouter/auto " ,
ProviderKind ::OpenCodeGo = > " kimi-k3 " ,
ProviderKind ::Ollama = > " llama3.2 " ,
ProviderKind ::Custom = > " local-model " ,
}
}
/// A few names to pick from before the real list is fetched.
pub fn starter_models ( self ) -> & 'static [ & 'static str ] {
match self {
ProviderKind ::Xai = > & [ " grok-4 " , " grok-4-fast " , " grok-3-mini " ] ,
ProviderKind ::OpenAi = > & [ " gpt-5 " , " gpt-5-mini " , " gpt-4.1 " ] ,
ProviderKind ::Anthropic = > & [ " claude-opus-5-5 " , " claude-sonnet-5-5 " , " claude-haiku-5-5 " , " claude-fable-5-1 " ] ,
ProviderKind ::Gemini = > & [ " gemini-2.5-flash " , " gemini-2.5-pro " ] ,
ProviderKind ::OpenRouter = > & [ " openrouter/auto " ] ,
ProviderKind ::OpenCodeGo = > & [ " kimi-k3 " , " glm-5.3 " , " deepseek-v4-pro " , " qwen3.8-max " , " minimax-m3 " , " mimo-v2.6-pro " ] ,
ProviderKind ::Ollama = > & [ " llama3.2 " , " qwen2.5 " , " gemma3 " ] ,
ProviderKind ::Custom = > & [ ] ,
}
}
pub fn needs_key ( self ) -> bool {
! matches! ( self , ProviderKind ::Ollama | ProviderKind ::Custom )
}
pub fn env_var ( self ) -> Option < & 'static str > {
match self {
ProviderKind ::Xai = > Some ( " XAI_API_KEY " ) ,
ProviderKind ::OpenAi = > Some ( " OPENAI_API_KEY " ) ,
ProviderKind ::Anthropic = > Some ( " ANTHROPIC_API_KEY " ) ,
ProviderKind ::Gemini = > Some ( " GEMINI_API_KEY " ) ,
ProviderKind ::OpenRouter = > Some ( " OPENROUTER_API_KEY " ) ,
ProviderKind ::OpenCodeGo = > Some ( " OPENCODE_API_KEY " ) ,
_ = > None ,
}
}
pub fn key_help ( self ) -> & 'static str {
match self {
ProviderKind ::Xai = > " 在 console.x.ai 取得 API 金鑰 " ,
ProviderKind ::OpenAi = > " 在 platform.openai.com 取得 API 金鑰 " ,
ProviderKind ::Anthropic = > " 在 console.anthropic.com 取得 API 金鑰 " ,
ProviderKind ::Gemini = > " 在 aistudio.google.com 取得 API 金鑰 " ,
ProviderKind ::OpenRouter = > " 在 openrouter.ai 取得 API 金鑰 " ,
ProviderKind ::OpenCodeGo = > " 在 opencode.ai 的 Console 訂閱 Go, 再複製 API 金鑰 " ,
ProviderKind ::Ollama = > " 先安裝並開啟 Ollama, 不需要金鑰 " ,
ProviderKind ::Custom = > " 任何 OpenAI 相容的服務( LM Studio、vLLM…) " ,
}
}
}
/// Everything one model call needs, copied so the worker thread owns it.
#[ derive(Clone) ]
pub struct ChatRequest {
pub kind : ProviderKind ,
pub base_url : String ,
pub api_key : String ,
pub model : String ,
pub system : String ,
pub history : Vec < Message > ,
/// Tool specs ({name, description, parameters}); empty for plain chat.
pub tools : Vec < Value > ,
/// One id per conversation (OpenCode Go routes and caches by it).
pub session : String ,
}
pub enum StreamEvent {
Text ( String ) ,
Thinking ( String ) ,
/// Any line from the provider (keeps the stall watchdog quiet).
Alive ,
/// A retry is about to happen; the text says why.
Retrying ( String ) ,
/// The model asked for a tool; shown at once, run after the step ends.
ToolCall ( ToolCall ) ,
/// A tool's state changed (waiting for approval, running, done…).
ToolStatus ( String , ToolStatus ) ,
/// Ask the person before running a tool; answer on the sender.
NeedApproval ( Approval ) ,
/// One model step finished: replace the streaming reply with the full message.
StepDone ( Message ) ,
/// A tool result to append to the chat.
ToolResult ( Message ) ,
/// The next model step starts: add a fresh reply bubble.
NextStep ( String ) ,
/// The oldest `n` messages were folded into this summary.
2026-11-05 07:48:20 +00:00
/// Older messages were folded into this summary; `cut` is the count in the model's view and
/// `boundary` the timestamp of the first kept message (the window's own copy may be longer).
Compacted ( usize , Message , i64 ) ,
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
/// A message the person sent mid-turn was handed to the model (add it to the chat).
Steered ( Message ) ,
/// Do something in the app (routines, bots, settings) and answer on the sender.
AppAction ( AppCall ) ,
/// 表情回應: the bot put an emoji on the person's latest message.
React ( String ) ,
/// The bot sent a file into the chat (a file card under its reply).
Attach ( crate ::attach ::Attachment ) ,
2026-11-06 15:12:53 +00:00
/// 自己進化: after the turn, the way that worked was kept as a skill (with the time of the
/// reply it learned from, which gets a line saying so).
Learned ( crate ::memory ::Skill , i64 ) ,
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
Done ,
Error ( String ) ,
}
pub struct Approval {
pub call_id : String ,
pub tool : String ,
pub reason : String ,
/// What "always allow" would remember; empty when it can't be remembered.
pub grant : String ,
pub reply : Sender < ApprovalAnswer > ,
pub kind : ApprovalKind ,
}
/// What the person is asked.
#[ derive(Clone, Copy, PartialEq, Eq, Debug, Default) ]
pub enum ApprovalKind {
/// One action (run this command, edit this file…).
#[ default ]
Action ,
/// This bot's computer tools are off: let it use them? ("always" turns them on for the bot.)
EnableTools ,
/// 在這台電腦上執行 is off: let it work on this Mac? ("always" turns the setting on.)
EnableThisComputer ,
2026-11-05 09:29:55 +00:00
/// 需要處理: a step only the person can do on a computer (signing in, a captcha…). `reason`
/// is what to do, `grant` the computer's name; Once = 「我完成了」, Reject = 「跳過」 (the
/// original's box hand-off card).
Handoff ,
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
}
pub struct AppCall {
pub call_id : String ,
pub name : String ,
pub args : String ,
/// (result text, ok)
pub reply : Sender < ( String , bool ) > ,
}
#[ derive(Clone, Debug, PartialEq) ]
pub enum ApprovalAnswer {
Once ,
Always ( String ) ,
Reject ,
}
/// What one model call produced.
#[ derive(Default, Debug) ]
pub struct StepOutput {
pub text : String ,
pub thinking : String ,
pub tool_calls : Vec < ToolCall > ,
/// Claude's raw content blocks, kept to replay thinking signatures unchanged.
pub blocks : Vec < Value > ,
}
/// Why a request failed, sorted the way the original's error catalog sorts them
/// (main-core.cjs @689060..692409, SAND-E0401..E0418).
pub struct Failure {
pub msg : String ,
pub retryable : bool ,
pub retry_after_ms : Option < u64 > ,
}
impl Failure {
fn retry ( msg : String ) -> Self {
Failure { msg , retryable : true , retry_after_ms : None }
}
fn fatal ( msg : String ) -> Self {
Failure { msg , retryable : false , retry_after_ms : None }
}
}
/// Delay before retry `n` (1-based): 1 s, 2 s, 4 s, capped at 4 s.
pub fn backoff_delay_ms ( retry : u32 ) -> u64 {
( RETRY_INITIAL_DELAY_MS * RETRY_BACKOFF_FACTOR . pow ( retry . saturating_sub ( 1 ) ) ) . min ( RETRY_MAX_DELAY_MS )
}
fn agent ( ) -> ureq ::Agent {
let config = ureq ::Agent ::config_builder ( )
. http_status_as_error ( false )
. timeout_connect ( Some ( Duration ::from_millis ( CONNECT_TIMEOUT_MS ) ) )
. timeout_recv_response ( Some ( Duration ::from_secs ( FIRST_RESPONSE_TIMEOUT_SECS ) ) )
2026-11-05 07:48:20 +00:00
// A stream that goes silent (a NAT that dropped the mapping, a server that died without
// a FIN) must end: without a read timeout the lines() loop blocks forever, and every
// such turn leaks a thread and its socket. 5 minutes of total silence is a dead stream.
. timeout_per_read ( Some ( Duration ::from_secs ( 300 ) ) )
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
. build ( ) ;
config . into ( )
}
pub fn resolve_key ( kind : ProviderKind , stored : & str ) -> String {
if ! stored . trim ( ) . is_empty ( ) {
return stored . trim ( ) . to_string ( ) ;
}
kind . env_var ( ) . and_then ( | v | std ::env ::var ( v ) . ok ( ) ) . unwrap_or_default ( )
}
pub fn resolve_base ( kind : ProviderKind , stored : & str ) -> String {
let b = if stored . trim ( ) . is_empty ( ) { kind . default_base_url ( ) } else { stored . trim ( ) } ;
b . trim_end_matches ( '/' ) . to_string ( )
}
/// Claude models that take adaptive thinking (older ones reject it).
fn claude_adaptive ( model : & str ) -> bool {
[ " claude-opus-5 " , " claude-sonnet-5 " , " claude-haiku-5 " , " claude-fable-5 " , " claude-mythos-5 " , " claude-opus-4-6 " , " claude-opus-4-7 " , " claude-opus-4-8 " , " claude-sonnet-4-6 " ]
. iter ( )
. any ( | p | model . starts_with ( p ) )
}
/// Claude models where the server can retry a declined request on another model.
fn claude_fallbacks ( model : & str ) -> bool {
[ " claude-opus-5-5 " , " claude-opus-5 " , " claude-fable-5-1 " , " claude-sonnet-5-5 " ] . contains ( & model )
}
2026-11-06 08:51:25 +00:00
/// An error body as words: a gateway's HTML error page (「錯誤 502: <!DOCTYPE HTML>…」 filled the
/// chat) becomes its text; anything else is kept as it is.
fn page_words ( body : & str ) -> String {
if ! body . trim_start ( ) . starts_with ( '<' ) {
return body . to_string ( ) ;
}
let no_style = body . rsplit ( " </style> " ) . next ( ) . unwrap_or ( body ) ;
let mut out = String ::new ( ) ;
let mut in_tag = false ;
for c in no_style . chars ( ) {
match c {
'<' = > {
in_tag = true ;
out . push ( ' ' ) ;
}
'>' = > in_tag = false ,
_ if ! in_tag = > out . push ( c ) ,
_ = > { }
}
}
out . split_whitespace ( ) . collect ::< Vec < _ > > ( ) . join ( " " )
}
2026-11-06 11:47:43 +00:00
/// An error the model sent inside its answer stream, in words: "at capacity" and the like are
/// waited out and tried again (it came through raw English, as the reply, and was never retried).
fn stream_failure ( msg : & str ) -> Failure {
let low = msg . to_lowercase ( ) ;
if [ " capacity " , " overloaded " , " rate limit " , " too many requests " , " try again " , " temporarily " , " unavailable " , " timeout " , " timed out " ] . iter ( ) . any ( | k | low . contains ( k ) ) {
Failure { msg : format ! ( " 模型供應商現在太忙,正在等一下再試。 \n ( {msg}) " ) , retryable : true , retry_after_ms : Some ( BUSY_RETRY_MS ) }
} else {
Failure ::fatal ( format! ( " 模型回報錯誤,這一步沒有完成。 \n ( {msg} ) " ) )
}
}
/// Wait before asking a busy model again (estimated; it says "a few minutes", 1– 4 s was too soon).
const BUSY_RETRY_MS : u64 = 8_000 ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
fn http_error ( status : u16 , body : & str , retry_after : Option < u64 > ) -> Failure {
let detail = serde_json ::from_str ::< Value > ( body )
. ok ( )
. and_then ( | v | {
v . pointer ( " /error/message " )
. or_else ( | | v . pointer ( " /message " ) )
. or_else ( | | v . get ( " error " ) )
. map ( | m | m . as_str ( ) . map ( str ::to_string ) . unwrap_or_else ( | | m . to_string ( ) ) )
} )
2026-11-06 08:51:25 +00:00
. unwrap_or_else ( | | page_words ( body ) . chars ( ) . take ( 300 ) . collect ( ) ) ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
let hint = match status {
401 | 403 = > " 金鑰無效或沒有權限,請到「設定 → AI 供應商」檢查。 " ,
404 = > " 找不到這個模型或網址,請檢查模型名稱。 " ,
429 = > " 模型供應商已超載或用量已達上限。通常是暫時的,請稍後再試。 " ,
500 ..= 599 = > " 模型供應商目前負載過高。通常是暫時的。 " ,
_ = > " " ,
} ;
let lower = detail . to_lowercase ( ) ;
// SAND-E0404 contextWindowOverflow: a retry would repeat it.
if ( status = = 400 | | status = = 413 ) & & [ " context " , " too long " , " too many tokens " , " maximum context " , " prompt is too long " ] . iter ( ) . any ( | k | lower . contains ( k ) ) {
return Failure ::fatal ( format! ( " 對話太長了,超過這個模型能讀的長度。請清除聊天記錄或換一個模型。 \n ( {detail} ) " ) ) ;
}
if status = = 400 & & lower . contains ( " tool " ) & & ( lower . contains ( " support " ) | | lower . contains ( " not available " ) ) {
return Failure ::fatal ( format! ( " 這個模型不支援工具。請換一個支援工具的模型,或在 Bot 設定裡關掉「電腦操作」。 \n ( {detail} ) " ) ) ;
}
let msg = format! ( " 錯誤 {status} : {detail} \n {hint} " ) . trim ( ) . to_string ( ) ;
// SAND-E0401 providerOverloaded / SAND-E0408 backendCapacityDeferred: retry.
if status = = 429 | | status = = 408 | | status = = 529 | | status > = 500 {
Failure { msg , retryable : true , retry_after_ms : retry_after }
} else {
Failure ::fatal ( msg )
}
}
fn retry_after_ms ( resp : & ureq ::http ::Response < ureq ::Body > ) -> Option < u64 > {
let v = resp . headers ( ) . get ( " retry-after " ) ? . to_str ( ) . ok ( ) ? ;
v . trim ( ) . parse ::< f64 > ( ) . ok ( ) . map ( | s | ( s * 1000.0 ) as u64 ) . map ( | ms | ms . min ( RETRY_AFTER_CAP_MS ) )
}
/// One model call with the original's retry rules: retry overloads, resets and unreachable
/// backends up to 3 tries (1 s, 2 s, 4 s or Retry-After), but never once output reached the screen.
pub fn run_step ( req : & ChatRequest , cancel : & AtomicBool , send : & dyn Fn ( StreamEvent ) ) -> Result < StepOutput , String > {
// The bot's instructions and tool descriptions in this system's words ("this computer" on Linux).
#[ cfg(not(target_os = " macos " )) ]
let localized = {
let mut r = req . clone ( ) ;
r . system = crate ::platform ::localize ( & r . system ) ;
r . tools . iter_mut ( ) . for_each ( crate ::platform ::localize_value ) ;
r
} ;
#[ cfg(not(target_os = " macos " )) ]
let req = & localized ;
if req . kind . needs_key ( ) & & req . api_key . is_empty ( ) {
return Err ( format! ( " 還沒有設定 {} 的 API 金鑰。請到「設定 → AI 供應商」貼上金鑰。 " , req . kind . label ( ) ) ) ;
}
let produced = std ::cell ::Cell ::new ( false ) ;
let tracked = | ev : StreamEvent | {
if matches! ( ev , StreamEvent ::Text ( _ ) | StreamEvent ::Thinking ( _ ) | StreamEvent ::ToolCall ( _ ) ) {
produced . set ( true ) ;
}
send ( ev ) ;
} ;
let mut attempt = 1 ;
// Steps left without a result (stopped while waiting for the person) are closed off.
let repaired = ChatRequest { history : repair_history ( & req . history ) , .. req . clone ( ) } ;
let req = & repaired ;
let mut blind_copy : Option < ChatRequest > = None ;
loop {
2026-11-04 13:52:06 +00:00
// A model that can't see pictures gets the request without them: the person's pictures
2026-11-05 00:21:34 +00:00
// and the newest screenshot described by a model that can (看圖小幫手).
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
if ! sees_images ( req ) & & blind_copy . is_none ( ) {
2026-11-04 13:52:06 +00:00
blind_copy = Some ( if DESCRIBING . with ( | d | d . get ( ) ) { without_images ( req ) } else { with_descriptions ( req , cancel , send ) } ) ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
}
let req = blind_copy . as_ref ( ) . unwrap_or ( req ) ;
let result = match req . kind {
ProviderKind ::Anthropic = > stream_anthropic ( req , cancel , & tracked ) ,
ProviderKind ::OpenCodeGo = > {
let mut r = req . clone ( ) ;
if opencode_go_style ( & req . base_url , & req . model ) = = " messages " {
r . base_url = format! ( " {} /anthropic/v1 " , req . base_url ) ;
stream_anthropic ( & r , cancel , & tracked )
} else {
r . base_url = format! ( " {} /openai/v1 " , req . base_url ) ;
stream_openai_compatible ( & r , cancel , & tracked )
}
}
_ = > stream_openai_compatible ( req , cancel , & tracked ) ,
} ;
match result {
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice
Web
- web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus.
- browser_open waits for the page to load and returns it like browser_read (one step fewer per page);
actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab.
The bot's computer (box v6)
- Numbers only on what can be seen: clipped to the window and the page's visible area, none under
another app's window (a link scrolled under the toolbar was numbered on Back).
- press_key with a typed character no longer swallows the keys after it (xdotool type --args 1).
- The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys).
- "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a
time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs.
- One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone).
The person's computers
- Linux: the window system's front app when it has one, and the reason when there is none (was 「」);
hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist.
The loop
- Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows):
it was a plain shell on the brain itself, keys in reach, asked nothing.
- The bot's own computer is re-checked each step; unreachable computers are failures, not safety
blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys
aren't counted as going in circles; steps aren't moved to a computer that can't do them.
- One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the
impossible server branches removed; duplicated prompt instructions merged.
Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses
on its box; remote E2E 28/28.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
// Nothing at all came back (no words, no tool call): a dropped stream or a hiccup. The
// turn used to end on 「(沒有回應內容)」; ask again instead.
Ok ( out ) if out . text . trim ( ) . is_empty ( ) & & out . tool_calls . is_empty ( ) & & ! produced . get ( ) & & attempt < RETRY_MAX_ATTEMPTS & & ! cancel . load ( Ordering ::Relaxed ) = > {
send ( StreamEvent ::Retrying ( format! ( " 模型這次沒有回任何內容,再問一次(第 {} 次,共 {} 次)… " , attempt + 1 , RETRY_MAX_ATTEMPTS ) ) ) ;
std ::thread ::sleep ( Duration ::from_millis ( backoff_delay_ms ( attempt ) ) ) ;
attempt + = 1 ;
}
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
Ok ( out ) = > return Ok ( out ) ,
Err ( f ) if image_refused ( & f . msg ) & & sees_images ( req ) & & ! produced . get ( ) & & ! cancel . load ( Ordering ::Relaxed ) = > {
mark_blind ( req ) ;
send ( StreamEvent ::Retrying ( " 這個模型不能看圖片,改成不附圖片再試一次… " . into ( ) ) ) ;
}
Err ( f ) if f . retryable & & ! produced . get ( ) & & attempt < RETRY_MAX_ATTEMPTS & & ! cancel . load ( Ordering ::Relaxed ) = > {
let wait = f . retry_after_ms . unwrap_or_else ( | | backoff_delay_ms ( attempt ) ) ;
send ( StreamEvent ::Retrying ( format! ( " {} \n {:.0} 秒後重試(第 {} 次,共 {} 次)… " , f . msg , wait as f64 / 1000.0 , attempt + 1 , RETRY_MAX_ATTEMPTS ) ) ) ;
let until = std ::time ::Instant ::now ( ) + Duration ::from_millis ( wait ) ;
while std ::time ::Instant ::now ( ) < until {
if cancel . load ( Ordering ::Relaxed ) {
return Ok ( StepOutput ::default ( ) ) ;
}
std ::thread ::sleep ( Duration ::from_millis ( 100 ) ) ;
}
attempt + = 1 ;
}
Err ( f ) = > return Err ( f . msg ) ,
}
}
}
2026-11-05 07:00:31 +00:00
fn read_sse ( body : ureq ::Body , cancel : & AtomicBool , send : & dyn Fn ( StreamEvent ) , on_data : impl FnMut ( & str , & str ) -> bool ) -> Result < ( ) , Failure > {
read_sse_from ( body . into_reader ( ) , cancel , send , on_data )
}
fn read_sse_from ( r : impl std ::io ::Read , cancel : & AtomicBool , send : & dyn Fn ( StreamEvent ) , mut on_data : impl FnMut ( & str , & str ) -> bool ) -> Result < ( ) , Failure > {
let reader = BufReader ::new ( r ) ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
let mut event = String ::new ( ) ;
for line in reader . lines ( ) {
if cancel . load ( Ordering ::Relaxed ) {
return Ok ( ( ) ) ;
}
// SAND-E0403 streamReset: retryable.
let line = line . map_err ( | e | Failure ::retry ( format! ( " 連線中斷: {e} " ) ) ) ? ;
send ( StreamEvent ::Alive ) ;
if let Some ( ev ) = line . strip_prefix ( " event: " ) {
event = ev . trim ( ) . to_string ( ) ;
} else if let Some ( data ) = line . strip_prefix ( " data: " ) {
if ! on_data ( & event , data . trim ( ) ) {
return Ok ( ( ) ) ;
}
} else if line . is_empty ( ) {
event . clear ( ) ;
}
}
Ok ( ( ) )
}
/// Summaries become part of the system prompt; everything else stays in order.
fn system_with_summaries ( req : & ChatRequest ) -> String {
let mut s = req . system . clone ( ) ;
for m in req . history . iter ( ) . filter ( | m | m . summary ) {
s . push_str ( " \n \n 先前對話的摘要: \n " ) ;
s . push_str ( & m . text ) ;
}
s
}
/// Which messages still show their screenshots: the newest `KEEP_SCREENSHOTS`.
fn kept_images ( history : & [ Message ] ) -> Vec < usize > {
let mut idx : Vec < usize > = history . iter ( ) . enumerate ( ) . filter ( | ( _ , m ) | ! m . images . is_empty ( ) & & ! m . summary & & m . role = = Role ::Tool ) . map ( | ( i , _ ) | i ) . collect ( ) ;
let drop = idx . len ( ) . saturating_sub ( crate ::computer ::KEEP_SCREENSHOTS ) ;
idx . drain ( .. drop ) ;
idx
}
const OLD_SCREENSHOT_NOTE : & str = " (較早的截圖已省略) " ;
/// Said in place of a screenshot when the model can't take pictures.
const BLIND_SCREENSHOT_NOTE : & str = " (截圖已拍好,但這個模型看不到圖片,所以沒有附上。請改用文字方式確認畫面,例如讀取視窗或網頁的文字。) " ;
2026-11-05 00:21:34 +00:00
/// What a helper is asked about a screenshot.
const SCREEN_ASK : & str = " 這是一台電腦螢幕的截圖。請描述:開著哪些視窗或 App、畫面上的文字( 照抄重要的字) 、按鈕和欄位, 以及它們大約的位置( 以這張圖的像素座標, 左上角是 0,0) 。不要猜看不清楚的字。 " ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
const BLIND_ATTACHMENT_NOTE : & str = " (使用者附了圖片,但這個模型看不到圖片。) " ;
/// Models (base url + name) that refused a picture: later requests leave pictures out.
static NO_VISION : std ::sync ::Mutex < Option < std ::collections ::HashSet < String > > > = std ::sync ::Mutex ::new ( None ) ;
fn vision_key ( req : & ChatRequest ) -> String {
format! ( " {} | {} " , req . base_url , req . model )
}
pub fn sees_images ( req : & ChatRequest ) -> bool {
! NO_VISION . lock ( ) . unwrap ( ) . as_ref ( ) . map ( | s | s . contains ( & vision_key ( req ) ) ) . unwrap_or ( false )
}
fn mark_blind ( req : & ChatRequest ) {
NO_VISION . lock ( ) . unwrap ( ) . get_or_insert_with ( Default ::default ) . insert ( vision_key ( req ) ) ;
}
/// The provider said this model can't take images ("At most 0 image(s)…", "does not support
/// image input", "unknown variant `image_url`"…).
pub fn image_refused ( msg : & str ) -> bool {
let l = msg . to_lowercase ( ) ;
let about_images = l . contains ( " image " ) | | l . contains ( " vision " ) | | l . contains ( " multimodal " ) | | l . contains ( " 圖片 " ) | | l . contains ( " 图片 " ) ;
about_images & & [ " at most 0 " , " not support " , " unsupported " , " doesn't support " , " does not support " , " no vision " , " not a vision " , " not multimodal " , " unknown variant " , " invalid type " , " only text " , " text-only " , " 不支援 " , " 不支持 " ] . iter ( ) . any ( | k | l . contains ( k ) )
}
/// Every tool call needs its result right after it (providers refuse the conversation
/// otherwise). A step stopped or refused while it waited for the person has none: give it one.
/// Results whose call is gone are dropped.
pub fn repair_history ( history : & [ Message ] ) -> Vec < Message > {
let mut out : Vec < Message > = Vec ::with_capacity ( history . len ( ) ) ;
let mut i = 0 ;
while i < history . len ( ) {
let m = & history [ i ] ;
if m . role = = Role ::Tool {
// A result with no call right before it.
i + = 1 ;
continue ;
}
out . push ( m . clone ( ) ) ;
i + = 1 ;
if m . role = = Role ::Assistant & & ! m . tool_calls . is_empty ( ) {
let ids : Vec < & str > = m . tool_calls . iter ( ) . map ( | c | c . id . as_str ( ) ) . collect ( ) ;
let mut seen = vec! [ ] ;
while i < history . len ( ) & & history [ i ] . role = = Role ::Tool {
let r = & history [ i ] ;
if ids . contains ( & r . tool_call_id . as_str ( ) ) & & ! seen . contains ( & r . tool_call_id ) {
seen . push ( r . tool_call_id . clone ( ) ) ;
out . push ( r . clone ( ) ) ;
}
i + = 1 ;
}
for c in & m . tool_calls {
if ! seen . contains ( & c . id ) {
let why = if c . status = = ToolStatus ::Denied { " (使用者沒有允許這個步驟,所以沒有執行。) " } else { " (這個步驟被中止了,沒有執行完。) " } ;
out . push ( Message ::tool_result ( & c . id , & c . name , why ) ) ;
}
}
}
}
out
}
2026-11-04 13:52:06 +00:00
// ---------- 看圖小幫手 ----------
/// A model of the person's that sees pictures (another provider, or another model).
#[ derive(Clone, Debug, PartialEq) ]
pub struct Eyes {
pub kind : ProviderKind ,
pub base_url : String ,
pub api_key : String ,
pub model : String ,
}
static EYES : std ::sync ::Mutex < Vec < Eyes > > = std ::sync ::Mutex ::new ( Vec ::new ( ) ) ;
thread_local! {
/// Inside a helper's own call (it never asks a helper itself).
static DESCRIBING : std ::cell ::Cell < bool > = const { std ::cell ::Cell ::new ( false ) } ;
}
/// The models to ask, best first (the app sets this from the person's providers).
pub fn set_eyes ( v : Vec < Eyes > ) {
* EYES . lock ( ) . unwrap ( ) = v ;
}
2026-11-05 07:48:20 +00:00
/// The helper list, the description cache and the failure list are process-wide: tests that
/// touch them must run one at a time, from a clean slate, or they undo each other (the
/// describe cache hit in one test made another's helper never get called).
#[ cfg(test) ]
pub fn vision_test_guard ( ) -> std ::sync ::MutexGuard < 'static , ( ) > {
static L : std ::sync ::Mutex < ( ) > = std ::sync ::Mutex ::new ( ( ) ) ;
let g = L . lock ( ) . unwrap_or_else ( | e | e . into_inner ( ) ) ;
notes ( ) . lock ( ) . unwrap ( ) . clear ( ) ;
failed_eyes ( ) . lock ( ) . unwrap ( ) . clear ( ) ;
set_eyes ( vec! [ ] ) ;
g
}
2026-11-04 13:52:06 +00:00
/// Models likely to see pictures, by provider.
pub fn seeing_models ( kind : ProviderKind ) -> & 'static [ & 'static str ] {
match kind {
ProviderKind ::Xai = > & [ " grok-4 " , " grok-4-fast " ] ,
ProviderKind ::Anthropic = > & [ " claude-sonnet-5-5 " , " claude-haiku-5-5 " ] ,
ProviderKind ::OpenAi = > & [ " gpt-5-mini " , " gpt-4.1 " ] ,
ProviderKind ::Gemini = > & [ " gemini-2.5-flash " ] ,
ProviderKind ::OpenRouter = > & [ " openrouter/auto " ] ,
_ = > & [ ] ,
}
}
const EYES_PROMPT : & str = " 你是「看圖小幫手」:替一個看不到圖片的 AI 助理看圖。用繁體中文把圖片完整描述出來,讓它不用看圖也能幫使用者:畫面是什麼、主要的東西和位置、所有看得到的文字(原文照抄,包括數字、表格、按鈕和錯誤訊息)、顏色或狀態等重點。只描述,不評論、不問問題、不說你是誰。 " ;
/// Descriptions already made (by the picture), kept across restarts.
fn notes_path ( ) -> std ::path ::PathBuf {
crate ::model ::State ::path ( ) . with_file_name ( " image-notes.json " )
}
fn notes ( ) -> & 'static std ::sync ::Mutex < std ::collections ::HashMap < String , ( String , String ) > > {
static N : std ::sync ::OnceLock < std ::sync ::Mutex < std ::collections ::HashMap < String , ( String , String ) > > > = std ::sync ::OnceLock ::new ( ) ;
N . get_or_init ( | | std ::sync ::Mutex ::new ( if cfg! ( test ) { Default ::default ( ) } else { std ::fs ::read ( notes_path ( ) ) . ok ( ) . and_then ( | b | serde_json ::from_slice ( & b ) . ok ( ) ) . unwrap_or_default ( ) } ) )
}
/// Helpers that just failed (wrong model name, no credit, down…), skipped for a while.
fn failed_eyes ( ) -> & 'static std ::sync ::Mutex < std ::collections ::HashMap < String , std ::time ::Instant > > {
static F : std ::sync ::OnceLock < std ::sync ::Mutex < std ::collections ::HashMap < String , std ::time ::Instant > > > = std ::sync ::OnceLock ::new ( ) ;
F . get_or_init ( Default ::default )
}
fn picture_key ( b64 : & str ) -> String {
let d = ring ::digest ::digest ( & ring ::digest ::SHA256 , b64 . as_bytes ( ) ) ;
d . as_ref ( ) [ .. 12 ] . iter ( ) . map ( | b | format! ( " {b:02x} " ) ) . collect ( )
}
/// Ask the first helper that can see pictures to describe this one: (description, model).
fn describe ( picture : & str , ask : & str , not : & ChatRequest , cancel : & AtomicBool ) -> Option < ( String , String ) > {
let key = picture_key ( picture ) ;
if let Some ( n ) = notes ( ) . lock ( ) . unwrap ( ) . get ( & key ) {
return Some ( n . clone ( ) ) ;
}
let helpers : Vec < Eyes > = EYES . lock ( ) . unwrap ( ) . clone ( ) ;
for e in helpers {
let req = ChatRequest {
kind : e . kind ,
base_url : e . base_url . clone ( ) ,
api_key : e . api_key . clone ( ) ,
model : e . model . clone ( ) ,
system : EYES_PROMPT . into ( ) ,
history : vec ! [ Message { images : vec ! [ picture . to_string ( ) ] , .. Message ::user ( ask ) } ] ,
tools : vec ! [ ] ,
session : String ::new ( ) ,
} ;
let id = vision_key ( & req ) ;
let resting = failed_eyes ( ) . lock ( ) . unwrap ( ) . get ( & id ) . map ( | t | t . elapsed ( ) . as_secs ( ) < 600 ) . unwrap_or ( false ) ;
if ( req . base_url = = not . base_url & & req . model = = not . model ) | | ! sees_images ( & req ) | | resting | | cancel . load ( Ordering ::Relaxed ) {
continue ;
}
DESCRIBING . with ( | d | d . set ( true ) ) ;
let out = run_step ( & req , cancel , & | _ | { } ) ;
DESCRIBING . with ( | d | d . set ( false ) ) ;
match out {
// It may have turned out not to see pictures after all (and answered without).
Ok ( o ) if sees_images ( & req ) & & ! o . text . trim ( ) . is_empty ( ) = > {
let note = ( o . text . trim ( ) . to_string ( ) , e . model . clone ( ) ) ;
let mut all = notes ( ) . lock ( ) . unwrap ( ) ;
if all . len ( ) > 500 {
all . clear ( ) ;
}
all . insert ( key , note . clone ( ) ) ;
if let ( false , Ok ( b ) ) = ( cfg! ( test ) , serde_json ::to_vec ( & * all ) ) {
let _ = std ::fs ::write ( notes_path ( ) , b ) ;
}
return Some ( note ) ;
}
_ = > {
failed_eyes ( ) . lock ( ) . unwrap ( ) . insert ( id , std ::time ::Instant ::now ( ) ) ;
}
}
}
None
}
/// The request for a model that can't see pictures: each of the person's pictures replaced by
/// what a helper saw in it; screenshots by a short note.
fn with_descriptions ( req : & ChatRequest , cancel : & AtomicBool , send : & dyn Fn ( StreamEvent ) ) -> ChatRequest {
let mut r = req . clone ( ) ;
let mut told = false ;
2026-11-05 00:21:34 +00:00
// The newest screenshot is described too (what the bot needs to act next); older ones stay a note.
let newest_shot = r . history . iter ( ) . rposition ( | m | m . role = = Role ::Tool & & ! m . images . is_empty ( ) ) ;
for ( i , m ) in r . history . iter_mut ( ) . enumerate ( ) . filter ( | ( _ , m ) | ! m . images . is_empty ( ) ) {
2026-11-04 13:52:06 +00:00
if m . role = = Role ::Tool {
2026-11-05 00:21:34 +00:00
let pictures = std ::mem ::take ( & mut m . images ) ;
let seen = if Some ( i ) = = newest_shot {
if ! told & & ! notes ( ) . lock ( ) . unwrap ( ) . contains_key ( & picture_key ( & pictures [ 0 ] ) ) {
send ( StreamEvent ::Retrying ( " 這個模型看不到圖片,正在請另一個看得懂圖的模型看這張截圖… " . into ( ) ) ) ;
told = true ;
}
describe ( & pictures [ 0 ] , SCREEN_ASK , req , cancel )
} else {
None
} ;
let add = match seen {
Some ( ( d , model ) ) = > format! ( " (截圖已拍好;你看不到圖片,以下是「 {model} 」看過這張截圖後的描述) \n {d} " ) ,
None = > BLIND_SCREENSHOT_NOTE . to_string ( ) ,
} ;
m . text = if m . text . is_empty ( ) { add } else { format! ( " {} \n {add} " , m . text ) } ;
2026-11-04 13:52:06 +00:00
continue ;
}
let pictures = std ::mem ::take ( & mut m . images ) ;
let ask = format! ( " 請描述這張圖片。使用者附上它時說:「 {} 」 " , m . text . chars ( ) . take ( 300 ) . collect ::< String > ( ) ) ;
let mut added = String ::new ( ) ;
for ( i , p ) in pictures . iter ( ) . enumerate ( ) {
if ! told & & ! notes ( ) . lock ( ) . unwrap ( ) . contains_key ( & picture_key ( p ) ) {
send ( StreamEvent ::Retrying ( " 這個模型看不到圖片,正在請另一個看得懂圖的模型先看一遍… " . into ( ) ) ) ;
told = true ;
}
let which = if pictures . len ( ) > 1 { format! ( " 第 {} 張圖片 " , i + 1 ) } else { " 這張圖片 " . to_string ( ) } ;
added . push_str ( & match describe ( p , & ask , req , cancel ) {
Some ( ( d , model ) ) = > format! ( " \n \n (你看不到圖片;以下是「 {model} 」看過 {which} 後的描述) \n {d} " ) ,
None = > format! ( " \n \n {BLIND_ATTACHMENT_NOTE} (帳號裡也沒有看得懂圖片的模型可以代看。可以請使用者到「設定 › 模型」加一個,例如 Grok、Claude、GPT 或 Gemini。) " ) ,
} ) ;
}
m . text . push_str ( & added ) ;
}
r
}
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
/// The same request with pictures replaced by a short note.
fn without_images ( req : & ChatRequest ) -> ChatRequest {
let mut r = req . clone ( ) ;
for m in r . history . iter_mut ( ) . filter ( | m | ! m . images . is_empty ( ) ) {
m . images . clear ( ) ;
let note = if m . role = = Role ::Tool { BLIND_SCREENSHOT_NOTE } else { BLIND_ATTACHMENT_NOTE } ;
m . text = if m . text . is_empty ( ) { note . to_string ( ) } else { format! ( " {} \n {note} " , m . text ) } ;
}
r
}
pub fn openai_messages ( req : & ChatRequest ) -> Vec < Value > {
let mut messages = vec! [ ] ;
let system = system_with_summaries ( req ) ;
if ! system . trim ( ) . is_empty ( ) {
messages . push ( json! ( { " role " : " system " , " content " : system } ) ) ;
}
let kept = kept_images ( & req . history ) ;
// Tool messages carry text only; their screenshots follow as one user message after the run
// of tool results (a user message between tool results would break the call/answer pairs).
let mut pending : Vec < Value > = vec! [ ] ;
let flush = | messages : & mut Vec < Value > , pending : & mut Vec < Value > | {
if ! pending . is_empty ( ) {
let mut parts = vec! [ json! ( { " type " : " text " , " text " : " (上面工具的螢幕截圖) " } ) ] ;
parts . append ( pending ) ;
messages . push ( json! ( { " role " : " user " , " content " : parts } ) ) ;
}
} ;
for ( i , m ) in req . history . iter ( ) . enumerate ( ) . filter ( | ( _ , m ) | ! m . summary ) {
if m . role ! = Role ::Tool {
flush ( & mut messages , & mut pending ) ;
}
match m . role {
Role ::User if ! m . images . is_empty ( ) = > {
// Attached pictures go with the person's words.
let mut parts = vec! [ json! ( { " type " : " text " , " text " : if m . text . is_empty ( ) { " (附件) " . to_string ( ) } else { m . text . clone ( ) } } ) ] ;
parts . extend ( m . images . iter ( ) . map ( | b | json! ( { " type " : " image_url " , " image_url " : { " url " : format ! ( " data:image/jpeg;base64,{b} " ) } } ) ) ) ;
messages . push ( json! ( { " role " : " user " , " content " : parts } ) ) ;
}
Role ::User = > {
if ! m . text . is_empty ( ) {
messages . push ( json! ( { " role " : " user " , " content " : m . text } ) ) ;
}
}
Role ::Assistant = > {
if m . error & & m . tool_calls . is_empty ( ) {
continue ;
}
if m . tool_calls . is_empty ( ) {
if ! m . text . is_empty ( ) {
messages . push ( json! ( { " role " : " assistant " , " content " : m . text } ) ) ;
}
} else {
let calls : Vec < Value > = m . tool_calls . iter ( ) . map ( | c | json! ( { " id " : c . id , " type " : " function " , " function " : { " name " : c . name , " arguments " : if c . args . trim ( ) . is_empty ( ) { " {} " } else { & c . args } } } ) ) . collect ( ) ;
messages . push ( json! ( { " role " : " assistant " , " content " : if m . text . is_empty ( ) { Value ::Null } else { json! ( m . text ) } , " tool_calls " : calls } ) ) ;
}
}
Role ::Tool = > {
let shown = kept . contains ( & i ) ;
let text = if ! m . images . is_empty ( ) & & ! shown { format! ( " {} \n {OLD_SCREENSHOT_NOTE} " , m . text ) } else { m . text . clone ( ) } ;
messages . push ( json! ( { " role " : " tool " , " tool_call_id " : m . tool_call_id , " content " : text } ) ) ;
if shown {
pending . extend ( m . images . iter ( ) . map ( | b | json! ( { " type " : " image_url " , " image_url " : { " url " : format ! ( " data:image/jpeg;base64,{b} " ) } } ) ) ) ;
}
}
}
}
flush ( & mut messages , & mut pending ) ;
messages
}
fn stream_openai_compatible ( req : & ChatRequest , cancel : & AtomicBool , send : & dyn Fn ( StreamEvent ) ) -> Result < StepOutput , Failure > {
let mut body = json! ( { " model " : req . model , " messages " : openai_messages ( req ) , " stream " : true } ) ;
if ! req . tools . is_empty ( ) {
body [ " tools " ] = json! ( req . tools . iter ( ) . map ( | t | json! ( { " type " : " function " , " function " : t } ) ) . collect ::< Vec < _ > > ( ) ) ;
}
let url = format! ( " {} /chat/completions " , req . base_url ) ;
2026-11-05 07:00:31 +00:00
let mut headers : Vec < ( String , String ) > = vec! [ ] ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
if ! req . api_key . is_empty ( ) {
2026-11-05 07:00:31 +00:00
headers . push ( ( " Authorization " . into ( ) , format! ( " Bearer {} " , req . api_key ) ) ) ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
}
if req . kind = = ProviderKind ::OpenRouter {
2026-11-05 07:00:31 +00:00
headers . push ( ( " X-Title " . into ( ) , " AnyBot " . into ( ) ) ) ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
}
if req . kind = = ProviderKind ::OpenCodeGo {
2026-11-05 07:00:31 +00:00
headers . push ( ( " x-opencode-session " . into ( ) , opencode_session ( req ) ) ) ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
}
2026-11-05 07:00:31 +00:00
// A brain on a server, a model on the person's own network: through their computer (the
// whole answer comes back at once, so no word-by-word streaming).
let via_hands : Option < Vec < u8 > > = match crate ::remote ::model_hands_for ( & url ) {
Some ( hand ) = > {
let ( status , bytes ) = crate ::remote ::model_via_hands ( & hand , & url , & headers , & body . to_string ( ) ) . map_err ( Failure ::retry ) ? ;
if status > = 400 {
return Err ( http_error ( status , & String ::from_utf8_lossy ( & bytes ) , None ) ) ;
}
Some ( bytes )
}
None = > None ,
} ;
let resp = match via_hands {
Some ( _ ) = > None ,
None = > {
let mut r = agent ( ) . post ( & url ) . header ( " Content-Type " , " application/json " ) ;
for ( k , v ) in & headers {
r = r . header ( k , v ) ;
}
// SAND-E0418 turnBackendUnreachable: retryable.
let resp = r . send_json ( & body ) . map_err ( | e | Failure ::retry ( connect_error ( req . kind , & e . to_string ( ) ) ) ) ? ;
let status = resp . status ( ) . as_u16 ( ) ;
if status > = 400 {
let ra = retry_after_ms ( & resp ) ;
let text = resp . into_body ( ) . read_to_string ( ) . unwrap_or_default ( ) ;
return Err ( http_error ( status , & text , ra ) ) ;
}
Some ( resp )
}
} ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
let mut out = StepOutput ::default ( ) ;
// Tool calls arrive in pieces keyed by index.
let mut pending : Vec < ( String , String , String ) > = vec! [ ] ;
let mut error = None ;
2026-11-05 07:00:31 +00:00
let reader : Box < dyn std ::io ::Read > = match ( resp , via_hands ) {
( Some ( r ) , _ ) = > Box ::new ( r . into_body ( ) . into_reader ( ) ) ,
( None , b ) = > Box ::new ( std ::io ::Cursor ::new ( b . unwrap_or_default ( ) ) ) ,
} ;
read_sse_from ( reader , cancel , send , | _ev , data | {
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
if data = = " [DONE] " {
return false ;
}
if let Ok ( v ) = serde_json ::from_str ::< Value > ( data ) {
if let Some ( e ) = v . get ( " error " ) {
error = Some ( e . get ( " message " ) . and_then ( Value ::as_str ) . unwrap_or ( " 未知錯誤 " ) . to_string ( ) ) ;
return false ;
}
let delta = & v [ " choices " ] [ 0 ] [ " delta " ] ;
for k in [ " reasoning_content " , " reasoning " ] {
if let Some ( t ) = delta . get ( k ) . and_then ( Value ::as_str ) {
if ! t . is_empty ( ) {
out . thinking . push_str ( t ) ;
send ( StreamEvent ::Thinking ( t . to_string ( ) ) ) ;
}
}
}
if let Some ( t ) = delta . get ( " content " ) . and_then ( Value ::as_str ) {
if ! t . is_empty ( ) {
out . text . push_str ( t ) ;
send ( StreamEvent ::Text ( t . to_string ( ) ) ) ;
}
}
if let Some ( calls ) = delta . get ( " tool_calls " ) . and_then ( Value ::as_array ) {
for c in calls {
2026-11-05 07:48:20 +00:00
let i = match c [ " index " ] . as_u64 ( ) {
Some ( i ) = > i as usize ,
// Some OpenAI-compatible proxies drop the index: a part carrying an
// id or a name starts a new call; an arguments-only part continues the
// last one (starting a fresh slot used to lose the call's tail).
None = > {
let starts = c [ " id " ] . is_string ( ) | | c [ " function " ] [ " name " ] . is_string ( ) ;
if starts | | pending . is_empty ( ) {
pending . len ( )
} else {
pending . len ( ) - 1
}
}
} ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
while pending . len ( ) < = i {
pending . push ( Default ::default ( ) ) ;
}
if let Some ( id ) = c [ " id " ] . as_str ( ) {
pending [ i ] . 0 = id . to_string ( ) ;
}
if let Some ( n ) = c [ " function " ] [ " name " ] . as_str ( ) {
pending [ i ] . 1. push_str ( n ) ;
}
if let Some ( a ) = c [ " function " ] [ " arguments " ] . as_str ( ) {
pending [ i ] . 2. push_str ( a ) ;
}
}
}
}
true
} ) ? ;
if let Some ( e ) = error {
2026-11-06 11:47:43 +00:00
return Err ( stream_failure ( & e ) ) ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
}
for ( i , ( id , name , args ) ) in pending . into_iter ( ) . enumerate ( ) {
if name . is_empty ( ) {
continue ;
}
let id = if id . is_empty ( ) { format! ( " call_ {i} " ) } else { id } ;
let call = ToolCall { id , name , args , status : ToolStatus ::Waiting } ;
send ( StreamEvent ::ToolCall ( call . clone ( ) ) ) ;
out . tool_calls . push ( call ) ;
}
Ok ( out )
}
pub fn anthropic_messages ( req : & ChatRequest ) -> Vec < Value > {
let mut messages : Vec < Value > = vec! [ ] ;
let mut push = | role : & str , blocks : Vec < Value > | {
if blocks . is_empty ( ) {
return ;
}
// The API wants user/assistant turns to alternate; merge repeats.
if let Some ( last ) = messages . last_mut ( ) {
if last [ " role " ] = = role {
last [ " content " ] . as_array_mut ( ) . unwrap ( ) . extend ( blocks ) ;
return ;
}
}
messages . push ( json! ( { " role " : role , " content " : blocks } ) ) ;
} ;
let kept = kept_images ( & req . history ) ;
for ( i , m ) in req . history . iter ( ) . enumerate ( ) . filter ( | ( _ , m ) | ! m . summary ) {
match m . role {
Role ::Tool if ! m . images . is_empty ( ) = > {
let content = if kept . contains ( & i ) {
let mut c = vec! [ json! ( { " type " : " text " , " text " : m . text } ) ] ;
c . extend ( m . images . iter ( ) . map ( | b | json! ( { " type " : " image " , " source " : { " type " : " base64 " , " media_type " : " image/jpeg " , " data " : b } } ) ) ) ;
json! ( c )
} else {
json! ( format! ( " {} \n {OLD_SCREENSHOT_NOTE} " , m . text ) )
} ;
push ( " user " , vec! [ json! ( { " type " : " tool_result " , " tool_use_id " : m . tool_call_id , " content " : content } ) ] ) ;
}
Role ::User if ! m . images . is_empty ( ) = > {
let mut b = vec! [ json! ( { " type " : " text " , " text " : if m . text . is_empty ( ) { " (附件) " . to_string ( ) } else { m . text . clone ( ) } } ) ] ;
b . extend ( m . images . iter ( ) . map ( | x | json! ( { " type " : " image " , " source " : { " type " : " base64 " , " media_type " : " image/jpeg " , " data " : x } } ) ) ) ;
push ( " user " , b ) ;
}
Role ::User if ! m . text . is_empty ( ) = > push ( " user " , vec! [ json! ( { " type " : " text " , " text " : m . text } ) ] ) ,
Role ::User = > { }
Role ::Assistant = > {
if ! m . blocks . is_empty ( ) {
push ( " assistant " , m . blocks . clone ( ) ) ;
} else if ! m . error | | ! m . tool_calls . is_empty ( ) {
let mut b = vec! [ ] ;
if ! m . text . is_empty ( ) {
b . push ( json! ( { " type " : " text " , " text " : m . text } ) ) ;
}
for c in & m . tool_calls {
b . push ( json! ( { " type " : " tool_use " , " id " : c . id , " name " : c . name , " input " : serde_json ::from_str ::< Value > ( & c . args ) . unwrap_or ( json! ( { } ) ) } ) ) ;
}
push ( " assistant " , b ) ;
}
}
Role ::Tool = > push ( " user " , vec! [ json! ( { " type " : " tool_result " , " tool_use_id " : m . tool_call_id , " content " : m . text } ) ] ) ,
}
}
messages
}
fn stream_anthropic ( req : & ChatRequest , cancel : & AtomicBool , send : & dyn Fn ( StreamEvent ) ) -> Result < StepOutput , Failure > {
let mut body = json! ( {
" model " : req . model ,
" max_tokens " : MAX_OUTPUT_TOKENS_ANTHROPIC ,
" messages " : anthropic_messages ( req ) ,
" stream " : true ,
} ) ;
let system = system_with_summaries ( req ) ;
if ! system . trim ( ) . is_empty ( ) {
body [ " system " ] = json! ( system ) ;
}
if claude_adaptive ( & req . model ) {
body [ " thinking " ] = json! ( { " type " : " adaptive " , " display " : " summarized " } ) ;
}
if ! req . tools . is_empty ( ) {
body [ " tools " ] = json! ( req . tools . iter ( ) . map ( | t | json! ( { " name " : t [ " name " ] , " description " : t [ " description " ] , " input_schema " : t [ " parameters " ] } ) ) . collect ::< Vec < _ > > ( ) ) ;
}
let mut r = agent ( )
. post ( & format! ( " {} /messages " , req . base_url ) )
. header ( " Content-Type " , " application/json " )
. header ( " x-api-key " , & req . api_key )
. header ( " anthropic-version " , ANTHROPIC_VERSION ) ;
if req . kind = = ProviderKind ::OpenCodeGo {
// Its Anthropic-style door takes the key as a bearer token too.
r = r . header ( " Authorization " , & format! ( " Bearer {} " , req . api_key ) ) . header ( " x-opencode-session " , & opencode_session ( req ) ) ;
}
if claude_fallbacks ( & req . model ) {
body [ " fallbacks " ] = json! ( " default " ) ;
r = r . header ( " anthropic-beta " , " server-side-fallback-2026-07-01 " ) ;
}
// SAND-E0418 turnBackendUnreachable: retryable.
let resp = r . send_json ( & body ) . map_err ( | e | Failure ::retry ( connect_error ( req . kind , & e . to_string ( ) ) ) ) ? ;
let status = resp . status ( ) . as_u16 ( ) ;
if status > = 400 {
let ra = retry_after_ms ( & resp ) ;
let text = resp . into_body ( ) . read_to_string ( ) . unwrap_or_default ( ) ;
return Err ( http_error ( status , & text , ra ) ) ;
}
let mut out = StepOutput ::default ( ) ;
let mut blocks : Vec < Value > = vec! [ ] ;
let mut partial_json : Vec < String > = vec! [ ] ;
let mut error = None ;
read_sse ( resp . into_body ( ) , cancel , send , | ev , data | {
let Ok ( v ) = serde_json ::from_str ::< Value > ( data ) else { return true } ;
let kind = if ev . is_empty ( ) { v [ " type " ] . as_str ( ) . unwrap_or ( " " ) } else { ev } ;
match kind {
" content_block_start " = > {
let i = v [ " index " ] . as_u64 ( ) . unwrap_or ( blocks . len ( ) as u64 ) as usize ;
while blocks . len ( ) < = i {
blocks . push ( Value ::Null ) ;
partial_json . push ( String ::new ( ) ) ;
}
blocks [ i ] = v [ " content_block " ] . clone ( ) ;
}
" content_block_delta " = > {
let i = v [ " index " ] . as_u64 ( ) . unwrap_or ( 0 ) as usize ;
let d = & v [ " delta " ] ;
let block = blocks . get_mut ( i ) ;
match d [ " type " ] . as_str ( ) {
Some ( " text_delta " ) = > {
let t = d [ " text " ] . as_str ( ) . unwrap_or ( " " ) ;
out . text . push_str ( t ) ;
if let Some ( b ) = block {
let joined = format! ( " {} {} " , b [ " text " ] . as_str ( ) . unwrap_or ( " " ) , t ) ;
b [ " text " ] = json! ( joined ) ;
}
send ( StreamEvent ::Text ( t . to_string ( ) ) ) ;
}
Some ( " thinking_delta " ) = > {
let t = d [ " thinking " ] . as_str ( ) . unwrap_or ( " " ) ;
out . thinking . push_str ( t ) ;
if let Some ( b ) = block {
let joined = format! ( " {} {} " , b [ " thinking " ] . as_str ( ) . unwrap_or ( " " ) , t ) ;
b [ " thinking " ] = json! ( joined ) ;
}
send ( StreamEvent ::Thinking ( t . to_string ( ) ) ) ;
}
Some ( " signature_delta " ) = > {
if let Some ( b ) = block {
let joined = format! ( " {} {} " , b [ " signature " ] . as_str ( ) . unwrap_or ( " " ) , d [ " signature " ] . as_str ( ) . unwrap_or ( " " ) ) ;
b [ " signature " ] = json! ( joined ) ;
}
}
Some ( " input_json_delta " ) = > {
if let Some ( p ) = partial_json . get_mut ( i ) {
p . push_str ( d [ " partial_json " ] . as_str ( ) . unwrap_or ( " " ) ) ;
}
}
_ = > { }
}
}
" content_block_stop " = > {
let i = v [ " index " ] . as_u64 ( ) . unwrap_or ( 0 ) as usize ;
if let Some ( b ) = blocks . get_mut ( i ) {
if b [ " type " ] = = " tool_use " {
let raw = partial_json . get ( i ) . cloned ( ) . unwrap_or_default ( ) ;
let input : Value = if raw . trim ( ) . is_empty ( ) { json! ( { } ) } else { serde_json ::from_str ( & raw ) . unwrap_or ( json! ( { } ) ) } ;
b [ " input " ] = input . clone ( ) ;
let call = ToolCall { id : b [ " id " ] . as_str ( ) . unwrap_or ( " " ) . into ( ) , name : b [ " name " ] . as_str ( ) . unwrap_or ( " " ) . into ( ) , args : input . to_string ( ) , status : ToolStatus ::Waiting } ;
send ( StreamEvent ::ToolCall ( call . clone ( ) ) ) ;
out . tool_calls . push ( call ) ;
}
}
}
" message_delta " = > {
if v [ " delta " ] [ " stop_reason " ] = = " refusal " {
out . text . push_str ( " \n \n (模型拒絕回答這個要求。) " ) ;
send ( StreamEvent ::Text ( " \n \n (模型拒絕回答這個要求。) " . into ( ) ) ) ;
} else if v [ " delta " ] [ " stop_reason " ] = = " max_tokens " {
out . text . push_str ( " \n \n (回答太長,已達長度上限。) " ) ;
send ( StreamEvent ::Text ( " \n \n (回答太長,已達長度上限。) " . into ( ) ) ) ;
}
}
" message_stop " = > return false ,
" error " = > {
error = Some ( v [ " error " ] [ " message " ] . as_str ( ) . unwrap_or ( " 未知錯誤 " ) . to_string ( ) ) ;
return false ;
}
_ = > { }
}
true
} ) ? ;
if let Some ( e ) = error {
2026-11-06 11:47:43 +00:00
return Err ( stream_failure ( & e ) ) ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
}
out . blocks = blocks . into_iter ( ) . filter ( | b | ! b . is_null ( ) ) . collect ( ) ;
Ok ( out )
}
fn connect_error ( kind : ProviderKind , e : & str ) -> String {
if kind = = ProviderKind ::Ollama {
format! ( " 連不上 Ollama: {e} \n 請先開啟 Ollama, 並用 `ollama pull` 下載模型。 " )
} else {
format! ( " 連不上 {} : {e} " , kind . label ( ) )
}
}
/// OpenCode Go: which API style each model speaks, from the public models list (fetched once).
static OPENCODE_STYLES : std ::sync ::Mutex < Option < std ::collections ::HashMap < String , String > > > = std ::sync ::Mutex ::new ( None ) ;
fn remember_opencode_styles ( _base : & str , list : & Value ) {
let map = opencode_styles_of ( list ) ;
if ! map . is_empty ( ) {
* OPENCODE_STYLES . lock ( ) . unwrap ( ) = Some ( map ) ;
}
}
/// {"data":[{"id":"qwen3.8-max","endpoint":{"messages":{…}}}, …]} → id → "messages" | "chat-completions" | …
pub fn opencode_styles_of ( list : & Value ) -> std ::collections ::HashMap < String , String > {
list [ " data " ]
. as_array ( )
. into_iter ( )
. flatten ( )
. filter_map ( | m | {
let id = m [ " id " ] . as_str ( ) ? ;
let ep = m [ " endpoint " ] . as_object ( ) ? ;
// A model with a chat-completions door uses it (AnyBot's best-tested path); else the first.
let style = if ep . contains_key ( " chat-completions " ) { " chat-completions " } else if ep . contains_key ( " messages " ) { " messages " } else { ep . keys ( ) . next ( ) ? } ;
Some ( ( id . to_string ( ) , style . to_string ( ) ) )
} )
. collect ( )
}
/// The API style for a model: "messages" (Anthropic-style) or "chat-completions" (everything
/// else; OpenCode translates for models whose own style is Responses or Gemini).
pub fn opencode_go_style ( base : & str , model : & str ) -> String {
let known = OPENCODE_STYLES . lock ( ) . unwrap ( ) . as_ref ( ) . and_then ( | m | m . get ( model ) . cloned ( ) ) ;
let style = known . or_else ( | | {
let v : Value = agent ( ) . get ( & format! ( " {base} /v1/models " ) ) . call ( ) . ok ( ) ? . into_body ( ) . read_json ( ) . ok ( ) ? ;
remember_opencode_styles ( base , & v ) ;
OPENCODE_STYLES . lock ( ) . unwrap ( ) . as_ref ( ) . and_then ( | m | m . get ( model ) . cloned ( ) )
} ) ;
style . unwrap_or_else ( | | {
// Offline guess from the published list: Claude, Qwen and MiniMax M3 are Anthropic-style.
let m = model . to_lowercase ( ) ;
if m . starts_with ( " claude " ) | | m . starts_with ( " qwen " ) | | m = = " minimax-m3 " { " messages " . into ( ) } else { " chat-completions " . into ( ) }
} )
}
fn opencode_session ( req : & ChatRequest ) -> String {
if req . session . is_empty ( ) { " anybot " . into ( ) } else { req . session . clone ( ) }
}
/// Ask the provider which models it offers.
pub fn list_models ( kind : ProviderKind , base_url : & str , api_key : & str ) -> Result < Vec < String > , String > {
if kind . needs_key ( ) & & api_key . is_empty ( ) {
return Err ( " 請先貼上 API 金鑰 " . into ( ) ) ;
}
let url = if kind = = ProviderKind ::OpenCodeGo { format! ( " {base_url} /v1/models " ) } else { format! ( " {base_url} /models " ) } ;
let mut r = agent ( ) . get ( & url ) ;
if kind = = ProviderKind ::Anthropic {
r = r . header ( " x-api-key " , api_key ) . header ( " anthropic-version " , ANTHROPIC_VERSION ) ;
} else if ! api_key . is_empty ( ) {
r = r . header ( " Authorization " , & format! ( " Bearer {api_key} " ) ) ;
}
let resp = r . call ( ) . map_err ( | e | connect_error ( kind , & e . to_string ( ) ) ) ? ;
let status = resp . status ( ) . as_u16 ( ) ;
let text = resp . into_body ( ) . read_to_string ( ) . map_err ( | e | e . to_string ( ) ) ? ;
if status > = 400 {
return Err ( http_error ( status , & text , None ) . msg ) ;
}
let v : Value = serde_json ::from_str ( & text ) . map_err ( | e | e . to_string ( ) ) ? ;
if kind = = ProviderKind ::OpenCodeGo {
remember_opencode_styles ( base_url , & v ) ;
}
let list = v [ " data " ] . as_array ( ) . or_else ( | | v [ " models " ] . as_array ( ) ) . cloned ( ) . unwrap_or_default ( ) ;
let mut ids : Vec < String > = list
. iter ( )
. filter_map ( | m | m [ " id " ] . as_str ( ) . or_else ( | | m [ " name " ] . as_str ( ) ) )
. map ( | s | s . trim_start_matches ( " models/ " ) . to_string ( ) )
. collect ( ) ;
ids . sort ( ) ;
ids . dedup ( ) ;
if ids . is_empty ( ) {
return Err ( " 供應商沒有回傳任何模型 " . into ( ) ) ;
}
Ok ( ids )
}
#[ cfg(test) ]
mod tests {
use super ::* ;
use std ::io ::{ Read , Write } ;
use std ::net ::TcpListener ;
/// Serve one canned SSE reply on a local port and return its base URL.
pub fn fake_server ( body : & 'static str ) -> String {
let l = TcpListener ::bind ( " 127.0.0.1:0 " ) . unwrap ( ) ;
let port = l . local_addr ( ) . unwrap ( ) . port ( ) ;
std ::thread ::spawn ( move | | {
let ( mut s , _ ) = l . accept ( ) . unwrap ( ) ;
// Read the whole request (headers + body) before answering.
let mut req = Vec ::new ( ) ;
let mut buf = [ 0 u8 ; 4096 ] ;
loop {
let n = s . read ( & mut buf ) . unwrap_or ( 0 ) ;
if n = = 0 {
break ;
}
req . extend_from_slice ( & buf [ .. n ] ) ;
let text = String ::from_utf8_lossy ( & req ) . to_string ( ) ;
if let Some ( end ) = text . find ( " \r \n \r \n " ) {
let len = text [ .. end ]
. lines ( )
. find_map ( | l | l . to_ascii_lowercase ( ) . strip_prefix ( " content-length: " ) . map ( | v | v . trim ( ) . parse ::< usize > ( ) . unwrap_or ( 0 ) ) )
. unwrap_or ( 0 ) ;
if req . len ( ) > = end + 4 + len {
break ;
}
}
}
let head = " HTTP/1.1 200 OK \r \n Content-Type: text/event-stream \r \n Connection: close \r \n \r \n " ;
let _ = s . write_all ( head . as_bytes ( ) ) ;
let _ = s . write_all ( body . as_bytes ( ) ) ;
} ) ;
format! ( " http://127.0.0.1: {port} /v1 " )
}
fn run ( kind : ProviderKind , body : & 'static str ) -> StepOutput {
let req = ChatRequest {
kind ,
base_url : fake_server ( body ) ,
api_key : " test " . into ( ) ,
model : " m " . into ( ) ,
system : " s " . into ( ) ,
history : vec ! [ Message ::user ( " hi " ) ] ,
tools : crate ::tools ::specs ( ) ,
session : String ::new ( ) ,
} ;
let cancel = AtomicBool ::new ( false ) ;
match run_step ( & req , & cancel , & | _ | { } ) {
Ok ( o ) = > o ,
Err ( e ) = > panic! ( " {e} " ) ,
}
}
2026-11-06 11:47:43 +00:00
#[ test ]
fn a_busy_model_mid_stream_is_waited_out ( ) {
let f = stream_failure ( " The model is currently at capacity due to high demand. Please try again in a few minutes " ) ;
assert! ( f . retryable & & f . retry_after_ms = = Some ( BUSY_RETRY_MS ) & & f . msg . starts_with ( " 模型供應商現在太忙 " ) , " {} " , f . msg ) ;
let f = stream_failure ( " Invalid tool schema " ) ;
assert! ( ! f . retryable & & f . msg . starts_with ( " 模型回報錯誤 " ) , " {} " , f . msg ) ;
}
2026-11-06 08:51:25 +00:00
#[ test ]
fn a_gateways_error_page_reads_as_words ( ) {
let page = " <!DOCTYPE HTML> \n <html lang= \" en \" > \n <head> \n <meta charset= \" utf-8 \" > \n <style type= \" text/css \" > \n :root { \n color-scheme: light dark; \n } \n </style> \n <title>Error response</title> \n </head> \n <body> \n <h1>Error response</h1> \n <p>Error code: 502</p> \n <p>Message: Bad Gateway.</p> \n </body> \n </html> \n " ;
let f = http_error ( 502 , page , None ) ;
assert! ( f . retryable ) ;
assert! ( ! f . msg . contains ( '<' ) & & ! f . msg . contains ( " color-scheme " ) , " {} " , f . msg ) ;
assert! ( f . msg . contains ( " Bad Gateway " ) , " {} " , f . msg ) ;
assert_eq! ( page_words ( " upstream timed out " ) , " upstream timed out " ) ;
}
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
#[ test ]
fn backoff_matches_original ( ) {
assert_eq! ( [ backoff_delay_ms ( 1 ) , backoff_delay_ms ( 2 ) , backoff_delay_ms ( 3 ) ] , [ 1000 , 2000 , 4000 ] ) ;
assert! ( http_error ( 429 , " {} " , None ) . retryable ) ;
assert! ( http_error ( 503 , " {} " , Some ( 2000 ) ) . retry_after_ms = = Some ( 2000 ) ) ;
assert! ( ! http_error ( 401 , " {} " , None ) . retryable ) ;
assert! ( ! http_error ( 400 , " { \" error \" :{ \" message \" : \" prompt is too long \" }} " , None ) . retryable ) ;
}
#[ test ]
fn openai_style_stream ( ) {
let body = " data: { \" choices \" :[{ \" delta \" :{ \" reasoning_content \" : \" hm \" }}]} \n \n data: { \" choices \" :[{ \" delta \" :{ \" content \" : \" 你好 \" }}]} \n \n data: { \" choices \" :[{ \" delta \" :{ \" content \" : \" ! \" }}]} \n \n data: [DONE] \n \n " ;
let o = run ( ProviderKind ::Xai , body ) ;
assert_eq! ( o . text , " 你好! " ) ;
assert_eq! ( o . thinking , " hm " ) ;
}
#[ test ]
fn openai_tool_calls_in_pieces ( ) {
let body = " data: { \" choices \" :[{ \" delta \" :{ \" tool_calls \" :[{ \" index \" :0, \" id \" : \" call_a \" , \" function \" :{ \" name \" : \" read_file \" , \" arguments \" : \" { \\ \" pa \" }}]}}]} \n \n data: { \" choices \" :[{ \" delta \" :{ \" tool_calls \" :[{ \" index \" :0, \" function \" :{ \" arguments \" : \" th \\ \" : \\ \" a.txt \\ \" } \" }}]}}]} \n \n data: [DONE] \n \n " ;
let o = run ( ProviderKind ::OpenAi , body ) ;
assert_eq! ( o . tool_calls . len ( ) , 1 ) ;
assert_eq! ( o . tool_calls [ 0 ] . name , " read_file " ) ;
assert_eq! ( o . tool_calls [ 0 ] . args , " { \" path \" : \" a.txt \" } " ) ;
}
#[ test ]
fn claude_style_stream ( ) {
let body = " event: message_start \n data: { \" type \" : \" message_start \" } \n \n event: content_block_start \n data: { \" type \" : \" content_block_start \" , \" index \" :0, \" content_block \" :{ \" type \" : \" thinking \" , \" thinking \" : \" \" }} \n \n event: content_block_delta \n data: { \" type \" : \" content_block_delta \" , \" index \" :0, \" delta \" :{ \" type \" : \" thinking_delta \" , \" thinking \" : \" ok \" }} \n \n event: content_block_delta \n data: { \" type \" : \" content_block_delta \" , \" index \" :0, \" delta \" :{ \" type \" : \" signature_delta \" , \" signature \" : \" sig \" }} \n \n event: content_block_start \n data: { \" type \" : \" content_block_start \" , \" index \" :1, \" content_block \" :{ \" type \" : \" text \" , \" text \" : \" \" }} \n \n event: content_block_delta \n data: { \" type \" : \" content_block_delta \" , \" index \" :1, \" delta \" :{ \" type \" : \" text_delta \" , \" text \" : \" Hello \" }} \n \n event: content_block_start \n data: { \" type \" : \" content_block_start \" , \" index \" :2, \" content_block \" :{ \" type \" : \" tool_use \" , \" id \" : \" tu1 \" , \" name \" : \" list_dir \" , \" input \" :{}}} \n \n event: content_block_delta \n data: { \" type \" : \" content_block_delta \" , \" index \" :2, \" delta \" :{ \" type \" : \" input_json_delta \" , \" partial_json \" : \" { \\ \" path \\ \" : \\ \" x \\ \" } \" }} \n \n event: content_block_stop \n data: { \" type \" : \" content_block_stop \" , \" index \" :2} \n \n event: message_stop \n data: { \" type \" : \" message_stop \" } \n \n " ;
let o = run ( ProviderKind ::Anthropic , body ) ;
assert_eq! ( o . text , " Hello " ) ;
assert_eq! ( o . thinking , " ok " ) ;
assert_eq! ( o . tool_calls [ 0 ] . name , " list_dir " ) ;
assert_eq! ( o . blocks . len ( ) , 3 ) ;
assert_eq! ( o . blocks [ 0 ] [ " signature " ] , " sig " ) ;
assert_eq! ( o . blocks [ 2 ] [ " input " ] [ " path " ] , " x " ) ;
}
#[ test ]
fn history_keeps_tool_turns ( ) {
let mut a = Message ::assistant ( " " , " x " ) ;
a . tool_calls = vec! [ ToolCall { id : " c1 " . into ( ) , name : " list_dir " . into ( ) , args : " {} " . into ( ) , status : ToolStatus ::Done } ] ;
let mut s = Message ::assistant ( " old stuff " , " " ) ;
s . summary = true ;
let req = ChatRequest {
kind : ProviderKind ::Anthropic ,
base_url : String ::new ( ) ,
api_key : String ::new ( ) ,
model : " m " . into ( ) ,
system : " sys " . into ( ) ,
history : vec ! [ s , Message ::user ( " hi " ) , a , Message ::tool_result ( " c1 " , " list_dir " , " a.txt " ) , Message ::user ( " next " ) ] ,
tools : vec ! [ ] ,
session : String ::new ( ) ,
} ;
let m = anthropic_messages ( & req ) ;
assert_eq! ( m . len ( ) , 3 ) ;
assert_eq! ( m [ 1 ] [ " content " ] [ 0 ] [ " type " ] , " tool_use " ) ;
assert_eq! ( m [ 2 ] [ " content " ] [ 0 ] [ " type " ] , " tool_result " ) ;
assert_eq! ( m [ 2 ] [ " content " ] [ 1 ] [ " text " ] , " next " ) ;
let o = openai_messages ( & ChatRequest { kind : ProviderKind ::OpenAi , .. req } ) ;
assert! ( o [ 0 ] [ " content " ] . as_str ( ) . unwrap ( ) . contains ( " old stuff " ) ) ;
assert_eq! ( o [ 3 ] [ " role " ] , " tool " ) ;
}
#[ test ]
fn screenshots_reach_the_model_and_old_ones_drop ( ) {
let n = crate ::computer ::KEEP_SCREENSHOTS + 1 ;
let mut history = vec! [ Message ::user ( " look " ) ] ;
for i in 0 .. n {
let mut a = Message ::assistant ( " " , " x " ) ;
a . tool_calls = vec! [ ToolCall { id : format ! ( " s{i} " ) , name : " computer_use_screenshot " . into ( ) , args : " {} " . into ( ) , status : ToolStatus ::Done } ] ;
history . push ( a ) ;
let mut r = Message ::tool_result ( & format! ( " s {i} " ) , " computer_use_screenshot " , " shot " ) ;
r . images = vec! [ format! ( " IMG {i} " ) ] ;
history . push ( r ) ;
}
let req = ChatRequest { kind : ProviderKind ::Anthropic , base_url : String ::new ( ) , api_key : String ::new ( ) , model : " m " . into ( ) , system : String ::new ( ) , history , tools : vec ! [ ] , session : String ::new ( ) } ;
let m = anthropic_messages ( & req ) ;
let all = serde_json ::to_string ( & m ) . unwrap ( ) ;
assert! ( ! all . contains ( " IMG0 " ) & & all . contains ( OLD_SCREENSHOT_NOTE ) ) ;
assert! ( all . contains ( & format! ( " IMG {} " , n - 1 ) ) & & all . contains ( " \" media_type \" : \" image/jpeg \" " ) ) ;
let o = openai_messages ( & ChatRequest { kind : ProviderKind ::OpenAi , .. req } ) ;
// Each tool answer is followed by its picture as a user message (never between tool results).
for w in o . windows ( 2 ) {
if w [ 0 ] [ " role " ] = = " user " & & w [ 0 ] [ " content " ] . is_array ( ) & & w [ 1 ] [ " role " ] = = " tool " {
panic! ( " picture between tool results " ) ;
}
}
let pics : Vec < & Value > = o . iter ( ) . filter ( | x | x [ " role " ] = = " user " & & x [ " content " ] . is_array ( ) ) . collect ( ) ;
assert_eq! ( pics . len ( ) , crate ::computer ::KEEP_SCREENSHOTS ) ;
assert! ( pics . last ( ) . unwrap ( ) [ " content " ] [ 1 ] [ " image_url " ] [ " url " ] . as_str ( ) . unwrap ( ) . starts_with ( " data:image/jpeg;base64,IMG " ) ) ;
}
/// A server that records the request it gets, then answers with `body`.
fn recording_server ( body : & 'static str ) -> ( String , std ::sync ::mpsc ::Receiver < String > ) {
let l = TcpListener ::bind ( " 127.0.0.1:0 " ) . unwrap ( ) ;
let port = l . local_addr ( ) . unwrap ( ) . port ( ) ;
let ( tx , rx ) = std ::sync ::mpsc ::channel ( ) ;
std ::thread ::spawn ( move | | {
let ( mut s , _ ) = l . accept ( ) . unwrap ( ) ;
let mut req = Vec ::new ( ) ;
let mut buf = [ 0 u8 ; 8192 ] ;
loop {
let n = s . read ( & mut buf ) . unwrap_or ( 0 ) ;
if n = = 0 {
break ;
}
req . extend_from_slice ( & buf [ .. n ] ) ;
let text = String ::from_utf8_lossy ( & req ) . to_string ( ) ;
if let Some ( end ) = text . find ( " \r \n \r \n " ) {
let len = text [ .. end ] . lines ( ) . find_map ( | l | l . to_ascii_lowercase ( ) . strip_prefix ( " content-length: " ) . map ( | v | v . trim ( ) . parse ::< usize > ( ) . unwrap_or ( 0 ) ) ) . unwrap_or ( 0 ) ;
if req . len ( ) > = end + 4 + len {
break ;
}
}
}
let _ = tx . send ( String ::from_utf8_lossy ( & req ) . to_string ( ) ) ;
let _ = s . write_all ( b " HTTP/1.1 200 OK \r \n Content-Type: text/event-stream \r \n Connection: close \r \n \r \n " ) ;
let _ = s . write_all ( body . as_bytes ( ) ) ;
} ) ;
( format! ( " http://127.0.0.1: {port} " ) , rx )
}
#[ test ]
fn opencode_go_routes_each_model_its_way ( ) {
// The public list's shape (trimmed).
let list : Value = serde_json ::from_str ( r #" { " data " :[
{ " id " :" kimi-k3 " , " endpoint " :{ " chat-completions " :{ " base_url " :" https://opencode.ai/inference/go/openai/v1 " } } } ,
{ " id " :" qwen3.8-max " , " endpoint " :{ " messages " :{ " base_url " :" https://opencode.ai/inference/go/anthropic/v1 " } } } ,
{ " id " :" grok-4.7 " , " endpoint " :{ " responses " :{ " base_url " :" https://opencode.ai/inference/go/openai/v1 " } } } ] } " #).unwrap();
let styles = opencode_styles_of ( & list ) ;
assert_eq! ( styles [ " kimi-k3 " ] , " chat-completions " ) ;
assert_eq! ( styles [ " qwen3.8-max " ] , " messages " ) ;
assert_eq! ( styles [ " grok-4.7 " ] , " responses " ) ;
remember_opencode_styles ( " " , & list ) ;
let cancel = AtomicBool ::new ( false ) ;
let ask = | model : & str , body : & 'static str | {
let ( base , rx ) = recording_server ( body ) ;
let req = ChatRequest { kind : ProviderKind ::OpenCodeGo , base_url : base , api_key : " sk-go " . into ( ) , model : model . into ( ) , system : " s " . into ( ) , history : vec ! [ Message ::user ( " hi " ) ] , tools : vec ! [ ] , session : " anybot-bot-7 " . into ( ) } ;
let out = run_step ( & req , & cancel , & | _ | { } ) . unwrap ( ) ;
( out , rx . recv ( ) . unwrap ( ) )
} ;
let ( out , seen ) = ask ( " kimi-k3 " , " data: { \" choices \" :[{ \" delta \" :{ \" content \" : \" 你好 \" }}]} \n \n data: [DONE] \n \n " ) ;
assert_eq! ( out . text , " 你好 " ) ;
assert! ( seen . starts_with ( " POST /openai/v1/chat/completions " ) , " {seen} " ) ;
let low = seen . to_lowercase ( ) ;
assert! ( low . contains ( " authorization: bearer sk-go " ) & & low . contains ( " x-opencode-session: anybot-bot-7 " ) , " {seen} " ) ;
let ( out , seen ) = ask ( " qwen3.8-max " , " event: content_block_delta \n data: { \" type \" : \" content_block_delta \" , \" index \" :0, \" delta \" :{ \" type \" : \" text_delta \" , \" text \" : \" 嗨 \" }} \n \n event: message_stop \n data: { \" type \" : \" message_stop \" } \n \n " ) ;
assert_eq! ( out . text , " 嗨 " ) ;
assert! ( seen . starts_with ( " POST /anthropic/v1/messages " ) , " {seen} " ) ;
let low = seen . to_lowercase ( ) ;
assert! ( low . contains ( " x-api-key: sk-go " ) & & low . contains ( " x-opencode-session: anybot-bot-7 " ) , " {seen} " ) ;
// Responses-style models go through OpenCode's chat-completions translation.
Whole control flow checked against real chats: what the bot sees is what is there, every step lands, nothing done twice
Web
- web_fetch kept pages with a <header> only up to it (taken for <head>): articles came back as their menus.
- browser_open waits for the page to load and returns it like browser_read (one step fewer per page);
actions wait for the page to hold still instead of fixed 1-2.5 s sleeps; one way to reach the tab.
The bot's computer (box v6)
- Numbers only on what can be seen: clipped to the window and the page's visible area, none under
another app's window (a link scrolled under the toolbar was numbered on Back).
- press_key with a typed character no longer swallows the keys after it (xdotool type --args 1).
- The person's input through noVNC/TigerVNC is noticed again (fence, continuous updates, QEMU keys).
- "Nothing changed" only after clicks, scrolls and drags; long text is pasted; one screen step at a
time; one docker inspect per step when the box is known ready; poppler-utils for long PDFs.
- One typing path and fewer round trips (stdin variant, unused picture path, extra batch execs gone).
The person's computers
- Linux: the window system's front app when it has one, and the reason when there is none (was 「」);
hidden and off-screen controls get no number; wording no longer names a tool that doesn't exist.
The loop
- Without Docker, the bot's own computer runs no commands where there is no sandbox (Linux, Windows):
it was a plain shell on the brain itself, keys in reach, asked nothing.
- The bot's own computer is re-checked each step; unreachable computers are failures, not safety
blocks; no direct-mode note on every result; an empty reply is asked again; scrolling and keys
aren't counted as going in circles; steps aren't moved to a computer that can't do them.
- One finish() for every step's result, keep_grant(), dead appctl list_computers/SERVING and the
impossible server branches removed; duplicated prompt instructions merged.
Tests: 162 (Mac) / 166 (Linux) unit; real box drills 6/6 on box v6; a brain without Chrome browses
on its box; remote E2E 28/28.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4
2026-11-06 09:44:59 +00:00
let ( _ , seen ) = ask ( " grok-4.7 " , " data: { \" choices \" :[{ \" delta \" :{ \" content \" : \" ok \" }}]} \n \n data: [DONE] \n \n " ) ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
assert! ( seen . starts_with ( " POST /openai/v1/chat/completions " ) , " {seen} " ) ;
}
/// Network: OpenCode Go's real model list (public) and the style of a few models.
#[ test ]
#[ ignore ]
fn opencode_go_live_list ( ) {
// A wrong key is refused (so pasting a bad key shows 金鑰無效 right away).
assert! ( list_models ( ProviderKind ::OpenCodeGo , OPENCODE_GO_BASE , " sk-wrong " ) . unwrap_err ( ) . contains ( " 401 " ) ) ;
// The styles come from the public list (no key).
assert_eq! ( opencode_go_style ( OPENCODE_GO_BASE , " qwen3.8-max " ) , " messages " ) ;
assert_eq! ( opencode_go_style ( OPENCODE_GO_BASE , " glm-5.3 " ) , " chat-completions " ) ;
}
#[ test ]
fn a_model_that_cannot_see_gets_no_pictures ( ) {
2026-11-05 07:48:20 +00:00
let _g = vision_test_guard ( ) ;
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
assert! ( image_refused ( " 錯誤 400: At most 0 image(s) may be provided in one prompt. (parameter=image) " ) ) ;
assert! ( image_refused ( " This model does not support image input " ) ) ;
assert! ( image_refused ( " unknown variant `image_url`, expected `text` " ) ) ;
assert! ( ! image_refused ( " 錯誤 400: Invalid credential " ) ) ;
// The first request fails like the user's model did; the retry has no picture.
let l = TcpListener ::bind ( " 127.0.0.1:0 " ) . unwrap ( ) ;
let port = l . local_addr ( ) . unwrap ( ) . port ( ) ;
let ( tx , rx ) = std ::sync ::mpsc ::channel ::< String > ( ) ;
std ::thread ::spawn ( move | | {
for ( n , s ) in l . incoming ( ) . flatten ( ) . enumerate ( ) {
let mut s = s ;
let mut req = Vec ::new ( ) ;
let mut buf = [ 0 u8 ; 65536 ] ;
loop {
let k = s . read ( & mut buf ) . unwrap_or ( 0 ) ;
if k = = 0 {
break ;
}
req . extend_from_slice ( & buf [ .. k ] ) ;
let text = String ::from_utf8_lossy ( & req ) . to_string ( ) ;
if let Some ( end ) = text . find ( " \r \n \r \n " ) {
let len = text [ .. end ] . lines ( ) . find_map ( | l | l . to_ascii_lowercase ( ) . strip_prefix ( " content-length: " ) . map ( | v | v . trim ( ) . parse ::< usize > ( ) . unwrap_or ( 0 ) ) ) . unwrap_or ( 0 ) ;
if req . len ( ) > = end + 4 + len {
break ;
}
}
}
let _ = tx . send ( String ::from_utf8_lossy ( & req ) . to_string ( ) ) ;
if n = = 0 {
let body = r # "{"error":{"message":"At most 0 image(s) may be provided in one prompt. (parameter=image)"}}"# ;
let _ = write! ( s , " HTTP/1.1 400 Bad Request \r \n Content-Type: application/json \r \n Content-Length: {} \r \n Connection: close \r \n \r \n {body} " , body . len ( ) ) ;
} else {
let _ = s . write_all ( b " HTTP/1.1 200 OK \r \n Content-Type: text/event-stream \r \n Connection: close \r \n \r \n data: { \" choices \" :[{ \" delta \" :{ \" content \" : \" ok \" }}]} \n \n data: [DONE] \n \n " ) ;
}
}
} ) ;
let mut shot = Message ::tool_result ( " c1 " , " computer_use_screenshot " , " 你的螢幕截圖 " ) ;
shot . images = vec! [ " SHOTDATA " . into ( ) ] ;
let mut asked = Message ::assistant ( " " , " " ) ;
asked . tool_calls = vec! [ ToolCall { id : " c1 " . into ( ) , name : " computer_use_screenshot " . into ( ) , args : " {} " . into ( ) , status : ToolStatus ::Done } ] ;
let req = ChatRequest { kind : ProviderKind ::Custom , base_url : format ! ( " http://127.0.0.1:{port}/v1 " ) , api_key : String ::new ( ) , model : " blind-model " . into ( ) , system : String ::new ( ) , history : vec ! [ Message ::user ( " 看一下畫面 " ) , asked , shot ] , tools : vec ! [ ] , session : String ::new ( ) } ;
let cancel = AtomicBool ::new ( false ) ;
let out = run_step ( & req , & cancel , & | _ | { } ) . unwrap ( ) ;
assert_eq! ( out . text , " ok " ) ;
let first = rx . recv ( ) . unwrap ( ) ;
let second = rx . recv ( ) . unwrap ( ) ;
assert! ( first . contains ( " SHOTDATA " ) & & ! second . contains ( " SHOTDATA " ) & & second . contains ( " 看不到圖片 " ) , " {second} " ) ;
assert! ( ! sees_images ( & req ) ) ;
}
2026-11-04 13:52:06 +00:00
/// A fake model: refuses pictures when `blind`, else answers `reply`; hands over each request.
fn fake_model ( blind : bool , reply : & 'static str ) -> ( String , std ::sync ::mpsc ::Receiver < String > ) {
let l = TcpListener ::bind ( " 127.0.0.1:0 " ) . unwrap ( ) ;
let port = l . local_addr ( ) . unwrap ( ) . port ( ) ;
let ( tx , rx ) = std ::sync ::mpsc ::channel ::< String > ( ) ;
std ::thread ::spawn ( move | | {
for s in l . incoming ( ) . flatten ( ) {
let mut s = s ;
let mut req = Vec ::new ( ) ;
let mut buf = [ 0 u8 ; 65536 ] ;
loop {
let k = s . read ( & mut buf ) . unwrap_or ( 0 ) ;
if k = = 0 {
break ;
}
req . extend_from_slice ( & buf [ .. k ] ) ;
let text = String ::from_utf8_lossy ( & req ) . to_string ( ) ;
if let Some ( end ) = text . find ( " \r \n \r \n " ) {
let len = text [ .. end ] . lines ( ) . find_map ( | l | l . to_ascii_lowercase ( ) . strip_prefix ( " content-length: " ) . map ( | v | v . trim ( ) . parse ::< usize > ( ) . unwrap_or ( 0 ) ) ) . unwrap_or ( 0 ) ;
if req . len ( ) > = end + 4 + len {
break ;
}
}
}
let text = String ::from_utf8_lossy ( & req ) . to_string ( ) ;
let _ = tx . send ( text . clone ( ) ) ;
if blind & & text . contains ( " image_url " ) {
let body = r # "{"error":{"message":"This model does not support image input"}}"# ;
let _ = write! ( s , " HTTP/1.1 400 Bad Request \r \n Content-Type: application/json \r \n Content-Length: {} \r \n Connection: close \r \n \r \n {body} " , body . len ( ) ) ;
} else {
let chunk = serde_json ::json! ( { " choices " : [ { " delta " : { " content " : reply } } ] } ) ;
let _ = write! ( s , " HTTP/1.1 200 OK \r \n Content-Type: text/event-stream \r \n Connection: close \r \n \r \n data: {chunk} \n \n data: [DONE] \n \n " ) ;
}
}
} ) ;
( format! ( " http://127.0.0.1: {port} /v1 " ) , rx )
}
#[ test ]
fn a_model_that_sees_describes_pictures_for_one_that_cannot ( ) {
2026-11-05 07:48:20 +00:00
let _g = vision_test_guard ( ) ;
2026-11-04 13:52:06 +00:00
let ( blind_url , blind_rx ) = fake_model ( true , " 好的,我看到描述了 " ) ;
let ( eyes_url , eyes_rx ) = fake_model ( false , " 一張報表截圖,上面寫著「十月營收 42 萬」。 " ) ;
set_eyes ( vec! [
// Not there (a typo in a model name, a dead server…): the next one is asked.
Eyes { kind : ProviderKind ::Custom , base_url : " http://127.0.0.1:9/v1 " . into ( ) , api_key : String ::new ( ) , model : " gone " . into ( ) } ,
Eyes { kind : ProviderKind ::Custom , base_url : eyes_url , api_key : String ::new ( ) , model : " eyes-model " . into ( ) } ,
] ) ;
let mut photo = Message ::user ( " 幫我看這張圖 " ) ;
photo . images = vec! [ " UElDVFVSRS1PTkU= " . into ( ) ] ;
let req = ChatRequest { kind : ProviderKind ::Custom , base_url : blind_url , api_key : String ::new ( ) , model : " text-only " . into ( ) , system : String ::new ( ) , history : vec ! [ photo ] , tools : vec ! [ ] , session : String ::new ( ) } ;
let said = std ::sync ::Mutex ::new ( vec! [ ] ) ;
let out = run_step ( & req , & AtomicBool ::new ( false ) , & | e | {
if let StreamEvent ::Retrying ( m ) = e {
said . lock ( ) . unwrap ( ) . push ( m ) ;
}
} )
. unwrap ( ) ;
assert_eq! ( out . text , " 好的,我看到描述了 " ) ;
// The helper got the picture and what the person said with it.
let helper = eyes_rx . recv ( ) . unwrap ( ) ;
assert! ( helper . contains ( " UElDVFVSRS1PTkU= " ) & & helper . contains ( " 幫我看這張圖 " ) ) ;
// The blind model got the description instead of the picture.
let _refused = blind_rx . recv ( ) . unwrap ( ) ;
let again = blind_rx . recv ( ) . unwrap ( ) ;
assert! ( ! again . contains ( " UElDVFVSRS1PTkU= " ) & & again . contains ( " 十月營收 42 萬 " ) & & again . contains ( " eyes-model " ) , " {again} " ) ;
assert! ( said . lock ( ) . unwrap ( ) . iter ( ) . any ( | m | m . contains ( " 看得懂圖 " ) ) ) ;
// The same picture later: described from memory, no second call to the helper.
let mut photo2 = Message ::user ( " 再看一次 " ) ;
photo2 . images = vec! [ " UElDVFVSRS1PTkU= " . into ( ) ] ;
let req2 = ChatRequest { history : vec ! [ photo2 ] , .. req . clone ( ) } ;
run_step ( & req2 , & AtomicBool ::new ( false ) , & | _ | { } ) . unwrap ( ) ;
assert! ( blind_rx . recv ( ) . unwrap ( ) . contains ( " 十月營收 42 萬 " ) ) ;
assert! ( eyes_rx . recv_timeout ( std ::time ::Duration ::from_millis ( 300 ) ) . is_err ( ) ) ;
set_eyes ( vec! [ ] ) ;
}
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
#[ test ]
fn steps_without_results_are_closed ( ) {
let mut a = Message ::assistant ( " " , " " ) ;
a . tool_calls = vec! [
ToolCall { id : " t1 " . into ( ) , name : " shell " . into ( ) , args : " {} " . into ( ) , status : ToolStatus ::Denied } ,
ToolCall { id : " t2 " . into ( ) , name : " browser_fill_login " . into ( ) , args : " {} " . into ( ) , status : ToolStatus ::Waiting } ,
] ;
let h = vec! [ Message ::user ( " a " ) , a , Message ::tool_result ( " t2 " , " browser_fill_login " , " ok " ) , Message ::tool_result ( " zz " , " x " , " stray " ) , Message ::user ( " 再一次 " ) ] ;
let r = repair_history ( & h ) ;
let roles : Vec < String > = r . iter ( ) . map ( | m | format! ( " {:?} : {} " , m . role , m . tool_call_id ) ) . collect ( ) ;
assert_eq! ( roles , vec! [ " User: " , " Assistant: " , " Tool:t2 " , " Tool:t1 " , " User: " ] ) ;
assert! ( r [ 3 ] . text . contains ( " 沒有允許 " ) ) ;
// The OpenAI-style body pairs every call with a result.
let req = ChatRequest { kind : ProviderKind ::Custom , base_url : String ::new ( ) , api_key : String ::new ( ) , model : " m " . into ( ) , system : String ::new ( ) , history : r , tools : vec ! [ ] , session : String ::new ( ) } ;
let msgs = openai_messages ( & req ) ;
let i = msgs . iter ( ) . position ( | m | m [ " tool_calls " ] . is_array ( ) ) . unwrap ( ) ;
assert_eq! ( msgs [ i + 1 ] [ " role " ] , " tool " ) ;
assert_eq! ( msgs [ i + 2 ] [ " role " ] , " tool " ) ;
}
}