// The switchable AI providers and how to talk to each one. // Every provider streams its reply piece by piece over Server-Sent Events. use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use std::io::{BufRead, BufReader}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::Sender; use std::time::Duration; use crate::model::{Message, Role, ToolCall, ToolStatus}; #[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Debug, Hash)] pub enum ProviderKind { Xai, OpenAi, Anthropic, Gemini, OpenRouter, OpenCodeGo, Ollama, Custom, } pub const ALL_PROVIDERS: [ProviderKind; 8] = [ ProviderKind::Xai, ProviderKind::OpenAi, ProviderKind::Anthropic, ProviderKind::Gemini, ProviderKind::OpenRouter, ProviderKind::OpenCodeGo, ProviderKind::Ollama, ProviderKind::Custom, ]; /// How long to wait for the first bytes before giving up /// (the original shows "模型未能及時開始回應" in that case; its deadline is server-side: estimated). pub const FIRST_RESPONSE_TIMEOUT_SECS: u64 = 120; /// Connect deadline: main-core.cjs @1082927, `$D=15e3` (gateway-sse-connect). pub const CONNECT_TIMEOUT_MS: u64 = 15_000; /// Mid-stream silence that counts as a stall: main-core.cjs @1082927, `VD=35e3` (gateway-sse-stall idleMs). pub const STREAM_IDLE_STALL_MS: u64 = 35_000; /// Retry policy: main-core.cjs @1082927, `D3={maxAttempts:3,initialDelayMs:1e3,maxDelayMs:4e3,backoffFactor:2}`. pub const RETRY_MAX_ATTEMPTS: u32 = 3; pub const RETRY_INITIAL_DELAY_MS: u64 = 1_000; pub const RETRY_MAX_DELAY_MS: u64 = 4_000; pub const RETRY_BACKOFF_FACTOR: u64 = 2; /// Longest server-paced Retry-After we honour (SAND-E0408 backendCapacityDeferred): estimated cap. pub const RETRY_AFTER_CAP_MS: u64 = 30_000; pub const MAX_OUTPUT_TOKENS_ANTHROPIC: u64 = 64000; pub const ANTHROPIC_VERSION: &str = "2026-10-21"; /// OpenCode Go (opencode.ai/docs/go): one subscription, many open models. Each model is served /// in its own API style — the public models list says which (`chat-completions` → {base}/openai/v1, /// `messages` → {base}/anthropic/v1); every request carries a stable `x-opencode-session`. pub const OPENCODE_GO_BASE: &str = "https://opencode.ai/inference/go"; impl ProviderKind { pub fn key(self) -> &'static str { match self { ProviderKind::Xai => "xai", ProviderKind::OpenAi => "openai", ProviderKind::Anthropic => "anthropic", ProviderKind::Gemini => "gemini", ProviderKind::OpenRouter => "openrouter", ProviderKind::OpenCodeGo => "opencode-go", ProviderKind::Ollama => "ollama", ProviderKind::Custom => "custom", } } pub fn label(self) -> &'static str { match self { ProviderKind::Xai => "xAI Grok", ProviderKind::OpenAi => "OpenAI", ProviderKind::Anthropic => "Claude", ProviderKind::Gemini => "Google Gemini", ProviderKind::OpenRouter => "OpenRouter", ProviderKind::OpenCodeGo => "OpenCode Go", ProviderKind::Ollama => "Ollama（本機）", ProviderKind::Custom => "自訂（OpenAI 相容）", } } pub fn default_base_url(self) -> &'static str { match self { ProviderKind::Xai => "https://api.x.ai/v1", ProviderKind::OpenAi => "https://api.openai.com/v1", ProviderKind::Anthropic => "https://api.anthropic.com/v1", ProviderKind::Gemini => "https://generativelanguage.googleapis.com/v1beta/openai", ProviderKind::OpenRouter => "https://openrouter.ai/api/v1", // The models list is {base}/v1/models; chat goes to {base}/openai/v1 or {base}/anthropic/v1. ProviderKind::OpenCodeGo => OPENCODE_GO_BASE, ProviderKind::Ollama => "http://localhost:11434/v1", ProviderKind::Custom => "http://localhost:8080/v1", } } pub fn suggested_model(self) -> &'static str { match self { ProviderKind::Xai => "grok-4", ProviderKind::OpenAi => "gpt-5", ProviderKind::Anthropic => "claude-opus-5-5", ProviderKind::Gemini => "gemini-2.5-flash", ProviderKind::OpenRouter => "openrouter/auto", ProviderKind::OpenCodeGo => "kimi-k3", ProviderKind::Ollama => "llama3.2", ProviderKind::Custom => "local-model", } } /// A few names to pick from before the real list is fetched. pub fn starter_models(self) -> &'static [&'static str] { match self { ProviderKind::Xai => &["grok-4", "grok-4-fast", "grok-3-mini"], ProviderKind::OpenAi => &["gpt-5", "gpt-5-mini", "gpt-4.1"], ProviderKind::Anthropic => &["claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5", "claude-fable-5-1"], ProviderKind::Gemini => &["gemini-2.5-flash", "gemini-2.5-pro"], ProviderKind::OpenRouter => &["openrouter/auto"], ProviderKind::OpenCodeGo => &["kimi-k3", "glm-5.3", "deepseek-v4-pro", "qwen3.8-max", "minimax-m3", "mimo-v2.6-pro"], ProviderKind::Ollama => &["llama3.2", "qwen2.5", "gemma3"], ProviderKind::Custom => &[], } } pub fn needs_key(self) -> bool { !matches!(self, ProviderKind::Ollama | ProviderKind::Custom) } pub fn env_var(self) -> Option<&'static str> { match self { ProviderKind::Xai => Some("XAI_API_KEY"), ProviderKind::OpenAi => Some("OPENAI_API_KEY"), ProviderKind::Anthropic => Some("ANTHROPIC_API_KEY"), ProviderKind::Gemini => Some("GEMINI_API_KEY"), ProviderKind::OpenRouter => Some("OPENROUTER_API_KEY"), ProviderKind::OpenCodeGo => Some("OPENCODE_API_KEY"), _ => None, } } pub fn key_help(self) -> &'static str { match self { ProviderKind::Xai => "在 console.x.ai 取得 API 金鑰", ProviderKind::OpenAi => "在 platform.openai.com 取得 API 金鑰", ProviderKind::Anthropic => "在 console.anthropic.com 取得 API 金鑰", ProviderKind::Gemini => "在 aistudio.google.com 取得 API 金鑰", ProviderKind::OpenRouter => "在 openrouter.ai 取得 API 金鑰", ProviderKind::OpenCodeGo => "在 opencode.ai 的 Console 訂閱 Go，再複製 API 金鑰", ProviderKind::Ollama => "先安裝並開啟 Ollama，不需要金鑰", ProviderKind::Custom => "任何 OpenAI 相容的服務（LM Studio、vLLM…）", } } } /// Everything one model call needs, copied so the worker thread owns it. #[derive(Clone)] pub struct ChatRequest { pub kind: ProviderKind, pub base_url: String, pub api_key: String, pub model: String, pub system: String, pub history: Vec, /// Tool specs ({name, description, parameters}); empty for plain chat. pub tools: Vec, /// One id per conversation (OpenCode Go routes and caches by it). pub session: String, } pub enum StreamEvent { Text(String), Thinking(String), /// Any line from the provider (keeps the stall watchdog quiet). Alive, /// A retry is about to happen; the text says why. Retrying(String), /// The model asked for a tool; shown at once, run after the step ends. ToolCall(ToolCall), /// A tool's state changed (waiting for approval, running, done…). ToolStatus(String, ToolStatus), /// Ask the person before running a tool; answer on the sender. NeedApproval(Approval), /// One model step finished: replace the streaming reply with the full message. StepDone(Message), /// A tool result to append to the chat. ToolResult(Message), /// The next model step starts: add a fresh reply bubble. NextStep(String), /// The oldest `n` messages were folded into this summary. /// Older messages were folded into this summary; `cut` is the count in the model's view and /// `boundary` the timestamp of the first kept message (the window's own copy may be longer). Compacted(usize, Message, i64), /// A message the person sent mid-turn was handed to the model (add it to the chat). Steered(Message), /// Do something in the app (routines, bots, settings) and answer on the sender. AppAction(AppCall), /// 表情回應: the bot put an emoji on the person's latest message. React(String), /// The bot sent a file into the chat (a file card under its reply). Attach(crate::attach::Attachment), /// 自己進化: after the turn, the way that worked was kept as a skill (with the time of the /// reply it learned from, which gets a line saying so). Learned(crate::memory::Skill, i64), Done, Error(String), } pub struct Approval { pub call_id: String, pub tool: String, pub reason: String, /// What "always allow" would remember; empty when it can't be remembered. pub grant: String, pub reply: Sender, pub kind: ApprovalKind, } /// What the person is asked. #[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] pub enum ApprovalKind { /// One action (run this command, edit this file…). #[default] Action, /// This bot's computer tools are off: let it use them? ("always" turns them on for the bot.) EnableTools, /// 在這台電腦上執行 is off: let it work on this Mac? ("always" turns the setting on.) EnableThisComputer, /// 需要處理: a step only the person can do on a computer (signing in, a captcha…). `reason` /// is what to do, `grant` the computer's name; Once = 「我完成了」, Reject = 「跳過」 (the /// original's box hand-off card). Handoff, } pub struct AppCall { pub call_id: String, pub name: String, pub args: String, /// (result text, ok) pub reply: Sender<(String, bool)>, } #[derive(Clone, Debug, PartialEq)] pub enum ApprovalAnswer { Once, Always(String), Reject, } /// What one model call produced. #[derive(Default, Debug)] pub struct StepOutput { pub text: String, pub thinking: String, pub tool_calls: Vec, /// Claude's raw content blocks, kept to replay thinking signatures unchanged. pub blocks: Vec, } /// Why a request failed, sorted the way the original's error catalog sorts them /// (main-core.cjs @689060..692409, SAND-E0401..E0418). pub struct Failure { pub msg: String, pub retryable: bool, pub retry_after_ms: Option, } impl Failure { fn retry(msg: String) -> Self { Failure { msg, retryable: true, retry_after_ms: None } } fn fatal(msg: String) -> Self { Failure { msg, retryable: false, retry_after_ms: None } } } /// Delay before retry `n` (1-based): 1 s, 2 s, 4 s, capped at 4 s. pub fn backoff_delay_ms(retry: u32) -> u64 { (RETRY_INITIAL_DELAY_MS * RETRY_BACKOFF_FACTOR.pow(retry.saturating_sub(1))).min(RETRY_MAX_DELAY_MS) } fn agent() -> ureq::Agent { let config = ureq::Agent::config_builder() .http_status_as_error(false) .timeout_connect(Some(Duration::from_millis(CONNECT_TIMEOUT_MS))) .timeout_recv_response(Some(Duration::from_secs(FIRST_RESPONSE_TIMEOUT_SECS))) // A stream that goes silent (a NAT that dropped the mapping, a server that died without // a FIN) must end: without a read timeout the lines() loop blocks forever, and every // such turn leaks a thread and its socket. 5 minutes of total silence is a dead stream. .timeout_per_read(Some(Duration::from_secs(300))) .build(); config.into() } pub fn resolve_key(kind: ProviderKind, stored: &str) -> String { if !stored.trim().is_empty() { return stored.trim().to_string(); } kind.env_var().and_then(|v| std::env::var(v).ok()).unwrap_or_default() } pub fn resolve_base(kind: ProviderKind, stored: &str) -> String { let b = if stored.trim().is_empty() { kind.default_base_url() } else { stored.trim() }; b.trim_end_matches('/').to_string() } /// Claude models that take adaptive thinking (older ones reject it). fn claude_adaptive(model: &str) -> bool { ["claude-opus-5", "claude-sonnet-5", "claude-haiku-5", "claude-fable-5", "claude-mythos-5", "claude-opus-4-6", "claude-opus-4-7", "claude-opus-4-8", "claude-sonnet-4-6"] .iter() .any(|p| model.starts_with(p)) } /// Claude models where the server can retry a declined request on another model. fn claude_fallbacks(model: &str) -> bool { ["claude-opus-5-5", "claude-opus-5", "claude-fable-5-1", "claude-sonnet-5-5"].contains(&model) } /// An error body as words: a gateway's HTML error page (「錯誤 502：…」 filled the /// chat) becomes its text; anything else is kept as it is. fn page_words(body: &str) -> String { if !body.trim_start().starts_with('<') { return body.to_string(); } let no_style = body.rsplit("").next().unwrap_or(body); let mut out = String::new(); let mut in_tag = false; for c in no_style.chars() { match c { '<' => { in_tag = true; out.push(' '); } '>' => in_tag = false, _ if !in_tag => out.push(c), _ => {} } } out.split_whitespace().collect::>().join(" ") } /// An error the model sent inside its answer stream, in words: "at capacity" and the like are /// waited out and tried again (it came through raw English, as the reply, and was never retried). fn stream_failure(msg: &str) -> Failure { let low = msg.to_lowercase(); if ["capacity", "overloaded", "rate limit", "too many requests", "try again", "temporarily", "unavailable", "timeout", "timed out"].iter().any(|k| low.contains(k)) { Failure { msg: format!("模型供應商現在太忙，正在等一下再試。\n（{msg}）"), retryable: true, retry_after_ms: Some(BUSY_RETRY_MS) } } else { Failure::fatal(format!("模型回報錯誤，這一步沒有完成。\n（{msg}）")) } } /// Wait before asking a busy model again (estimated; it says "a few minutes", 1–4 s was too soon). const BUSY_RETRY_MS: u64 = 8_000; fn http_error(status: u16, body: &str, retry_after: Option) -> Failure { let detail = serde_json::from_str::(body) .ok() .and_then(|v| { v.pointer("/error/message") .or_else(|| v.pointer("/message")) .or_else(|| v.get("error")) .map(|m| m.as_str().map(str::to_string).unwrap_or_else(|| m.to_string())) }) .unwrap_or_else(|| page_words(body).chars().take(300).collect()); let hint = match status { 401 | 403 => "金鑰無效或沒有權限，請到「設定 → AI 供應商」檢查。", 404 => "找不到這個模型或網址，請檢查模型名稱。", 429 => "模型供應商已超載或用量已達上限。通常是暫時的，請稍後再試。", 500..=599 => "模型供應商目前負載過高。通常是暫時的。", _ => "", }; let lower = detail.to_lowercase(); // SAND-E0404 contextWindowOverflow: a retry would repeat it. if (status == 400 || status == 413) && ["context", "too long", "too many tokens", "maximum context", "prompt is too long"].iter().any(|k| lower.contains(k)) { return Failure::fatal(format!("對話太長了，超過這個模型能讀的長度。請清除聊天記錄或換一個模型。\n（{detail}）")); } if status == 400 && lower.contains("tool") && (lower.contains("support") || lower.contains("not available")) { return Failure::fatal(format!("這個模型不支援工具。請換一個支援工具的模型，或在 Bot 設定裡關掉「電腦操作」。\n（{detail}）")); } let msg = format!("錯誤 {status}：{detail}\n{hint}").trim().to_string(); // SAND-E0401 providerOverloaded / SAND-E0408 backendCapacityDeferred: retry. if status == 429 || status == 408 || status == 529 || status >= 500 { Failure { msg, retryable: true, retry_after_ms: retry_after } } else { Failure::fatal(msg) } } fn retry_after_ms(resp: &ureq::http::Response) -> Option { let v = resp.headers().get("retry-after")?.to_str().ok()?; v.trim().parse::().ok().map(|s| (s * 1000.0) as u64).map(|ms| ms.min(RETRY_AFTER_CAP_MS)) } /// One model call with the original's retry rules: retry overloads, resets and unreachable /// backends up to 3 tries (1 s, 2 s, 4 s or Retry-After), but never once output reached the screen. pub fn run_step(req: &ChatRequest, cancel: &AtomicBool, send: &dyn Fn(StreamEvent)) -> Result { // The bot's instructions and tool descriptions in this system's words ("this computer" on Linux). #[cfg(not(target_os = "macos"))] let localized = { let mut r = req.clone(); r.system = crate::platform::localize(&r.system); r.tools.iter_mut().for_each(crate::platform::localize_value); r }; #[cfg(not(target_os = "macos"))] let req = &localized; if req.kind.needs_key() && req.api_key.is_empty() { return Err(format!("還沒有設定 {} 的 API 金鑰。請到「設定 → AI 供應商」貼上金鑰。", req.kind.label())); } let produced = std::cell::Cell::new(false); let tracked = |ev: StreamEvent| { if matches!(ev, StreamEvent::Text(_) | StreamEvent::Thinking(_) | StreamEvent::ToolCall(_)) { produced.set(true); } send(ev); }; let mut attempt = 1; // Steps left without a result (stopped while waiting for the person) are closed off. let repaired = ChatRequest { history: repair_history(&req.history), ..req.clone() }; let req = &repaired; let mut blind_copy: Option = None; loop { // A model that can't see pictures gets the request without them: the person's pictures // and the newest screenshot described by a model that can (看圖小幫手). if !sees_images(req) && blind_copy.is_none() { blind_copy = Some(if DESCRIBING.with(|d| d.get()) { without_images(req) } else { with_descriptions(req, cancel, send) }); } let req = blind_copy.as_ref().unwrap_or(req); let result = match req.kind { ProviderKind::Anthropic => stream_anthropic(req, cancel, &tracked), ProviderKind::OpenCodeGo => { let mut r = req.clone(); if opencode_go_style(&req.base_url, &req.model) == "messages" { r.base_url = format!("{}/anthropic/v1", req.base_url); stream_anthropic(&r, cancel, &tracked) } else { r.base_url = format!("{}/openai/v1", req.base_url); stream_openai_compatible(&r, cancel, &tracked) } } _ => stream_openai_compatible(req, cancel, &tracked), }; match result { // Nothing at all came back (no words, no tool call): a dropped stream or a hiccup. The // turn used to end on 「（沒有回應內容）」; ask again instead. Ok(out) if out.text.trim().is_empty() && out.tool_calls.is_empty() && !produced.get() && attempt < RETRY_MAX_ATTEMPTS && !cancel.load(Ordering::Relaxed) => { send(StreamEvent::Retrying(format!("模型這次沒有回任何內容，再問一次（第 {} 次，共 {} 次）…", attempt + 1, RETRY_MAX_ATTEMPTS))); std::thread::sleep(Duration::from_millis(backoff_delay_ms(attempt))); attempt += 1; } Ok(out) => return Ok(out), Err(f) if image_refused(&f.msg) && sees_images(req) && !produced.get() && !cancel.load(Ordering::Relaxed) => { mark_blind(req); send(StreamEvent::Retrying("這個模型不能看圖片，改成不附圖片再試一次…".into())); } Err(f) if f.retryable && !produced.get() && attempt < RETRY_MAX_ATTEMPTS && !cancel.load(Ordering::Relaxed) => { let wait = f.retry_after_ms.unwrap_or_else(|| backoff_delay_ms(attempt)); send(StreamEvent::Retrying(format!("{}\n{:.0} 秒後重試（第 {} 次，共 {} 次）…", f.msg, wait as f64 / 1000.0, attempt + 1, RETRY_MAX_ATTEMPTS))); let until = std::time::Instant::now() + Duration::from_millis(wait); while std::time::Instant::now() < until { if cancel.load(Ordering::Relaxed) { return Ok(StepOutput::default()); } std::thread::sleep(Duration::from_millis(100)); } attempt += 1; } Err(f) => return Err(f.msg), } } } fn read_sse(body: ureq::Body, cancel: &AtomicBool, send: &dyn Fn(StreamEvent), on_data: impl FnMut(&str, &str) -> bool) -> Result<(), Failure> { read_sse_from(body.into_reader(), cancel, send, on_data) } fn read_sse_from(r: impl std::io::Read, cancel: &AtomicBool, send: &dyn Fn(StreamEvent), mut on_data: impl FnMut(&str, &str) -> bool) -> Result<(), Failure> { let reader = BufReader::new(r); let mut event = String::new(); for line in reader.lines() { if cancel.load(Ordering::Relaxed) { return Ok(()); } // SAND-E0403 streamReset: retryable. let line = line.map_err(|e| Failure::retry(format!("連線中斷：{e}")))?; send(StreamEvent::Alive); if let Some(ev) = line.strip_prefix("event:") { event = ev.trim().to_string(); } else if let Some(data) = line.strip_prefix("data:") { if !on_data(&event, data.trim()) { return Ok(()); } } else if line.is_empty() { event.clear(); } } Ok(()) } /// Summaries become part of the system prompt; everything else stays in order. fn system_with_summaries(req: &ChatRequest) -> String { let mut s = req.system.clone(); for m in req.history.iter().filter(|m| m.summary) { s.push_str("\n\n先前對話的摘要：\n"); s.push_str(&m.text); } s } /// Which messages still show their screenshots: the newest `KEEP_SCREENSHOTS`. fn kept_images(history: &[Message]) -> Vec { let mut idx: Vec = history.iter().enumerate().filter(|(_, m)| !m.images.is_empty() && !m.summary && m.role == Role::Tool).map(|(i, _)| i).collect(); let drop = idx.len().saturating_sub(crate::computer::KEEP_SCREENSHOTS); idx.drain(..drop); idx } const OLD_SCREENSHOT_NOTE: &str = "（較早的截圖已省略）"; /// Said in place of a screenshot when the model can't take pictures. const BLIND_SCREENSHOT_NOTE: &str = "（截圖已拍好，但這個模型看不到圖片，所以沒有附上。請改用文字方式確認畫面，例如讀取視窗或網頁的文字。）"; /// What a helper is asked about a screenshot. const SCREEN_ASK: &str = "這是一台電腦螢幕的截圖。請描述：開著哪些視窗或 App、畫面上的文字（照抄重要的字）、按鈕和欄位，以及它們大約的位置（以這張圖的像素座標，左上角是 0,0）。不要猜看不清楚的字。"; const BLIND_ATTACHMENT_NOTE: &str = "（使用者附了圖片，但這個模型看不到圖片。）"; /// Models (base url + name) that refused a picture: later requests leave pictures out. static NO_VISION: std::sync::Mutex>> = std::sync::Mutex::new(None); fn vision_key(req: &ChatRequest) -> String { format!("{}|{}", req.base_url, req.model) } pub fn sees_images(req: &ChatRequest) -> bool { !NO_VISION.lock().unwrap().as_ref().map(|s| s.contains(&vision_key(req))).unwrap_or(false) } fn mark_blind(req: &ChatRequest) { NO_VISION.lock().unwrap().get_or_insert_with(Default::default).insert(vision_key(req)); } /// The provider said this model can't take images ("At most 0 image(s)…", "does not support /// image input", "unknown variant `image_url`"…). pub fn image_refused(msg: &str) -> bool { let l = msg.to_lowercase(); let about_images = l.contains("image") || l.contains("vision") || l.contains("multimodal") || l.contains("圖片") || l.contains("图片"); about_images && ["at most 0", "not support", "unsupported", "doesn't support", "does not support", "no vision", "not a vision", "not multimodal", "unknown variant", "invalid type", "only text", "text-only", "不支援", "不支持"].iter().any(|k| l.contains(k)) } /// Every tool call needs its result right after it (providers refuse the conversation /// otherwise). A step stopped or refused while it waited for the person has none: give it one. /// Results whose call is gone are dropped. pub fn repair_history(history: &[Message]) -> Vec { let mut out: Vec = Vec::with_capacity(history.len()); let mut i = 0; while i < history.len() { let m = &history[i]; if m.role == Role::Tool { // A result with no call right before it. i += 1; continue; } out.push(m.clone()); i += 1; if m.role == Role::Assistant && !m.tool_calls.is_empty() { let ids: Vec<&str> = m.tool_calls.iter().map(|c| c.id.as_str()).collect(); let mut seen = vec![]; while i < history.len() && history[i].role == Role::Tool { let r = &history[i]; if ids.contains(&r.tool_call_id.as_str()) && !seen.contains(&r.tool_call_id) { seen.push(r.tool_call_id.clone()); out.push(r.clone()); } i += 1; } for c in &m.tool_calls { if !seen.contains(&c.id) { let why = if c.status == ToolStatus::Denied { "（使用者沒有允許這個步驟，所以沒有執行。）" } else { "（這個步驟被中止了，沒有執行完。）" }; out.push(Message::tool_result(&c.id, &c.name, why)); } } } } out } // ---------- 看圖小幫手 ---------- /// A model of the person's that sees pictures (another provider, or another model). #[derive(Clone, Debug, PartialEq)] pub struct Eyes { pub kind: ProviderKind, pub base_url: String, pub api_key: String, pub model: String, } static EYES: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); thread_local! { /// Inside a helper's own call (it never asks a helper itself). static DESCRIBING: std::cell::Cell = const { std::cell::Cell::new(false) }; } /// The models to ask, best first (the app sets this from the person's providers). pub fn set_eyes(v: Vec) { *EYES.lock().unwrap() = v; } /// The helper list, the description cache and the failure list are process-wide: tests that /// touch them must run one at a time, from a clean slate, or they undo each other (the /// describe cache hit in one test made another's helper never get called). #[cfg(test)] pub fn vision_test_guard() -> std::sync::MutexGuard<'static, ()> { static L: std::sync::Mutex<()> = std::sync::Mutex::new(()); let g = L.lock().unwrap_or_else(|e| e.into_inner()); notes().lock().unwrap().clear(); failed_eyes().lock().unwrap().clear(); set_eyes(vec![]); g } /// Models likely to see pictures, by provider. pub fn seeing_models(kind: ProviderKind) -> &'static [&'static str] { match kind { ProviderKind::Xai => &["grok-4", "grok-4-fast"], ProviderKind::Anthropic => &["claude-sonnet-5-5", "claude-haiku-5-5"], ProviderKind::OpenAi => &["gpt-5-mini", "gpt-4.1"], ProviderKind::Gemini => &["gemini-2.5-flash"], ProviderKind::OpenRouter => &["openrouter/auto"], _ => &[], } } const EYES_PROMPT: &str = "你是「看圖小幫手」：替一個看不到圖片的 AI 助理看圖。用繁體中文把圖片完整描述出來，讓它不用看圖也能幫使用者：畫面是什麼、主要的東西和位置、所有看得到的文字（原文照抄，包括數字、表格、按鈕和錯誤訊息）、顏色或狀態等重點。只描述，不評論、不問問題、不說你是誰。"; /// Descriptions already made (by the picture), kept across restarts. fn notes_path() -> std::path::PathBuf { crate::model::State::path().with_file_name("image-notes.json") } fn notes() -> &'static std::sync::Mutex> { static N: std::sync::OnceLock>> = std::sync::OnceLock::new(); N.get_or_init(|| std::sync::Mutex::new(if cfg!(test) { Default::default() } else { std::fs::read(notes_path()).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default() })) } /// Helpers that just failed (wrong model name, no credit, down…), skipped for a while. fn failed_eyes() -> &'static std::sync::Mutex> { static F: std::sync::OnceLock>> = std::sync::OnceLock::new(); F.get_or_init(Default::default) } fn picture_key(b64: &str) -> String { let d = ring::digest::digest(&ring::digest::SHA256, b64.as_bytes()); d.as_ref()[..12].iter().map(|b| format!("{b:02x}")).collect() } /// Ask the first helper that can see pictures to describe this one: (description, model). fn describe(picture: &str, ask: &str, not: &ChatRequest, cancel: &AtomicBool) -> Option<(String, String)> { let key = picture_key(picture); if let Some(n) = notes().lock().unwrap().get(&key) { return Some(n.clone()); } let helpers: Vec = EYES.lock().unwrap().clone(); for e in helpers { let req = ChatRequest { kind: e.kind, base_url: e.base_url.clone(), api_key: e.api_key.clone(), model: e.model.clone(), system: EYES_PROMPT.into(), history: vec![Message { images: vec![picture.to_string()], ..Message::user(ask) }], tools: vec![], session: String::new(), }; let id = vision_key(&req); let resting = failed_eyes().lock().unwrap().get(&id).map(|t| t.elapsed().as_secs() < 600).unwrap_or(false); if (req.base_url == not.base_url && req.model == not.model) || !sees_images(&req) || resting || cancel.load(Ordering::Relaxed) { continue; } DESCRIBING.with(|d| d.set(true)); let out = run_step(&req, cancel, &|_| {}); DESCRIBING.with(|d| d.set(false)); match out { // It may have turned out not to see pictures after all (and answered without). Ok(o) if sees_images(&req) && !o.text.trim().is_empty() => { let note = (o.text.trim().to_string(), e.model.clone()); let mut all = notes().lock().unwrap(); if all.len() > 500 { all.clear(); } all.insert(key, note.clone()); if let (false, Ok(b)) = (cfg!(test), serde_json::to_vec(&*all)) { let _ = std::fs::write(notes_path(), b); } return Some(note); } _ => { failed_eyes().lock().unwrap().insert(id, std::time::Instant::now()); } } } None } /// The request for a model that can't see pictures: each of the person's pictures replaced by /// what a helper saw in it; screenshots by a short note. fn with_descriptions(req: &ChatRequest, cancel: &AtomicBool, send: &dyn Fn(StreamEvent)) -> ChatRequest { let mut r = req.clone(); let mut told = false; // The newest screenshot is described too (what the bot needs to act next); older ones stay a note. let newest_shot = r.history.iter().rposition(|m| m.role == Role::Tool && !m.images.is_empty()); for (i, m) in r.history.iter_mut().enumerate().filter(|(_, m)| !m.images.is_empty()) { if m.role == Role::Tool { let pictures = std::mem::take(&mut m.images); let seen = if Some(i) == newest_shot { if !told && !notes().lock().unwrap().contains_key(&picture_key(&pictures[0])) { send(StreamEvent::Retrying("這個模型看不到圖片，正在請另一個看得懂圖的模型看這張截圖…".into())); told = true; } describe(&pictures[0], SCREEN_ASK, req, cancel) } else { None }; let add = match seen { Some((d, model)) => format!("（截圖已拍好；你看不到圖片，以下是「{model}」看過這張截圖後的描述）\n{d}"), None => BLIND_SCREENSHOT_NOTE.to_string(), }; m.text = if m.text.is_empty() { add } else { format!("{}\n{add}", m.text) }; continue; } let pictures = std::mem::take(&mut m.images); let ask = format!("請描述這張圖片。使用者附上它時說：「{}」", m.text.chars().take(300).collect::()); let mut added = String::new(); for (i, p) in pictures.iter().enumerate() { if !told && !notes().lock().unwrap().contains_key(&picture_key(p)) { send(StreamEvent::Retrying("這個模型看不到圖片，正在請另一個看得懂圖的模型先看一遍…".into())); told = true; } let which = if pictures.len() > 1 { format!("第 {} 張圖片", i + 1) } else { "這張圖片".to_string() }; added.push_str(&match describe(p, &ask, req, cancel) { Some((d, model)) => format!("\n\n（你看不到圖片；以下是「{model}」看過{which}後的描述）\n{d}"), None => format!("\n\n{BLIND_ATTACHMENT_NOTE}（帳號裡也沒有看得懂圖片的模型可以代看。可以請使用者到「設定 › 模型」加一個，例如 Grok、Claude、GPT 或 Gemini。）"), }); } m.text.push_str(&added); } r } /// The same request with pictures replaced by a short note. fn without_images(req: &ChatRequest) -> ChatRequest { let mut r = req.clone(); for m in r.history.iter_mut().filter(|m| !m.images.is_empty()) { m.images.clear(); let note = if m.role == Role::Tool { BLIND_SCREENSHOT_NOTE } else { BLIND_ATTACHMENT_NOTE }; m.text = if m.text.is_empty() { note.to_string() } else { format!("{}\n{note}", m.text) }; } r } pub fn openai_messages(req: &ChatRequest) -> Vec { let mut messages = vec![]; let system = system_with_summaries(req); if !system.trim().is_empty() { messages.push(json!({"role": "system", "content": system})); } let kept = kept_images(&req.history); // Tool messages carry text only; their screenshots follow as one user message after the run // of tool results (a user message between tool results would break the call/answer pairs). let mut pending: Vec = vec![]; let flush = |messages: &mut Vec, pending: &mut Vec| { if !pending.is_empty() { let mut parts = vec![json!({"type": "text", "text": "（上面工具的螢幕截圖）"})]; parts.append(pending); messages.push(json!({"role": "user", "content": parts})); } }; for (i, m) in req.history.iter().enumerate().filter(|(_, m)| !m.summary) { if m.role != Role::Tool { flush(&mut messages, &mut pending); } match m.role { Role::User if !m.images.is_empty() => { // Attached pictures go with the person's words. let mut parts = vec![json!({"type": "text", "text": if m.text.is_empty() { "（附件）".to_string() } else { m.text.clone() }})]; parts.extend(m.images.iter().map(|b| json!({"type": "image_url", "image_url": {"url": format!("data:image/jpeg;base64,{b}")}}))); messages.push(json!({"role": "user", "content": parts})); } Role::User => { if !m.text.is_empty() { messages.push(json!({"role": "user", "content": m.text})); } } Role::Assistant => { if m.error && m.tool_calls.is_empty() { continue; } if m.tool_calls.is_empty() { if !m.text.is_empty() { messages.push(json!({"role": "assistant", "content": m.text})); } } else { let calls: Vec = m.tool_calls.iter().map(|c| json!({"id": c.id, "type": "function", "function": {"name": c.name, "arguments": if c.args.trim().is_empty() { "{}" } else { &c.args }}})).collect(); messages.push(json!({"role": "assistant", "content": if m.text.is_empty() { Value::Null } else { json!(m.text) }, "tool_calls": calls})); } } Role::Tool => { let shown = kept.contains(&i); let text = if !m.images.is_empty() && !shown { format!("{}\n{OLD_SCREENSHOT_NOTE}", m.text) } else { m.text.clone() }; messages.push(json!({"role": "tool", "tool_call_id": m.tool_call_id, "content": text})); if shown { pending.extend(m.images.iter().map(|b| json!({"type": "image_url", "image_url": {"url": format!("data:image/jpeg;base64,{b}")}}))); } } } } flush(&mut messages, &mut pending); messages } fn stream_openai_compatible(req: &ChatRequest, cancel: &AtomicBool, send: &dyn Fn(StreamEvent)) -> Result { let mut body = json!({"model": req.model, "messages": openai_messages(req), "stream": true}); if !req.tools.is_empty() { body["tools"] = json!(req.tools.iter().map(|t| json!({"type": "function", "function": t})).collect::>()); } let url = format!("{}/chat/completions", req.base_url); let mut headers: Vec<(String, String)> = vec![]; if !req.api_key.is_empty() { headers.push(("Authorization".into(), format!("Bearer {}", req.api_key))); } if req.kind == ProviderKind::OpenRouter { headers.push(("X-Title".into(), "AnyBot".into())); } if req.kind == ProviderKind::OpenCodeGo { headers.push(("x-opencode-session".into(), opencode_session(req))); } // A brain on a server, a model on the person's own network: through their computer (the // whole answer comes back at once, so no word-by-word streaming). let via_hands: Option> = match crate::remote::model_hands_for(&url) { Some(hand) => { let (status, bytes) = crate::remote::model_via_hands(&hand, &url, &headers, &body.to_string()).map_err(Failure::retry)?; if status >= 400 { return Err(http_error(status, &String::from_utf8_lossy(&bytes), None)); } Some(bytes) } None => None, }; let resp = match via_hands { Some(_) => None, None => { let mut r = agent().post(&url).header("Content-Type", "application/json"); for (k, v) in &headers { r = r.header(k, v); } // SAND-E0418 turnBackendUnreachable: retryable. let resp = r.send_json(&body).map_err(|e| Failure::retry(connect_error(req.kind, &e.to_string())))?; let status = resp.status().as_u16(); if status >= 400 { let ra = retry_after_ms(&resp); let text = resp.into_body().read_to_string().unwrap_or_default(); return Err(http_error(status, &text, ra)); } Some(resp) } }; let mut out = StepOutput::default(); // Tool calls arrive in pieces keyed by index. let mut pending: Vec<(String, String, String)> = vec![]; let mut error = None; let reader: Box = match (resp, via_hands) { (Some(r), _) => Box::new(r.into_body().into_reader()), (None, b) => Box::new(std::io::Cursor::new(b.unwrap_or_default())), }; read_sse_from(reader, cancel, send, |_ev, data| { if data == "[DONE]" { return false; } if let Ok(v) = serde_json::from_str::(data) { if let Some(e) = v.get("error") { error = Some(e.get("message").and_then(Value::as_str).unwrap_or("未知錯誤").to_string()); return false; } let delta = &v["choices"][0]["delta"]; for k in ["reasoning_content", "reasoning"] { if let Some(t) = delta.get(k).and_then(Value::as_str) { if !t.is_empty() { out.thinking.push_str(t); send(StreamEvent::Thinking(t.to_string())); } } } if let Some(t) = delta.get("content").and_then(Value::as_str) { if !t.is_empty() { out.text.push_str(t); send(StreamEvent::Text(t.to_string())); } } if let Some(calls) = delta.get("tool_calls").and_then(Value::as_array) { for c in calls { let i = match c["index"].as_u64() { Some(i) => i as usize, // Some OpenAI-compatible proxies drop the index: a part carrying an // id or a name starts a new call; an arguments-only part continues the // last one (starting a fresh slot used to lose the call's tail). None => { let starts = c["id"].is_string() || c["function"]["name"].is_string(); if starts || pending.is_empty() { pending.len() } else { pending.len() - 1 } } }; while pending.len() <= i { pending.push(Default::default()); } if let Some(id) = c["id"].as_str() { pending[i].0 = id.to_string(); } if let Some(n) = c["function"]["name"].as_str() { pending[i].1.push_str(n); } if let Some(a) = c["function"]["arguments"].as_str() { pending[i].2.push_str(a); } } } } true })?; if let Some(e) = error { return Err(stream_failure(&e)); } for (i, (id, name, args)) in pending.into_iter().enumerate() { if name.is_empty() { continue; } let id = if id.is_empty() { format!("call_{i}") } else { id }; let call = ToolCall { id, name, args, status: ToolStatus::Waiting }; send(StreamEvent::ToolCall(call.clone())); out.tool_calls.push(call); } Ok(out) } pub fn anthropic_messages(req: &ChatRequest) -> Vec { let mut messages: Vec = vec![]; let mut push = |role: &str, blocks: Vec| { if blocks.is_empty() { return; } // The API wants user/assistant turns to alternate; merge repeats. if let Some(last) = messages.last_mut() { if last["role"] == role { last["content"].as_array_mut().unwrap().extend(blocks); return; } } messages.push(json!({"role": role, "content": blocks})); }; let kept = kept_images(&req.history); for (i, m) in req.history.iter().enumerate().filter(|(_, m)| !m.summary) { match m.role { Role::Tool if !m.images.is_empty() => { let content = if kept.contains(&i) { let mut c = vec![json!({"type": "text", "text": m.text})]; c.extend(m.images.iter().map(|b| json!({"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b}}))); json!(c) } else { json!(format!("{}\n{OLD_SCREENSHOT_NOTE}", m.text)) }; push("user", vec![json!({"type": "tool_result", "tool_use_id": m.tool_call_id, "content": content})]); } Role::User if !m.images.is_empty() => { let mut b = vec![json!({"type": "text", "text": if m.text.is_empty() { "（附件）".to_string() } else { m.text.clone() }})]; b.extend(m.images.iter().map(|x| json!({"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": x}}))); push("user", b); } Role::User if !m.text.is_empty() => push("user", vec![json!({"type": "text", "text": m.text})]), Role::User => {} Role::Assistant => { if !m.blocks.is_empty() { push("assistant", m.blocks.clone()); } else if !m.error || !m.tool_calls.is_empty() { let mut b = vec![]; if !m.text.is_empty() { b.push(json!({"type": "text", "text": m.text})); } for c in &m.tool_calls { b.push(json!({"type": "tool_use", "id": c.id, "name": c.name, "input": serde_json::from_str::(&c.args).unwrap_or(json!({}))})); } push("assistant", b); } } Role::Tool => push("user", vec![json!({"type": "tool_result", "tool_use_id": m.tool_call_id, "content": m.text})]), } } messages } fn stream_anthropic(req: &ChatRequest, cancel: &AtomicBool, send: &dyn Fn(StreamEvent)) -> Result { let mut body = json!({ "model": req.model, "max_tokens": MAX_OUTPUT_TOKENS_ANTHROPIC, "messages": anthropic_messages(req), "stream": true, }); let system = system_with_summaries(req); if !system.trim().is_empty() { body["system"] = json!(system); } if claude_adaptive(&req.model) { body["thinking"] = json!({"type": "adaptive", "display": "summarized"}); } if !req.tools.is_empty() { body["tools"] = json!(req.tools.iter().map(|t| json!({"name": t["name"], "description": t["description"], "input_schema": t["parameters"]})).collect::>()); } let mut r = agent() .post(&format!("{}/messages", req.base_url)) .header("Content-Type", "application/json") .header("x-api-key", &req.api_key) .header("anthropic-version", ANTHROPIC_VERSION); if req.kind == ProviderKind::OpenCodeGo { // Its Anthropic-style door takes the key as a bearer token too. r = r.header("Authorization", &format!("Bearer {}", req.api_key)).header("x-opencode-session", &opencode_session(req)); } if claude_fallbacks(&req.model) { body["fallbacks"] = json!("default"); r = r.header("anthropic-beta", "server-side-fallback-2026-07-01"); } // SAND-E0418 turnBackendUnreachable: retryable. let resp = r.send_json(&body).map_err(|e| Failure::retry(connect_error(req.kind, &e.to_string())))?; let status = resp.status().as_u16(); if status >= 400 { let ra = retry_after_ms(&resp); let text = resp.into_body().read_to_string().unwrap_or_default(); return Err(http_error(status, &text, ra)); } let mut out = StepOutput::default(); let mut blocks: Vec = vec![]; let mut partial_json: Vec = vec![]; let mut error = None; read_sse(resp.into_body(), cancel, send, |ev, data| { let Ok(v) = serde_json::from_str::(data) else { return true }; let kind = if ev.is_empty() { v["type"].as_str().unwrap_or("") } else { ev }; match kind { "content_block_start" => { let i = v["index"].as_u64().unwrap_or(blocks.len() as u64) as usize; while blocks.len() <= i { blocks.push(Value::Null); partial_json.push(String::new()); } blocks[i] = v["content_block"].clone(); } "content_block_delta" => { let i = v["index"].as_u64().unwrap_or(0) as usize; let d = &v["delta"]; let block = blocks.get_mut(i); match d["type"].as_str() { Some("text_delta") => { let t = d["text"].as_str().unwrap_or(""); out.text.push_str(t); if let Some(b) = block { let joined = format!("{}{}", b["text"].as_str().unwrap_or(""), t); b["text"] = json!(joined); } send(StreamEvent::Text(t.to_string())); } Some("thinking_delta") => { let t = d["thinking"].as_str().unwrap_or(""); out.thinking.push_str(t); if let Some(b) = block { let joined = format!("{}{}", b["thinking"].as_str().unwrap_or(""), t); b["thinking"] = json!(joined); } send(StreamEvent::Thinking(t.to_string())); } Some("signature_delta") => { if let Some(b) = block { let joined = format!("{}{}", b["signature"].as_str().unwrap_or(""), d["signature"].as_str().unwrap_or("")); b["signature"] = json!(joined); } } Some("input_json_delta") => { if let Some(p) = partial_json.get_mut(i) { p.push_str(d["partial_json"].as_str().unwrap_or("")); } } _ => {} } } "content_block_stop" => { let i = v["index"].as_u64().unwrap_or(0) as usize; if let Some(b) = blocks.get_mut(i) { if b["type"] == "tool_use" { let raw = partial_json.get(i).cloned().unwrap_or_default(); let input: Value = if raw.trim().is_empty() { json!({}) } else { serde_json::from_str(&raw).unwrap_or(json!({})) }; b["input"] = input.clone(); let call = ToolCall { id: b["id"].as_str().unwrap_or("").into(), name: b["name"].as_str().unwrap_or("").into(), args: input.to_string(), status: ToolStatus::Waiting }; send(StreamEvent::ToolCall(call.clone())); out.tool_calls.push(call); } } } "message_delta" => { if v["delta"]["stop_reason"] == "refusal" { out.text.push_str("\n\n（模型拒絕回答這個要求。）"); send(StreamEvent::Text("\n\n（模型拒絕回答這個要求。）".into())); } else if v["delta"]["stop_reason"] == "max_tokens" { out.text.push_str("\n\n（回答太長，已達長度上限。）"); send(StreamEvent::Text("\n\n（回答太長，已達長度上限。）".into())); } } "message_stop" => return false, "error" => { error = Some(v["error"]["message"].as_str().unwrap_or("未知錯誤").to_string()); return false; } _ => {} } true })?; if let Some(e) = error { return Err(stream_failure(&e)); } out.blocks = blocks.into_iter().filter(|b| !b.is_null()).collect(); Ok(out) } fn connect_error(kind: ProviderKind, e: &str) -> String { if kind == ProviderKind::Ollama { format!("連不上 Ollama：{e}\n請先開啟 Ollama，並用 `ollama pull` 下載模型。") } else { format!("連不上 {}：{e}", kind.label()) } } /// OpenCode Go: which API style each model speaks, from the public models list (fetched once). static OPENCODE_STYLES: std::sync::Mutex>> = std::sync::Mutex::new(None); fn remember_opencode_styles(_base: &str, list: &Value) { let map = opencode_styles_of(list); if !map.is_empty() { *OPENCODE_STYLES.lock().unwrap() = Some(map); } } /// {"data":[{"id":"qwen3.8-max","endpoint":{"messages":{…}}}, …]} → id → "messages" | "chat-completions" | … pub fn opencode_styles_of(list: &Value) -> std::collections::HashMap { list["data"] .as_array() .into_iter() .flatten() .filter_map(|m| { let id = m["id"].as_str()?; let ep = m["endpoint"].as_object()?; // A model with a chat-completions door uses it (AnyBot's best-tested path); else the first. let style = if ep.contains_key("chat-completions") { "chat-completions" } else if ep.contains_key("messages") { "messages" } else { ep.keys().next()? }; Some((id.to_string(), style.to_string())) }) .collect() } /// The API style for a model: "messages" (Anthropic-style) or "chat-completions" (everything /// else; OpenCode translates for models whose own style is Responses or Gemini). pub fn opencode_go_style(base: &str, model: &str) -> String { let known = OPENCODE_STYLES.lock().unwrap().as_ref().and_then(|m| m.get(model).cloned()); let style = known.or_else(|| { let v: Value = agent().get(&format!("{base}/v1/models")).call().ok()?.into_body().read_json().ok()?; remember_opencode_styles(base, &v); OPENCODE_STYLES.lock().unwrap().as_ref().and_then(|m| m.get(model).cloned()) }); style.unwrap_or_else(|| { // Offline guess from the published list: Claude, Qwen and MiniMax M3 are Anthropic-style. let m = model.to_lowercase(); if m.starts_with("claude") || m.starts_with("qwen") || m == "minimax-m3" { "messages".into() } else { "chat-completions".into() } }) } fn opencode_session(req: &ChatRequest) -> String { if req.session.is_empty() { "anybot".into() } else { req.session.clone() } } /// Ask the provider which models it offers. pub fn list_models(kind: ProviderKind, base_url: &str, api_key: &str) -> Result, String> { if kind.needs_key() && api_key.is_empty() { return Err("請先貼上 API 金鑰".into()); } let url = if kind == ProviderKind::OpenCodeGo { format!("{base_url}/v1/models") } else { format!("{base_url}/models") }; let mut r = agent().get(&url); if kind == ProviderKind::Anthropic { r = r.header("x-api-key", api_key).header("anthropic-version", ANTHROPIC_VERSION); } else if !api_key.is_empty() { r = r.header("Authorization", &format!("Bearer {api_key}")); } let resp = r.call().map_err(|e| connect_error(kind, &e.to_string()))?; let status = resp.status().as_u16(); let text = resp.into_body().read_to_string().map_err(|e| e.to_string())?; if status >= 400 { return Err(http_error(status, &text, None).msg); } let v: Value = serde_json::from_str(&text).map_err(|e| e.to_string())?; if kind == ProviderKind::OpenCodeGo { remember_opencode_styles(base_url, &v); } let list = v["data"].as_array().or_else(|| v["models"].as_array()).cloned().unwrap_or_default(); let mut ids: Vec = list .iter() .filter_map(|m| m["id"].as_str().or_else(|| m["name"].as_str())) .map(|s| s.trim_start_matches("models/").to_string()) .collect(); ids.sort(); ids.dedup(); if ids.is_empty() { return Err("供應商沒有回傳任何模型".into()); } Ok(ids) } #[cfg(test)] mod tests { use super::*; use std::io::{Read, Write}; use std::net::TcpListener; /// Serve one canned SSE reply on a local port and return its base URL. pub fn fake_server(body: &'static str) -> String { let l = TcpListener::bind("127.0.0.1:0").unwrap(); let port = l.local_addr().unwrap().port(); std::thread::spawn(move || { let (mut s, _) = l.accept().unwrap(); // Read the whole request (headers + body) before answering. let mut req = Vec::new(); let mut buf = [0u8; 4096]; loop { let n = s.read(&mut buf).unwrap_or(0); if n == 0 { break; } req.extend_from_slice(&buf[..n]); let text = String::from_utf8_lossy(&req).to_string(); if let Some(end) = text.find("\r\n\r\n") { let len = text[..end] .lines() .find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::().unwrap_or(0))) .unwrap_or(0); if req.len() >= end + 4 + len { break; } } } let head = "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n"; let _ = s.write_all(head.as_bytes()); let _ = s.write_all(body.as_bytes()); }); format!("http://127.0.0.1:{port}/v1") } fn run(kind: ProviderKind, body: &'static str) -> StepOutput { let req = ChatRequest { kind, base_url: fake_server(body), api_key: "test".into(), model: "m".into(), system: "s".into(), history: vec![Message::user("hi")], tools: crate::tools::specs(), session: String::new(), }; let cancel = AtomicBool::new(false); match run_step(&req, &cancel, &|_| {}) { Ok(o) => o, Err(e) => panic!("{e}"), } } #[test] fn a_busy_model_mid_stream_is_waited_out() { let f = stream_failure("The model is currently at capacity due to high demand. Please try again in a few minutes"); assert!(f.retryable && f.retry_after_ms == Some(BUSY_RETRY_MS) && f.msg.starts_with("模型供應商現在太忙"), "{}", f.msg); let f = stream_failure("Invalid tool schema"); assert!(!f.retryable && f.msg.starts_with("模型回報錯誤"), "{}", f.msg); } #[test] fn a_gateways_error_page_reads_as_words() { let page = "\n\n\n\n\nError response\n\n\n

Error response

\n

Error code: 502

\n

Message: Bad Gateway.

\n\n\n"; let f = http_error(502, page, None); assert!(f.retryable); assert!(!f.msg.contains('<') && !f.msg.contains("color-scheme"), "{}", f.msg); assert!(f.msg.contains("Bad Gateway"), "{}", f.msg); assert_eq!(page_words("upstream timed out"), "upstream timed out"); } #[test] fn backoff_matches_original() { assert_eq!([backoff_delay_ms(1), backoff_delay_ms(2), backoff_delay_ms(3)], [1000, 2000, 4000]); assert!(http_error(429, "{}", None).retryable); assert!(http_error(503, "{}", Some(2000)).retry_after_ms == Some(2000)); assert!(!http_error(401, "{}", None).retryable); assert!(!http_error(400, "{\"error\":{\"message\":\"prompt is too long\"}}", None).retryable); } #[test] fn openai_style_stream() { let body = "data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"hm\"}}]}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"你好\"}}]}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"！\"}}]}\n\ndata: [DONE]\n\n"; let o = run(ProviderKind::Xai, body); assert_eq!(o.text, "你好！"); assert_eq!(o.thinking, "hm"); } #[test] fn openai_tool_calls_in_pieces() { let body = "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_a\",\"function\":{\"name\":\"read_file\",\"arguments\":\"{\\\"pa\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"th\\\":\\\"a.txt\\\"}\"}}]}}]}\n\ndata: [DONE]\n\n"; let o = run(ProviderKind::OpenAi, body); assert_eq!(o.tool_calls.len(), 1); assert_eq!(o.tool_calls[0].name, "read_file"); assert_eq!(o.tool_calls[0].args, "{\"path\":\"a.txt\"}"); } #[test] fn claude_style_stream() { let body = "event: message_start\ndata: {\"type\":\"message_start\"}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"ok\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"sig\"}}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"Hello\"}}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":2,\"content_block\":{\"type\":\"tool_use\",\"id\":\"tu1\",\"name\":\"list_dir\",\"input\":{}}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"path\\\": \\\"x\\\"}\"}}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":2}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"; let o = run(ProviderKind::Anthropic, body); assert_eq!(o.text, "Hello"); assert_eq!(o.thinking, "ok"); assert_eq!(o.tool_calls[0].name, "list_dir"); assert_eq!(o.blocks.len(), 3); assert_eq!(o.blocks[0]["signature"], "sig"); assert_eq!(o.blocks[2]["input"]["path"], "x"); } #[test] fn history_keeps_tool_turns() { let mut a = Message::assistant("", "x"); a.tool_calls = vec![ToolCall { id: "c1".into(), name: "list_dir".into(), args: "{}".into(), status: ToolStatus::Done }]; let mut s = Message::assistant("old stuff", ""); s.summary = true; let req = ChatRequest { kind: ProviderKind::Anthropic, base_url: String::new(), api_key: String::new(), model: "m".into(), system: "sys".into(), history: vec![s, Message::user("hi"), a, Message::tool_result("c1", "list_dir", "a.txt"), Message::user("next")], tools: vec![], session: String::new(), }; let m = anthropic_messages(&req); assert_eq!(m.len(), 3); assert_eq!(m[1]["content"][0]["type"], "tool_use"); assert_eq!(m[2]["content"][0]["type"], "tool_result"); assert_eq!(m[2]["content"][1]["text"], "next"); let o = openai_messages(&ChatRequest { kind: ProviderKind::OpenAi, ..req }); assert!(o[0]["content"].as_str().unwrap().contains("old stuff")); assert_eq!(o[3]["role"], "tool"); } #[test] fn screenshots_reach_the_model_and_old_ones_drop() { let n = crate::computer::KEEP_SCREENSHOTS + 1; let mut history = vec![Message::user("look")]; for i in 0..n { let mut a = Message::assistant("", "x"); a.tool_calls = vec![ToolCall { id: format!("s{i}"), name: "computer_use_screenshot".into(), args: "{}".into(), status: ToolStatus::Done }]; history.push(a); let mut r = Message::tool_result(&format!("s{i}"), "computer_use_screenshot", "shot"); r.images = vec![format!("IMG{i}")]; history.push(r); } let req = ChatRequest { kind: ProviderKind::Anthropic, base_url: String::new(), api_key: String::new(), model: "m".into(), system: String::new(), history, tools: vec![], session: String::new() }; let m = anthropic_messages(&req); let all = serde_json::to_string(&m).unwrap(); assert!(!all.contains("IMG0") && all.contains(OLD_SCREENSHOT_NOTE)); assert!(all.contains(&format!("IMG{}", n - 1)) && all.contains("\"media_type\":\"image/jpeg\"")); let o = openai_messages(&ChatRequest { kind: ProviderKind::OpenAi, ..req }); // Each tool answer is followed by its picture as a user message (never between tool results). for w in o.windows(2) { if w[0]["role"] == "user" && w[0]["content"].is_array() && w[1]["role"] == "tool" { panic!("picture between tool results"); } } let pics: Vec<&Value> = o.iter().filter(|x| x["role"] == "user" && x["content"].is_array()).collect(); assert_eq!(pics.len(), crate::computer::KEEP_SCREENSHOTS); assert!(pics.last().unwrap()["content"][1]["image_url"]["url"].as_str().unwrap().starts_with("data:image/jpeg;base64,IMG")); } /// A server that records the request it gets, then answers with `body`. fn recording_server(body: &'static str) -> (String, std::sync::mpsc::Receiver) { let l = TcpListener::bind("127.0.0.1:0").unwrap(); let port = l.local_addr().unwrap().port(); let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { let (mut s, _) = l.accept().unwrap(); let mut req = Vec::new(); let mut buf = [0u8; 8192]; loop { let n = s.read(&mut buf).unwrap_or(0); if n == 0 { break; } req.extend_from_slice(&buf[..n]); let text = String::from_utf8_lossy(&req).to_string(); if let Some(end) = text.find("\r\n\r\n") { let len = text[..end].lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::().unwrap_or(0))).unwrap_or(0); if req.len() >= end + 4 + len { break; } } } let _ = tx.send(String::from_utf8_lossy(&req).to_string()); let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n"); let _ = s.write_all(body.as_bytes()); }); (format!("http://127.0.0.1:{port}"), rx) } #[test] fn opencode_go_routes_each_model_its_way() { // The public list's shape (trimmed). let list: Value = serde_json::from_str(r#"{"data":[ {"id":"kimi-k3","endpoint":{"chat-completions":{"base_url":"https://opencode.ai/inference/go/openai/v1"}}}, {"id":"qwen3.8-max","endpoint":{"messages":{"base_url":"https://opencode.ai/inference/go/anthropic/v1"}}}, {"id":"grok-4.7","endpoint":{"responses":{"base_url":"https://opencode.ai/inference/go/openai/v1"}}}]}"#).unwrap(); let styles = opencode_styles_of(&list); assert_eq!(styles["kimi-k3"], "chat-completions"); assert_eq!(styles["qwen3.8-max"], "messages"); assert_eq!(styles["grok-4.7"], "responses"); remember_opencode_styles("", &list); let cancel = AtomicBool::new(false); let ask = |model: &str, body: &'static str| { let (base, rx) = recording_server(body); let req = ChatRequest { kind: ProviderKind::OpenCodeGo, base_url: base, api_key: "sk-go".into(), model: model.into(), system: "s".into(), history: vec![Message::user("hi")], tools: vec![], session: "anybot-bot-7".into() }; let out = run_step(&req, &cancel, &|_| {}).unwrap(); (out, rx.recv().unwrap()) }; let (out, seen) = ask("kimi-k3", "data: {\"choices\":[{\"delta\":{\"content\":\"你好\"}}]}\n\ndata: [DONE]\n\n"); assert_eq!(out.text, "你好"); assert!(seen.starts_with("POST /openai/v1/chat/completions "), "{seen}"); let low = seen.to_lowercase(); assert!(low.contains("authorization: bearer sk-go") && low.contains("x-opencode-session: anybot-bot-7"), "{seen}"); let (out, seen) = ask("qwen3.8-max", "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"嗨\"}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"); assert_eq!(out.text, "嗨"); assert!(seen.starts_with("POST /anthropic/v1/messages "), "{seen}"); let low = seen.to_lowercase(); assert!(low.contains("x-api-key: sk-go") && low.contains("x-opencode-session: anybot-bot-7"), "{seen}"); // Responses-style models go through OpenCode's chat-completions translation. let (_, seen) = ask("grok-4.7", "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}]}\n\ndata: [DONE]\n\n"); assert!(seen.starts_with("POST /openai/v1/chat/completions "), "{seen}"); } /// Network: OpenCode Go's real model list (public) and the style of a few models. #[test] #[ignore] fn opencode_go_live_list() { // A wrong key is refused (so pasting a bad key shows 金鑰無效 right away). assert!(list_models(ProviderKind::OpenCodeGo, OPENCODE_GO_BASE, "sk-wrong").unwrap_err().contains("401")); // The styles come from the public list (no key). assert_eq!(opencode_go_style(OPENCODE_GO_BASE, "qwen3.8-max"), "messages"); assert_eq!(opencode_go_style(OPENCODE_GO_BASE, "glm-5.3"), "chat-completions"); } #[test] fn a_model_that_cannot_see_gets_no_pictures() { let _g = vision_test_guard(); assert!(image_refused("錯誤 400：At most 0 image(s) may be provided in one prompt. (parameter=image)")); assert!(image_refused("This model does not support image input")); assert!(image_refused("unknown variant `image_url`, expected `text`")); assert!(!image_refused("錯誤 400：Invalid credential")); // The first request fails like the user's model did; the retry has no picture. let l = TcpListener::bind("127.0.0.1:0").unwrap(); let port = l.local_addr().unwrap().port(); let (tx, rx) = std::sync::mpsc::channel::(); std::thread::spawn(move || { for (n, s) in l.incoming().flatten().enumerate() { let mut s = s; let mut req = Vec::new(); let mut buf = [0u8; 65536]; loop { let k = s.read(&mut buf).unwrap_or(0); if k == 0 { break; } req.extend_from_slice(&buf[..k]); let text = String::from_utf8_lossy(&req).to_string(); if let Some(end) = text.find("\r\n\r\n") { let len = text[..end].lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::().unwrap_or(0))).unwrap_or(0); if req.len() >= end + 4 + len { break; } } } let _ = tx.send(String::from_utf8_lossy(&req).to_string()); if n == 0 { let body = r#"{"error":{"message":"At most 0 image(s) may be provided in one prompt. (parameter=image)"}}"#; let _ = write!(s, "HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()); } else { let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\ndata: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}]}\n\ndata: [DONE]\n\n"); } } }); let mut shot = Message::tool_result("c1", "computer_use_screenshot", "你的螢幕截圖"); shot.images = vec!["SHOTDATA".into()]; let mut asked = Message::assistant("", ""); asked.tool_calls = vec![ToolCall { id: "c1".into(), name: "computer_use_screenshot".into(), args: "{}".into(), status: ToolStatus::Done }]; let req = ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "blind-model".into(), system: String::new(), history: vec![Message::user("看一下畫面"), asked, shot], tools: vec![], session: String::new() }; let cancel = AtomicBool::new(false); let out = run_step(&req, &cancel, &|_| {}).unwrap(); assert_eq!(out.text, "ok"); let first = rx.recv().unwrap(); let second = rx.recv().unwrap(); assert!(first.contains("SHOTDATA") && !second.contains("SHOTDATA") && second.contains("看不到圖片"), "{second}"); assert!(!sees_images(&req)); } /// A fake model: refuses pictures when `blind`, else answers `reply`; hands over each request. fn fake_model(blind: bool, reply: &'static str) -> (String, std::sync::mpsc::Receiver) { let l = TcpListener::bind("127.0.0.1:0").unwrap(); let port = l.local_addr().unwrap().port(); let (tx, rx) = std::sync::mpsc::channel::(); std::thread::spawn(move || { for s in l.incoming().flatten() { let mut s = s; let mut req = Vec::new(); let mut buf = [0u8; 65536]; loop { let k = s.read(&mut buf).unwrap_or(0); if k == 0 { break; } req.extend_from_slice(&buf[..k]); let text = String::from_utf8_lossy(&req).to_string(); if let Some(end) = text.find("\r\n\r\n") { let len = text[..end].lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::().unwrap_or(0))).unwrap_or(0); if req.len() >= end + 4 + len { break; } } } let text = String::from_utf8_lossy(&req).to_string(); let _ = tx.send(text.clone()); if blind && text.contains("image_url") { let body = r#"{"error":{"message":"This model does not support image input"}}"#; let _ = write!(s, "HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()); } else { let chunk = serde_json::json!({"choices": [{"delta": {"content": reply}}]}); let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\ndata: {chunk}\n\ndata: [DONE]\n\n"); } } }); (format!("http://127.0.0.1:{port}/v1"), rx) } #[test] fn a_model_that_sees_describes_pictures_for_one_that_cannot() { let _g = vision_test_guard(); let (blind_url, blind_rx) = fake_model(true, "好的，我看到描述了"); let (eyes_url, eyes_rx) = fake_model(false, "一張報表截圖，上面寫著「十月營收 42 萬」。"); set_eyes(vec![ // Not there (a typo in a model name, a dead server…): the next one is asked. Eyes { kind: ProviderKind::Custom, base_url: "http://127.0.0.1:9/v1".into(), api_key: String::new(), model: "gone".into() }, Eyes { kind: ProviderKind::Custom, base_url: eyes_url, api_key: String::new(), model: "eyes-model".into() }, ]); let mut photo = Message::user("幫我看這張圖"); photo.images = vec!["UElDVFVSRS1PTkU=".into()]; let req = ChatRequest { kind: ProviderKind::Custom, base_url: blind_url, api_key: String::new(), model: "text-only".into(), system: String::new(), history: vec![photo], tools: vec![], session: String::new() }; let said = std::sync::Mutex::new(vec![]); let out = run_step(&req, &AtomicBool::new(false), &|e| { if let StreamEvent::Retrying(m) = e { said.lock().unwrap().push(m); } }) .unwrap(); assert_eq!(out.text, "好的，我看到描述了"); // The helper got the picture and what the person said with it. let helper = eyes_rx.recv().unwrap(); assert!(helper.contains("UElDVFVSRS1PTkU=") && helper.contains("幫我看這張圖")); // The blind model got the description instead of the picture. let _refused = blind_rx.recv().unwrap(); let again = blind_rx.recv().unwrap(); assert!(!again.contains("UElDVFVSRS1PTkU=") && again.contains("十月營收 42 萬") && again.contains("eyes-model"), "{again}"); assert!(said.lock().unwrap().iter().any(|m| m.contains("看得懂圖"))); // The same picture later: described from memory, no second call to the helper. let mut photo2 = Message::user("再看一次"); photo2.images = vec!["UElDVFVSRS1PTkU=".into()]; let req2 = ChatRequest { history: vec![photo2], ..req.clone() }; run_step(&req2, &AtomicBool::new(false), &|_| {}).unwrap(); assert!(blind_rx.recv().unwrap().contains("十月營收 42 萬")); assert!(eyes_rx.recv_timeout(std::time::Duration::from_millis(300)).is_err()); set_eyes(vec![]); } #[test] fn steps_without_results_are_closed() { let mut a = Message::assistant("", ""); a.tool_calls = vec![ ToolCall { id: "t1".into(), name: "shell".into(), args: "{}".into(), status: ToolStatus::Denied }, ToolCall { id: "t2".into(), name: "browser_fill_login".into(), args: "{}".into(), status: ToolStatus::Waiting }, ]; let h = vec![Message::user("a"), a, Message::tool_result("t2", "browser_fill_login", "ok"), Message::tool_result("zz", "x", "stray"), Message::user("再一次")]; let r = repair_history(&h); let roles: Vec = r.iter().map(|m| format!("{:?}:{}", m.role, m.tool_call_id)).collect(); assert_eq!(roles, vec!["User:", "Assistant:", "Tool:t2", "Tool:t1", "User:"]); assert!(r[3].text.contains("沒有允許")); // The OpenAI-style body pairs every call with a result. let req = ChatRequest { kind: ProviderKind::Custom, base_url: String::new(), api_key: String::new(), model: "m".into(), system: String::new(), history: r, tools: vec![], session: String::new() }; let msgs = openai_messages(&req); let i = msgs.iter().position(|m| m["tool_calls"].is_array()).unwrap(); assert_eq!(msgs[i + 1]["role"], "tool"); assert_eq!(msgs[i + 2]["role"], "tool"); } }