diff --git a/src/app.rs b/src/app.rs index 73b5b3c..024340a 100644 --- a/src/app.rs +++ b/src/app.rs @@ -707,6 +707,8 @@ impl App { let default_screen = if screens.contains(&default_computer) { default_computer.clone() } else { screens[0].clone() }; tool_specs.extend(crate::computer::specs_for(&screens, &default_screen)); } + // 看圖小幫手: the person's models that may see pictures, for a bot whose model can't. + providers::set_eyes(self.eyes()); let req = ChatRequest { kind: b.provider, base_url: providers::resolve_base(b.provider, &ps.base_url), @@ -1297,6 +1299,34 @@ impl App { } } + /// Models to describe pictures with, best first: each provider with a key, its own default + /// model, then models of it known to see pictures. + fn eyes(&self) -> Vec { + use ProviderKind as K; + let mut v = vec![]; + for kind in [K::Xai, K::Anthropic, K::OpenAi, K::Gemini, K::OpenRouter, K::OpenCodeGo, K::Custom, K::Ollama] { + let ps = self.state.provider(kind); + let key = providers::resolve_key(kind, &ps.api_key); + if kind.needs_key() && key.is_empty() { + continue; + } + if matches!(kind, K::Custom | K::Ollama) && ps.base_url.trim().is_empty() { + continue; + } + let base = providers::resolve_base(kind, &ps.base_url); + let mut models: Vec = vec![]; + if !ps.default_model.trim().is_empty() { + models.push(ps.default_model.trim().to_string()); + } + models.extend(providers::seeing_models(kind).iter().map(|m| m.to_string())); + models.dedup(); + for m in models { + v.push(providers::Eyes { kind, base_url: base.clone(), api_key: key.clone(), model: m }); + } + } + v + } + fn models_for(&self, kind: ProviderKind) -> Vec { let ps = self.state.provider(kind); let mut v: Vec = if ps.known_models.is_empty() { diff --git a/src/providers.rs b/src/providers.rs index a888237..c195298 100644 --- a/src/providers.rs +++ b/src/providers.rs @@ -371,9 +371,10 @@ pub fn run_step(req: &ChatRequest, cancel: &AtomicBool, send: &dyn Fn(StreamEven let req = &repaired; let mut blind_copy: Option = None; loop { - // A model that can't see pictures gets the request without them. + // A model that can't see pictures gets the request without them: the person's pictures + // described by a model that can (看圖小幫手), screenshots as a short note. if !sees_images(req) && blind_copy.is_none() { - blind_copy = Some(without_images(req)); + blind_copy = Some(if DESCRIBING.with(|d| d.get()) { without_images(req) } else { with_descriptions(req, cancel, send) }); } let req = blind_copy.as_ref().unwrap_or(req); let result = match req.kind { @@ -519,6 +520,142 @@ pub fn repair_history(history: &[Message]) -> Vec { out } +// ---------- 看圖小幫手 ---------- + +/// A model of the person's that sees pictures (another provider, or another model). +#[derive(Clone, Debug, PartialEq)] +pub struct Eyes { + pub kind: ProviderKind, + pub base_url: String, + pub api_key: String, + pub model: String, +} + +static EYES: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + +thread_local! { + /// Inside a helper's own call (it never asks a helper itself). + static DESCRIBING: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// The models to ask, best first (the app sets this from the person's providers). +pub fn set_eyes(v: Vec) { + *EYES.lock().unwrap() = v; +} + +/// Models likely to see pictures, by provider. +pub fn seeing_models(kind: ProviderKind) -> &'static [&'static str] { + match kind { + ProviderKind::Xai => &["grok-4", "grok-4-fast"], + ProviderKind::Anthropic => &["claude-sonnet-5-5", "claude-haiku-5-5"], + ProviderKind::OpenAi => &["gpt-5-mini", "gpt-4.1"], + ProviderKind::Gemini => &["gemini-2.5-flash"], + ProviderKind::OpenRouter => &["openrouter/auto"], + _ => &[], + } +} + +const EYES_PROMPT: &str = "你是「看圖小幫手」：替一個看不到圖片的 AI 助理看圖。用繁體中文把圖片完整描述出來，讓它不用看圖也能幫使用者：畫面是什麼、主要的東西和位置、所有看得到的文字（原文照抄，包括數字、表格、按鈕和錯誤訊息）、顏色或狀態等重點。只描述，不評論、不問問題、不說你是誰。"; + +/// Descriptions already made (by the picture), kept across restarts. +fn notes_path() -> std::path::PathBuf { + crate::model::State::path().with_file_name("image-notes.json") +} + +fn notes() -> &'static std::sync::Mutex> { + static N: std::sync::OnceLock>> = std::sync::OnceLock::new(); + N.get_or_init(|| std::sync::Mutex::new(if cfg!(test) { Default::default() } else { std::fs::read(notes_path()).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default() })) +} + +/// Helpers that just failed (wrong model name, no credit, down…), skipped for a while. +fn failed_eyes() -> &'static std::sync::Mutex> { + static F: std::sync::OnceLock>> = std::sync::OnceLock::new(); + F.get_or_init(Default::default) +} + +fn picture_key(b64: &str) -> String { + let d = ring::digest::digest(&ring::digest::SHA256, b64.as_bytes()); + d.as_ref()[..12].iter().map(|b| format!("{b:02x}")).collect() +} + +/// Ask the first helper that can see pictures to describe this one: (description, model). +fn describe(picture: &str, ask: &str, not: &ChatRequest, cancel: &AtomicBool) -> Option<(String, String)> { + let key = picture_key(picture); + if let Some(n) = notes().lock().unwrap().get(&key) { + return Some(n.clone()); + } + let helpers: Vec = EYES.lock().unwrap().clone(); + for e in helpers { + let req = ChatRequest { + kind: e.kind, + base_url: e.base_url.clone(), + api_key: e.api_key.clone(), + model: e.model.clone(), + system: EYES_PROMPT.into(), + history: vec![Message { images: vec![picture.to_string()], ..Message::user(ask) }], + tools: vec![], + session: String::new(), + }; + let id = vision_key(&req); + let resting = failed_eyes().lock().unwrap().get(&id).map(|t| t.elapsed().as_secs() < 600).unwrap_or(false); + if (req.base_url == not.base_url && req.model == not.model) || !sees_images(&req) || resting || cancel.load(Ordering::Relaxed) { + continue; + } + DESCRIBING.with(|d| d.set(true)); + let out = run_step(&req, cancel, &|_| {}); + DESCRIBING.with(|d| d.set(false)); + match out { + // It may have turned out not to see pictures after all (and answered without). + Ok(o) if sees_images(&req) && !o.text.trim().is_empty() => { + let note = (o.text.trim().to_string(), e.model.clone()); + let mut all = notes().lock().unwrap(); + if all.len() > 500 { + all.clear(); + } + all.insert(key, note.clone()); + if let (false, Ok(b)) = (cfg!(test), serde_json::to_vec(&*all)) { + let _ = std::fs::write(notes_path(), b); + } + return Some(note); + } + _ => { + failed_eyes().lock().unwrap().insert(id, std::time::Instant::now()); + } + } + } + None +} + +/// The request for a model that can't see pictures: each of the person's pictures replaced by +/// what a helper saw in it; screenshots by a short note. +fn with_descriptions(req: &ChatRequest, cancel: &AtomicBool, send: &dyn Fn(StreamEvent)) -> ChatRequest { + let mut r = req.clone(); + let mut told = false; + for m in r.history.iter_mut().filter(|m| !m.images.is_empty()) { + if m.role == Role::Tool { + m.images.clear(); + m.text = if m.text.is_empty() { BLIND_SCREENSHOT_NOTE.to_string() } else { format!("{}\n{BLIND_SCREENSHOT_NOTE}", m.text) }; + continue; + } + let pictures = std::mem::take(&mut m.images); + let ask = format!("請描述這張圖片。使用者附上它時說：「{}」", m.text.chars().take(300).collect::()); + let mut added = String::new(); + for (i, p) in pictures.iter().enumerate() { + if !told && !notes().lock().unwrap().contains_key(&picture_key(p)) { + send(StreamEvent::Retrying("這個模型看不到圖片，正在請另一個看得懂圖的模型先看一遍…".into())); + told = true; + } + let which = if pictures.len() > 1 { format!("第 {} 張圖片", i + 1) } else { "這張圖片".to_string() }; + added.push_str(&match describe(p, &ask, req, cancel) { + Some((d, model)) => format!("\n\n（你看不到圖片；以下是「{model}」看過{which}後的描述）\n{d}"), + None => format!("\n\n{BLIND_ATTACHMENT_NOTE}（帳號裡也沒有看得懂圖片的模型可以代看。可以請使用者到「設定 › 模型」加一個，例如 Grok、Claude、GPT 或 Gemini。）"), + }); + } + m.text.push_str(&added); + } + r +} + /// The same request with pictures replaced by a short note. fn without_images(req: &ChatRequest) -> ChatRequest { let mut r = req.clone(); @@ -1231,6 +1368,82 @@ mod tests { assert!(!sees_images(&req)); } + /// A fake model: refuses pictures when `blind`, else answers `reply`; hands over each request. + fn fake_model(blind: bool, reply: &'static str) -> (String, std::sync::mpsc::Receiver) { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = l.local_addr().unwrap().port(); + let (tx, rx) = std::sync::mpsc::channel::(); + std::thread::spawn(move || { + for s in l.incoming().flatten() { + let mut s = s; + let mut req = Vec::new(); + let mut buf = [0u8; 65536]; + loop { + let k = s.read(&mut buf).unwrap_or(0); + if k == 0 { + break; + } + req.extend_from_slice(&buf[..k]); + let text = String::from_utf8_lossy(&req).to_string(); + if let Some(end) = text.find("\r\n\r\n") { + let len = text[..end].lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::().unwrap_or(0))).unwrap_or(0); + if req.len() >= end + 4 + len { + break; + } + } + } + let text = String::from_utf8_lossy(&req).to_string(); + let _ = tx.send(text.clone()); + if blind && text.contains("image_url") { + let body = r#"{"error":{"message":"This model does not support image input"}}"#; + let _ = write!(s, "HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()); + } else { + let chunk = serde_json::json!({"choices": [{"delta": {"content": reply}}]}); + let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\ndata: {chunk}\n\ndata: [DONE]\n\n"); + } + } + }); + (format!("http://127.0.0.1:{port}/v1"), rx) + } + + #[test] + fn a_model_that_sees_describes_pictures_for_one_that_cannot() { + let (blind_url, blind_rx) = fake_model(true, "好的，我看到描述了"); + let (eyes_url, eyes_rx) = fake_model(false, "一張報表截圖，上面寫著「十月營收 42 萬」。"); + set_eyes(vec![ + // Not there (a typo in a model name, a dead server…): the next one is asked. + Eyes { kind: ProviderKind::Custom, base_url: "http://127.0.0.1:9/v1".into(), api_key: String::new(), model: "gone".into() }, + Eyes { kind: ProviderKind::Custom, base_url: eyes_url, api_key: String::new(), model: "eyes-model".into() }, + ]); + let mut photo = Message::user("幫我看這張圖"); + photo.images = vec!["UElDVFVSRS1PTkU=".into()]; + let req = ChatRequest { kind: ProviderKind::Custom, base_url: blind_url, api_key: String::new(), model: "text-only".into(), system: String::new(), history: vec![photo], tools: vec![], session: String::new() }; + let said = std::sync::Mutex::new(vec![]); + let out = run_step(&req, &AtomicBool::new(false), &|e| { + if let StreamEvent::Retrying(m) = e { + said.lock().unwrap().push(m); + } + }) + .unwrap(); + assert_eq!(out.text, "好的，我看到描述了"); + // The helper got the picture and what the person said with it. + let helper = eyes_rx.recv().unwrap(); + assert!(helper.contains("UElDVFVSRS1PTkU=") && helper.contains("幫我看這張圖")); + // The blind model got the description instead of the picture. + let _refused = blind_rx.recv().unwrap(); + let again = blind_rx.recv().unwrap(); + assert!(!again.contains("UElDVFVSRS1PTkU=") && again.contains("十月營收 42 萬") && again.contains("eyes-model"), "{again}"); + assert!(said.lock().unwrap().iter().any(|m| m.contains("看得懂圖"))); + // The same picture later: described from memory, no second call to the helper. + let mut photo2 = Message::user("再看一次"); + photo2.images = vec!["UElDVFVSRS1PTkU=".into()]; + let req2 = ChatRequest { history: vec![photo2], ..req.clone() }; + run_step(&req2, &AtomicBool::new(false), &|_| {}).unwrap(); + assert!(blind_rx.recv().unwrap().contains("十月營收 42 萬")); + assert!(eyes_rx.recv_timeout(std::time::Duration::from_millis(300)).is_err()); + set_eyes(vec![]); + } + #[test] fn steps_without_results_are_closed() { let mut a = Message::assistant("", ""); diff --git a/src/tools.rs b/src/tools.rs index 44d8d16..71800ea 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -427,6 +427,12 @@ pub fn protected_dirs() -> Vec { v } +/// Inside the protected folders, what commands may still read: the chat's attachments (a picture +/// the person sent is the bot's to look at; the keys next to it are not). +pub fn readable_dirs() -> Vec { + crate::model::State::path().parent().map(|d| vec![d.join("attachments")]).unwrap_or_default() +} + fn sandbox_exec() -> Option<&'static str> { let p = "/usr/bin/sandbox-exec"; (cfg!(target_os = "macos") && Path::new(p).exists()).then_some(p) @@ -440,12 +446,13 @@ fn real(p: &Path) -> String { /// The shell command, wrapped in the sandbox when the system has one. #[cfg(test)] pub fn build_command(cmd: &str, workdir: &Path, protected: &[PathBuf], conversation: &str, request: &str) -> Command { - build_command_sealed(cmd, workdir, protected, None, conversation, request) + build_command_sealed(cmd, workdir, protected, &[], None, conversation, request) } /// `seal = Some(home)`: the bot's own computer. Nothing under the person's home folder is /// visible or writable except the bot's folder (`workdir`), like a separate machine. -pub fn build_command_sealed(cmd: &str, workdir: &Path, protected: &[PathBuf], seal: Option<&Path>, conversation: &str, request: &str) -> Command { +/// `readable`: folders inside `protected` that may still be read (never written). +pub fn build_command_sealed(cmd: &str, workdir: &Path, protected: &[PathBuf], readable: &[PathBuf], seal: Option<&Path>, conversation: &str, request: &str) -> Command { let mut c = match sandbox_exec() { Some(se) if !protected.is_empty() || seal.is_some() => { let rules: String = (0..protected.len()).map(|i| format!(" (subpath (param \"P{i}\"))")).collect(); @@ -453,10 +460,18 @@ pub fn build_command_sealed(cmd: &str, workdir: &Path, protected: &[PathBuf], se if seal.is_some() { profile.push_str("(deny file-read* file-write* (with no-report) (require-all (subpath (param \"HOME\")) (require-not (subpath (param \"ROOT\")))))"); } + // Later rules win: reading these is allowed again (writing stays denied). + let open: Vec<&PathBuf> = readable.iter().filter(|r| protected.iter().any(|p| r.starts_with(p))).collect(); + for i in 0..open.len() { + profile.push_str(&format!("(allow file-read* (subpath (param \"R{i}\")))")); + } let mut c = Command::new(se); for (i, p) in protected.iter().enumerate() { c.arg("-D").arg(format!("P{i}={}", real(p))); } + for (i, r) in open.iter().enumerate() { + c.arg("-D").arg(format!("R{i}={}", real(r))); + } if let Some(home) = seal { c.arg("-D").arg(format!("HOME={}", real(home))).arg("-D").arg(format!("ROOT={}", real(workdir))); } @@ -962,7 +977,7 @@ fn edit_file(p: &Path, old: &str, new: &str) -> Result { } fn shell(cmd: &str, workdir: &Path, timeout_ms: u64, cancel: &AtomicBool, ids: &RunIds, seal: Option<&Path>) -> Result { - run_command(build_command_sealed(cmd, workdir, &protected_dirs(), seal, &ids.conversation, &ids.request), cmd, timeout_ms, cancel, ids) + run_command(build_command_sealed(cmd, workdir, &protected_dirs(), &readable_dirs(), seal, &ids.conversation, &ids.request), cmd, timeout_ms, cancel, ids) } /// Run a wrapper command (ssh / docker exec) the same way: background hand-off, stop, output. @@ -1185,6 +1200,16 @@ mod tests { assert!(!String::from_utf8_lossy(&out.stdout).contains("top-secret")); let out = build_command(&format!("cat '{}/k.txt'", secret.display()), &w, &[], "1", "r").output().unwrap(); assert!(String::from_utf8_lossy(&out.stdout).contains("top-secret")); + // The chat's attachments inside it: readable, not writable; the keys next to them: hidden. + let att = secret.join("attachments"); + std::fs::create_dir_all(&att).unwrap(); + std::fs::write(att.join("photo.txt"), "the-photo").unwrap(); + let run = |cmd: String| build_command_sealed(&cmd, &w, &[secret.clone()], &[att.clone()], None, "1", "r").output().unwrap(); + assert!(String::from_utf8_lossy(&run(format!("cat '{}/photo.txt'", att.display())).stdout).contains("the-photo")); + assert!(!String::from_utf8_lossy(&run(format!("cat '{}/k.txt'", secret.display())).stdout).contains("top-secret")); + run(format!("echo changed > '{}/photo.txt'", att.display())); + assert_eq!(std::fs::read_to_string(att.join("photo.txt")).unwrap(), "the-photo"); + let _ = std::fs::remove_dir_all(&secret); } #[test]