diff --git a/src/providers.rs b/src/providers.rs index c195298..f169b95 100644 --- a/src/providers.rs +++ b/src/providers.rs @@ -372,7 +372,7 @@ pub fn run_step(req: &ChatRequest, cancel: &AtomicBool, send: &dyn Fn(StreamEven let mut blind_copy: Option = None; loop { // A model that can't see pictures gets the request without them: the person's pictures - // described by a model that can (看圖小幫手), screenshots as a short note. + // and the newest screenshot described by a model that can (看圖小幫手). if !sees_images(req) && blind_copy.is_none() { blind_copy = Some(if DESCRIBING.with(|d| d.get()) { without_images(req) } else { with_descriptions(req, cancel, send) }); } @@ -458,6 +458,8 @@ fn kept_images(history: &[Message]) -> Vec { const OLD_SCREENSHOT_NOTE: &str = "(較早的截圖已省略)"; /// Said in place of a screenshot when the model can't take pictures. const BLIND_SCREENSHOT_NOTE: &str = "(截圖已拍好,但這個模型看不到圖片,所以沒有附上。請改用文字方式確認畫面,例如讀取視窗或網頁的文字。)"; +/// What a helper is asked about a screenshot. +const SCREEN_ASK: &str = "這是一台電腦螢幕的截圖。請描述:開著哪些視窗或 App、畫面上的文字(照抄重要的字)、按鈕和欄位,以及它們大約的位置(以這張圖的像素座標,左上角是 0,0)。不要猜看不清楚的字。"; const BLIND_ATTACHMENT_NOTE: &str = "(使用者附了圖片,但這個模型看不到圖片。)"; /// Models (base url + name) that refused a picture: later requests leave pictures out. @@ -631,10 +633,25 @@ fn describe(picture: &str, ask: &str, not: &ChatRequest, cancel: &AtomicBool) -> fn with_descriptions(req: &ChatRequest, cancel: &AtomicBool, send: &dyn Fn(StreamEvent)) -> ChatRequest { let mut r = req.clone(); let mut told = false; - for m in r.history.iter_mut().filter(|m| !m.images.is_empty()) { + // The newest screenshot is described too (what the bot needs to act next); older ones stay a note. + let newest_shot = r.history.iter().rposition(|m| m.role == Role::Tool && !m.images.is_empty()); + for (i, m) in r.history.iter_mut().enumerate().filter(|(_, m)| !m.images.is_empty()) { if m.role == Role::Tool { - m.images.clear(); - m.text = if m.text.is_empty() { BLIND_SCREENSHOT_NOTE.to_string() } else { format!("{}\n{BLIND_SCREENSHOT_NOTE}", m.text) }; + let pictures = std::mem::take(&mut m.images); + let seen = if Some(i) == newest_shot { + if !told && !notes().lock().unwrap().contains_key(&picture_key(&pictures[0])) { + send(StreamEvent::Retrying("這個模型看不到圖片,正在請另一個看得懂圖的模型看這張截圖…".into())); + told = true; + } + describe(&pictures[0], SCREEN_ASK, req, cancel) + } else { + None + }; + let add = match seen { + Some((d, model)) => format!("(截圖已拍好;你看不到圖片,以下是「{model}」看過這張截圖後的描述)\n{d}"), + None => BLIND_SCREENSHOT_NOTE.to_string(), + }; + m.text = if m.text.is_empty() { add } else { format!("{}\n{add}", m.text) }; continue; } let pictures = std::mem::take(&mut m.images);