看圖小幫手 also looks at the newest screenshot

A model that can't see pictures got only 「截圖已拍好，但看不到」 for computer_use_screenshot.
Now the newest screenshot is described by a model of the person's that can (windows, text,
buttons and roughly where they are); older screenshots stay a short note.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa

src/providers.rs

@@ -372,7 +372,7 @@ pub fn run_step(req: &ChatRequest, cancel: &AtomicBool, send: &dyn Fn(StreamEven
     let mut blind_copy: Option<ChatRequest> = None;
     loop {
         // A model that can't see pictures gets the request without them: the person's pictures
-        // described by a model that can (看圖小幫手), screenshots as a short note.
+        // and the newest screenshot described by a model that can (看圖小幫手).
         if !sees_images(req) && blind_copy.is_none() {
             blind_copy = Some(if DESCRIBING.with(|d| d.get()) { without_images(req) } else { with_descriptions(req, cancel, send) });
         }
@@ -458,6 +458,8 @@ fn kept_images(history: &[Message]) -> Vec<usize> {
 const OLD_SCREENSHOT_NOTE: &str = "（較早的截圖已省略）";
 /// Said in place of a screenshot when the model can't take pictures.
 const BLIND_SCREENSHOT_NOTE: &str = "（截圖已拍好，但這個模型看不到圖片，所以沒有附上。請改用文字方式確認畫面，例如讀取視窗或網頁的文字。）";
+/// What a helper is asked about a screenshot.
+const SCREEN_ASK: &str = "這是一台電腦螢幕的截圖。請描述：開著哪些視窗或 App、畫面上的文字（照抄重要的字）、按鈕和欄位，以及它們大約的位置（以這張圖的像素座標，左上角是 0,0）。不要猜看不清楚的字。";
 const BLIND_ATTACHMENT_NOTE: &str = "（使用者附了圖片，但這個模型看不到圖片。）";
 
 /// Models (base url + name) that refused a picture: later requests leave pictures out.
@@ -631,10 +633,25 @@ fn describe(picture: &str, ask: &str, not: &ChatRequest, cancel: &AtomicBool) ->
 fn with_descriptions(req: &ChatRequest, cancel: &AtomicBool, send: &dyn Fn(StreamEvent)) -> ChatRequest {
     let mut r = req.clone();
     let mut told = false;
-    for m in r.history.iter_mut().filter(|m| !m.images.is_empty()) {
+    // The newest screenshot is described too (what the bot needs to act next); older ones stay a note.
+    let newest_shot = r.history.iter().rposition(|m| m.role == Role::Tool && !m.images.is_empty());
+    for (i, m) in r.history.iter_mut().enumerate().filter(|(_, m)| !m.images.is_empty()) {
         if m.role == Role::Tool {
-            m.images.clear();
+            let pictures = std::mem::take(&mut m.images);
-            m.text = if m.text.is_empty() { BLIND_SCREENSHOT_NOTE.to_string() } else { format!("{}\n{BLIND_SCREENSHOT_NOTE}", m.text) };
+            let seen = if Some(i) == newest_shot {
+                if !told && !notes().lock().unwrap().contains_key(&picture_key(&pictures[0])) {
+                    send(StreamEvent::Retrying("這個模型看不到圖片，正在請另一個看得懂圖的模型看這張截圖…".into()));
+                    told = true;
+                }
+                describe(&pictures[0], SCREEN_ASK, req, cancel)
+            } else {
+                None
+            };
+            let add = match seen {
+                Some((d, model)) => format!("（截圖已拍好；你看不到圖片，以下是「{model}」看過這張截圖後的描述）\n{d}"),
+                None => BLIND_SCREENSHOT_NOTE.to_string(),
+            };
+            m.text = if m.text.is_empty() { add } else { format!("{}\n{add}", m.text) };
             continue;
         }
         let pictures = std::mem::take(&mut m.images);