diff --git a/mobile/index.html b/mobile/index.html index f5027ad..9c16e19 100644 --- a/mobile/index.html +++ b/mobile/index.html @@ -91,7 +91,8 @@ header .sub{font-size:12px;color:var(--text3);font-weight:400} .ic{vertical-align:-2px;flex:none} .tool.waiting .st{color:var(--accent)}.tool.failed .st{color:var(--danger)} .tool .d{font:12px var(--mono);color:var(--text3);white-space:nowrap;overflow:hidden;text-overflow:ellipsis;min-width:0} -.att{display:inline-flex;gap:6px;align-items:center;font-size:12.5px;background:var(--raise);border-radius:8px;padding:3px 8px;margin:3px 0 0} +.att{display:inline-flex;gap:6px;align-items:center;font-size:12.5px;background:var(--raise);border-radius:8px;padding:3px 8px;margin:3px 0 0;color:inherit;text-decoration:none;cursor:pointer} +.attimg{margin:6px 0 0;max-width:100%}.attimg img{display:block;max-width:100%;max-height:70vh;border-radius:12px;border:1px solid var(--line);cursor:zoom-in}.attimg .wait{font-size:12.5px;color:var(--text3)} .card{border:1px solid var(--line);border-radius:14px;padding:12px 14px;margin:6px 0;background:var(--surface);max-width:420px} .card .t{font-weight:600}.card .s{color:var(--text3);font-size:13px;margin-bottom:6px} .card .r{display:flex;justify-content:space-between;gap:10px;font-size:13.5px;padding:3px 0;border-top:1px solid var(--line)} @@ -541,7 +542,7 @@ function renderMsgs() { const busy = (S.bots.find((b) => b.id === S.open) || {}).busy; const html = S.msgs.map((m, i) => { if (m.role === "user") { - const atts = (m.attachments || []).map((a) => `
${ic(a.image ? "image" : "file", 14)} ${esc(a.name)}
`).join(""); + const atts = (m.attachments || []).map(attHtml).join(""); if (m.summary) return ""; return `
${m.wake ? `
${ic("alarm-clock", 13)} 例行任務
` : ""}${m.text ? `
${esc(m.text)}
` : ""}${atts}
`; } @@ -551,12 +552,14 @@ function renderMsgs() { const body = m.text ? `
${md(m.text)}
` : (last && busy && !tools ? `
` : ""); const who = m.author ? `
${esc(m.author)}
` : ""; const via = m.text && m.via && !busy ? `
${esc(m.via)}${(m.reactions || []).length ? " · " + m.reactions.join(" ") : ""}
` : ""; - return `
${who}${body}${tools}${last ? "" : ""}${via}
`; + const atts = (m.attachments || []).map(attHtml).join(""); + return `
${who}${body}${tools}${atts}${last ? "" : ""}${via}
`; }).join(""); $("msgs").innerHTML = html || `
還沒有訊息，說聲嗨吧
`; $("b-send").innerHTML = busy && !$("input").value.trim() && !S.pending.length ? `` : ``; + loadFiles(); } function card(c) { if (!c) return ""; @@ -597,6 +600,36 @@ async function openPage(id, rev) { } $("pver").onchange = () => openPage(S.page, +$("pver").value || 0); $("b-back3").onclick = () => { show(S.pageBack && S.pageBack !== "page" ? S.pageBack : "list"); if (S.view === "chat") loadChat(); }; +// Files in a chat: pictures shown in place (a screen the bot showed, a photo), other files open on a tap. +// Fetched with the login (an can't send it), kept for the next redraw. +const FILES = {}; +function attHtml(a) { + if (!a.path) return `
${ic(a.image ? "image" : "file", 14)} ${esc(a.name)}
`; + if (a.image) return `
${FILES[a.path] ? `${esc(a.name)}` : `
${ic("image", 14)} ${esc(a.name)}
`}
`; + return `${ic("file", 14)} ${esc(a.name)}`; +} +async function fileUrl(path) { + if (!FILES[path]) { + const r = await api(base() + "api/file?path=" + encodeURIComponent(path), {raw: true}); + if (!r.ok) throw new Error("找不到這個檔案"); + FILES[path] = URL.createObjectURL(await r.blob()); + } + return FILES[path]; +} +async function loadFiles() { + for (const el of document.querySelectorAll(".attimg[data-file]")) { + if (el.querySelector("img")) continue; + try { const u = await fileUrl(el.dataset.file); el.innerHTML = ``; } catch (_) { el.innerHTML = `
這張圖讀不到
`; } + } +} +document.addEventListener("click", async (e) => { + const img = e.target.closest && e.target.closest(".attimg img"); + if (img) { window.open(img.src, "_blank"); return; } + const f = e.target.closest && e.target.closest("a.att[data-file]"); + if (!f) return; + e.preventDefault(); + try { const u = await fileUrl(f.dataset.file); const a = document.createElement("a"); a.href = u; a.download = f.dataset.name || "檔案"; document.body.appendChild(a); a.click(); a.remove(); } catch (err) { toast(err.message); } +}); const dark = () => matchMedia("(prefers-color-scheme: dark)").matches; async function loadCharts() { for (const el of document.querySelectorAll(".chart[data-src]")) { diff --git a/src/agent.rs b/src/agent.rs index 668aa1b..7086500 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -2,6 +2,7 @@ // rules say so), give it the results, and repeat until it answers in plain words. // Long chats are folded into a summary before they outgrow the model's context. +use base64::Engine; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::{channel, Sender}; use std::sync::{Arc, Mutex}; @@ -279,7 +280,7 @@ fn element_named(history: &[Message], r: &str) -> Option { /// counted: doing those again and again is how a long page or a list is read. fn step_key(name: &str, args: &str) -> Option { if !(computer::is_computer_tool(name) || web::is_web_tool(name)) - || matches!(name, "computer_use_screenshot" | "browser_read" | "browser_screenshot" | "browser_wait" | "computer_use_scroll" | "browser_scroll" | "computer_use_press_key" | "browser_press" | "browser_back") + || matches!(name, "computer_use_screenshot" | "computer_use_show" | "browser_read" | "browser_screenshot" | "browser_wait" | "computer_use_scroll" | "browser_scroll" | "computer_use_press_key" | "browser_press" | "browser_back") { return None; } @@ -313,6 +314,37 @@ fn refresh_own(job: &mut Job, bot_id: u64) -> Option { } } +/// How many times in a row this very call (same tool, same arguments) already came back with the +/// very same answer among the latest steps. Waiting tools are left out: there the same answer is +/// the point. +fn same_answers(history: &[Message], name: &str, args: &str) -> usize { + if matches!(name, "read_terminal" | "browser_wait" | "write_terminal") { + return 0; + } + let want = tools::args_of(args); + let calls: std::collections::HashMap<&str, (&str, &str)> = history.iter().rev().take(80).flat_map(|m| m.tool_calls.iter()).map(|c| (c.id.as_str(), (c.name.as_str(), c.args.as_str()))).collect(); + // A warning put in front of an answer doesn't make it a different answer. + let bare = |t: &str| -> String { t.lines().skip_while(|l| l.starts_with("⚠️")).collect::>().join("\n") }; + let mut first: Option = None; + let mut n = 0; + for m in history.iter().rev().filter(|m| m.role == Role::Tool).take(12) { + let Some((cn, ca)) = calls.get(m.tool_call_id.as_str()) else { continue }; + if *cn != name || tools::args_of(ca) != want { + continue; + } + let text = bare(&m.text); + match &first { + None => { + first = Some(text); + n = 1; + } + Some(f) if *f == text => n += 1, + _ => break, + } + } + n +} + /// "Always allow" for this turn: the grant is kept (once). fn keep_grant(grants: &mut Vec, g: String) { if !g.is_empty() && !grants.contains(&g) { @@ -586,6 +618,22 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< repeat_note = "⚠️ 這一步你已經做了 3 次。如果畫面還不是你要的，下一步一定要換方法，不要再做同一步。\n".into(); } } + // The same look again and again with the same answer (a blank tab read eleven + // times in a row): the third time with a warning, after that not at all. + if repeat_note.is_empty() { + let same = same_answers(&history, &call.name, &call.args); + if same >= 3 { + let text = format!("沒有執行：同一個工具、同樣的參數你已經做了 {same} 次，每次結果都一模一樣，再做也不會不同。換一個做法（例如用 browser_open 打開要看的網址），或直接告訴使用者你看到了什麼。"); + send(StreamEvent::ToolStatus(call.id.clone(), ToolStatus::Failed)); + let result = Message::tool_result(&call.id, &call.name, &text); + send(StreamEvent::ToolResult(result.clone())); + history.push(result); + continue; + } + if same == 2 { + repeat_note = "⚠️ 同一步你已經做了 2 次，結果都一模一樣。如果這次還一樣，就換一個做法。\n".into(); + } + } // Every step ends the same way: its status, then its result (with the notes) and // any picture, shown and kept. let finish = |history: &mut Vec, output: &str, status: ToolStatus, image: Option| { @@ -721,13 +769,28 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< }, Ok(tools::Target { place: tools::Place::Sealed { root }, .. }) => Ok(tools::resolve(&root, &path)), Ok(tools::Target { place: tools::Place::Local { workdir }, .. }) => Ok(tools::resolve(&workdir, &path)), + // Another of the person's computers: it sends the file over, then it is + // put in the chat like any other. + Ok(tools::Target { place: tools::Place::Device { host, dir }, .. }) => { + let args = serde_json::json!({"path": path, "workdir": dir}).to_string(); + crate::remote::run_on_device(&host, crate::remote::FETCH_FILE, &args, &job.via_name).and_then(|v| { + let bytes = v["data"].as_str().and_then(|d| base64::engine::general_purpose::STANDARD.decode(d).ok()).ok_or_else(|| v["text"].as_str().unwrap_or("那台電腦沒有把檔案傳過來").to_string())?; + let dst = std::env::temp_dir().join(format!("anybot-fetch-{}", std::process::id())).join(v["name"].as_str().unwrap_or("檔案")); + let _ = std::fs::create_dir_all(dst.parent().unwrap_or(std::path::Path::new("."))); + std::fs::write(&dst, &bytes).map_err(|e| e.to_string())?; + Ok(dst) + }) + } Ok(_) => Err(format!("這台電腦上的檔案不能直接傳，請先複製到「{}」。", job.roles.own)), Err(e) => Err(e), }; // Secrets (SSH keys, the app's own data folder, .env…) never ride into the chat, // same rule as read_file/edit_file. let local = local.and_then(|p| if tools::is_ignored(&p) || tools::is_ignored(std::path::Path::new(&path)) { Err("這個檔案是機密的，不能傳到聊天。".to_string()) } else { Ok(p) }); - let (output, ok) = match local.and_then(|p| crate::attach::store(bot_id, &p)) { + let stored = local.and_then(|p| crate::attach::store(bot_id, &p)); + // (A file fetched from another computer was only parked here on its way.) + let _ = std::fs::remove_dir_all(std::env::temp_dir().join(format!("anybot-fetch-{}", std::process::id()))); + let (output, ok) = match stored { Ok(a) => { let out = format!("已把「{}」傳到聊天裡（{}），使用者點一下就能預覽。", a.name, crate::attach::human(a.bytes)); send(StreamEvent::Attach(a)); @@ -1033,15 +1096,16 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< (Ok(_), Ok(())) if is_screen && call.name == "computer_use_check_permissions" => Decision::Allow, // 裝置連結: another of the person's computers; seeing and operating it ask once each. (Ok(t), Ok(())) if is_screen && on_device.is_some() => { - let grant = if call.name == "computer_use_screenshot" { format!("device-see:{}", t.name) } else { format!("device-control:{}", t.name) }; + let looks = matches!(call.name.as_str(), "computer_use_screenshot" | "computer_use_show"); + let grant = if looks { format!("device-see:{}", t.name) } else { format!("device-control:{}", t.name) }; if grants.contains(&grant) { Decision::Allow } else { - Decision::Ask { reason: format!("{}「{}」的螢幕：{what}", if call.name == "computer_use_screenshot" { "看" } else { "操作" }, t.name), grant } + Decision::Ask { reason: format!("{}「{}」的螢幕：{what}", if looks { "看" } else { "操作" }, t.name), grant } } } (Ok(_), Ok(())) if is_screen => { - if call.name == "computer_use_screenshot" { + if matches!(call.name.as_str(), "computer_use_screenshot" | "computer_use_show") { if grants.iter().any(|g| g == "computer:see") { Decision::Allow } else { @@ -1169,6 +1233,15 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< (Ok(()), Err(e)) => (e.clone(), ToolStatus::Failed), (Err(x), _) => x, }; + // 給使用者看畫面: the picture goes into the chat for the person (the bot still sees it). + if call.name == "computer_use_show" { + if let Some(bytes) = image.as_deref().and_then(|b| base64::engine::general_purpose::STANDARD.decode(b).ok()) { + let name = format!("{} 的畫面 {}.jpg", target.as_ref().map(|t| t.name.as_str()).unwrap_or("電腦"), chrono::Local::now().format("%H%M%S")); + if let Ok(a) = crate::attach::store_bytes(bot_id, &name, &bytes) { + send(StreamEvent::Attach(a)); + } + } + } finish(&mut history, &output, status, image); if cancel.load(Ordering::Relaxed) { return; @@ -1227,6 +1300,25 @@ mod tests { assert!(!mentions("在 macOS 的登入機制下，幫 Bot 的電腦截圖", &mac, &roles)); } + #[test] + fn the_same_answer_again_and_again_is_counted() { + use crate::model::ToolCall; + let call = |id: &str, args: &str| Message { role: Role::Assistant, tool_calls: vec![ToolCall { id: id.into(), name: "browser_read".into(), args: args.into(), status: ToolStatus::Done }], ..Default::default() }; + let mut h = vec![Message::user("看得到我的 Threads 嗎？")]; + for i in 0..3 { + h.push(call(&format!("c{i}"), r#"{"computer":"Bot 的電腦"}"#)); + let warn = if i == 2 { "⚠️ 同一步你已經做了 2 次，結果都一模一樣。\n" } else { "" }; + h.push(Message::tool_result(&format!("c{i}"), "browser_read", &format!("{warn}網址：about:blank"))); + } + assert_eq!(same_answers(&h, "browser_read", r#"{ "computer": "Bot 的電腦" }"#), 3); + // Another page in between: counting starts again. + h.push(call("c9", r#"{"computer":"Bot 的電腦"}"#)); + h.push(Message::tool_result("c9", "browser_read", "網址：https://www.threads.com/")); + assert_eq!(same_answers(&h, "browser_read", r#"{"computer":"Bot 的電腦"}"#), 1); + // Waiting on a terminal is allowed to look the same. + assert_eq!(same_answers(&h, "read_terminal", "{}"), 0); + } + #[test] fn long_turns_check_in_instead_of_stopping() { let mut h = vec![Message::user("修好它"), Message::tool_result("t1", "shell", "ok")]; diff --git a/src/app.rs b/src/app.rs index b955a6f..7ee57fd 100644 --- a/src/app.rs +++ b/src/app.rs @@ -4024,7 +4024,7 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu 做完 App 操作後，用一句話告訴使用者結果（例如排程的時間）。\n\n\ 你也可以上網和用使用者的瀏覽器：\n\ - 公開網頁：web_fetch。\n\ - - 要登入才看得到的（Threads、Instagram、Facebook、Gmail、銀行…）：用 browser_open 在 AnyBot 自己的瀏覽器視窗打開：它會等頁面載入，直接回傳頁面內容和可以點的編號（不用再 browser_read；之後頁面變了才用 browser_read 重讀）。例如看 Threads 帳號：browser_open(\"https://www.threads.net\")。\n\ + - 要登入才看得到的（Threads、Instagram、Facebook、Gmail、銀行…）：用 browser_open 在 AnyBot 自己的瀏覽器視窗打開：它會等頁面載入，直接回傳頁面內容和可以點的編號（不用再 browser_read；之後頁面變了才用 browser_read 重讀）。例如看 Threads 帳號：browser_open(\"https://www.threads.net\")。要知道某個網站有沒有登入，一定要打開那個網站看，不要看目前的分頁（常常是空白頁）就下結論。\n\ - 網頁上的事（任何電腦、任何網站）一律用 browser_* 工具：先 browser_read（或 browser_screenshot）拿到每個連結、按鈕、欄位的編號和網址，再用 browser_click／browser_type／browser_select／browser_press 照編號操作；要去的頁面有網址就直接 browser_open。網頁上絕對不要看截圖猜座標點。表格用 browser_table 讀，找字用 browser_find，要等載入用 browser_wait，匯出報表用 browser_download。不要猜網址；找不到想要的功能（例如排序、篩選），就看欄位標題、篩選器、下拉選單或分頁標籤，還是沒有就照頁面現有的資料自己整理，並說明網站沒有這個功能。 - 絕對不要直接呼叫網站的內部 API、下載網站的程式碼來研究，或用程式大量抓資料：網站會把帳號當成機器人（例如跳出拼圖驗證、封鎖帳號）。也絕對不要用指令去讀瀏覽器的 Cookie、瀏覽紀錄、密碼或設定檔資料庫。 - 需要使用者親手做的步驟（讀到登入頁、輸入密碼、驗證碼、拼圖、簡訊或 email 驗證、確認付款）：先在那台電腦上把那一頁打開、點到要輸入的地方，再呼叫 hand_over_computer，instruction 寫清楚要他做什麼。聊天裡會出現那台電腦的即時畫面，他點「接管」做完後按「我完成了」，你再確認、接著做（登入一次之後就會記住）。不要只用文字叫他自己去找視窗，絕對不要替他打密碼，也不要叫他改瀏覽器的設定或選單、或說要從 Chrome 複製登入或 Cookie（都不需要，也沒有這個功能）。\n\ @@ -4090,6 +4090,8 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu s.push_str(b.instructions.trim()); } s.push_str("\n\n使用者要你追蹤、整理、比較、研究一件之後還會回來看的事（例如「幫我追蹤台灣 FSD 狀況」），就用 write_page 做成一頁：先寫結論和現況，再寫細節和來源（附日期），適合時用表格、```stats、```steps、圖表。回答時簡短說重點，並用 [頁面標題](page:ID) 連到它。要定期追蹤時，再用 create_routine 設例行任務，每次執行：read_page、查新消息，有實質進展才用 write_page 更新（note 寫改了什麼）並在聊天裡說；沒有新消息就簡短回答「沒有新消息」，不要更新頁面。"); + s.push_str("\n\n幫使用者寫貼文、留言或文案，要「用我的語氣」時：先打開他的帳號，讀最近幾則他自己寫的貼文，照那樣的長度、用詞、標點和開頭結尾寫，不要只看自介。給草稿就好，沒有他說「發」不要自己發出去。"); + s.push_str("\n\n使用者要看某台電腦的畫面（截圖給我看、讓我看看）時，用 computer_use_show：畫面會直接出現在聊天裡。"); s.push_str("\n\n做好的檔案（試算表、文件、Markdown、網頁、圖片）用 send_file 傳進聊天，使用者點一下就能預覽；不要只說檔案放在哪裡。適合的時候可以用 react 對使用者的訊息按個表情（例如謝謝你時按 ❤️），不用每次都按。"); s.push_str("\n\n"); s.push_str(&crate::i18n::reply_instruction(&st.language)); diff --git a/src/attach.rs b/src/attach.rs index 94d6d10..c7a5300 100644 --- a/src/attach.rs +++ b/src/attach.rs @@ -52,6 +52,18 @@ pub fn store(bot_id: u64, src: &Path) -> Result { Ok(Attachment { image: is_image_name(&name), name, path: dst.display().to_string(), bytes: meta.len(), on_box: String::new() }) } +/// A picture or file the bot hands over (a screen it shows, a file fetched from another computer). +pub fn store_bytes(bot_id: u64, name: &str, bytes: &[u8]) -> Result { + if bytes.len() as u64 > MAX_FILE_BYTES { + return Err(format!("「{name}」太大了（最多 50 MB）")); + } + let dir = dir_for(bot_id); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + let dst = dir.join(format!("{}-{name}", chrono::Local::now().format("%Y%m%d%H%M%S%3f"))); + std::fs::write(&dst, bytes).map_err(|e| format!("存不下來：{e}"))?; + Ok(Attachment { image: is_image_name(name), name: name.to_string(), path: dst.display().to_string(), bytes: bytes.len() as u64, on_box: String::new() }) +} + /// A picture on the clipboard (e.g. a screenshot copied with ⌃⇧⌘4) becomes an attachment. pub fn paste_image(bot_id: u64) -> Option { let dir = dir_for(bot_id); diff --git a/src/botbrowser.rs b/src/botbrowser.rs index 446c495..68c9861 100644 --- a/src/botbrowser.rs +++ b/src/botbrowser.rs @@ -407,6 +407,11 @@ pub fn read(max_chars: usize, profile: Option<&str>) -> Result { if login_hint { out.push_str("\n\n（這頁看起來要登入。請使用者在 AnyBot 瀏覽器視窗登入一次，之後就會記住。）"); } + // A blank tab says nothing about a site (a bot read one eleven times and called the person + // logged out of Threads without ever opening it). + if url.is_empty() || url == "about:blank" || url.starts_with("chrome://newtab") { + out.push_str("\n\n（這個分頁是空白的，不代表任何網站的狀態。要看哪個網站（例如有沒有登入），就用 browser_open 打開它。）"); + } Ok(out) } diff --git a/src/cloudbox.rs b/src/cloudbox.rs index 59d9c94..99445f6 100644 --- a/src/cloudbox.rs +++ b/src/cloudbox.rs @@ -1793,6 +1793,7 @@ pub fn computer_run(name: &str, v: &Value) -> Result<(String, Option), S match name { "computer_use_check_permissions" => return Ok(("這台電腦不需要另外的權限：可以截圖，也可以操作滑鼠和鍵盤。".into(), None)), "computer_use_screenshot" => return box_look(None, None, false), + "computer_use_show" => return box_show(), _ => {} } if HELD_BY_PERSON.load(Ordering::SeqCst) || crate::vnc::person_active() { @@ -1809,6 +1810,17 @@ pub fn computer_run(name: &str, v: &Value) -> Result<(String, Option), S box_look(Some(&script), Some(note), crate::computer::is_pointer_step(name)) } +/// The box's screen as the person would see it (no numbers), at the bot's picture size. +fn box_show() -> Result<(String, Option), String> { + let out = dk_with_input(&["exec", "-i", "-u", "bot", "-e", "DISPLAY=:1", container(), "sh", "-c", "import -window root -quality 90 jpeg:- 2>/dev/null"], Some(b""), 30_000).map_err(|e| format!("拍不到畫面：{e}"))?; + let img = image::load_from_memory(&out).map_err(|e| format!("截圖讀不出來：{e}"))?.to_rgb8(); + let (w, h) = img.dimensions(); + let (fw, fh, resize) = crate::computer::fit(w, h); + let pic = if resize { image::imageops::resize(&img, fw, fh, image::imageops::FilterType::Triangle) } else { img }; + let jpeg = crate::computer::encode_jpeg(&pic).ok_or("截圖太大了，沒辦法送出")?; + Ok((format!("{}（{fw}×{fh}）", crate::computer::SHOWN_NOTE), Some(base64::engine::general_purpose::STANDARD.encode(&jpeg)))) +} + const BATCH_MARK: &str = "/tmp/anybot-batch.done"; /// Do a planned sequence of steps on the box in one round trip: the steps' scripts chained with @@ -2098,6 +2110,19 @@ mod tests { assert!(again.contains("畫面和上一步完全一樣"), "{again}"); } + /// computer_use_show: a plain picture of the box's screen (no numbers) comes back for the chat. + #[test] + #[ignore] + fn box_show_is_a_plain_picture() { + ensure_running().expect("box starts"); + let (text, img) = computer_run("computer_use_show", &serde_json::json!({})).unwrap(); + assert!(text.contains("放進聊天"), "{text}"); + let jpeg = base64::engine::general_purpose::STANDARD.decode(img.expect("a picture")).unwrap(); + let pic = image::load_from_memory(&jpeg).unwrap(); + println!("{text} — {}×{}, {} KB", pic.width(), pic.height(), jpeg.len() / 1024); + assert_eq!((pic.width(), pic.height()), (1024, 640)); + } + /// Numbers only on what can be seen: links scrolled up under the browser's toolbar get none (or /// their visible part's centre), and nothing is numbered under a window that covers the page /// (xterm has no accessibility; clicking a number there used to type into the terminal). diff --git a/src/computer.rs b/src/computer.rs index 41869c9..a9b1297 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -12,7 +12,7 @@ use base64::Engine; use serde_json::{json, Value}; use std::sync::Mutex; -pub const NAMES: [&str; 10] = ["computer_use_screenshot", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app", "computer_use_batch"]; +pub const NAMES: [&str; 11] = ["computer_use_screenshot", "computer_use_show", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app", "computer_use_batch"]; /// Wait after an action before taking the picture that comes back with it (menus open, pages /// start to change; estimated). The box waits for the screen to stop changing instead, up to /// about this long. @@ -220,6 +220,9 @@ pub fn is_pointer_step(name: &str) -> bool { matches!(name, "computer_use_click" | "computer_use_scroll" | "computer_use_drag") } +/// What the bot is told after computer_use_show (the picture itself goes into the chat). +pub const SHOWN_NOTE: &str = "已把這台電腦現在的畫面（使用者看到的樣子，沒有編號）放進聊天，使用者看得到這張圖；你也看得到，可以用一兩句話說明畫面上有什麼"; + pub fn is_computer_tool(name: &str) -> bool { NAMES.contains(&name) } @@ -227,6 +230,7 @@ pub fn is_computer_tool(name: &str) -> bool { pub fn label(name: &str) -> &'static str { match name { "computer_use_screenshot" => "看你的螢幕", + "computer_use_show" => "把畫面傳給你看", "computer_use_click" => "點一下", "computer_use_typing" => "打字", "computer_use_press_key" => "按鍵", @@ -251,6 +255,7 @@ pub fn describe(name: &str, args: &str) -> String { }; match name { "computer_use_screenshot" => "螢幕截圖".into(), + "computer_use_show" => "畫面傳進聊天".into(), "computer_use_click" => { let b = v["button"].as_str().unwrap_or("left"); let n = v["count"].as_u64().unwrap_or(1); @@ -281,6 +286,8 @@ pub fn specs_for(computers: &[String], default: &str, chosen: bool) -> Vec Result<(String, Option), String> } match name { "computer_use_screenshot" => return look(None, false), + "computer_use_show" => { + let (w, h, jpeg) = plain_picture()?; + return Ok((format!("{SHOWN_NOTE}（{w}×{h}）"), Some(base64::engine::general_purpose::STANDARD.encode(&jpeg)))); + } "computer_use_check_permissions" => { #[cfg(target_os = "linux")] return Ok((crate::desk::describe().into(), None)); diff --git a/src/desktop.rs b/src/desktop.rs index ee32104..1dc6c8b 100644 --- a/src/desktop.rs +++ b/src/desktop.rs @@ -120,6 +120,19 @@ fn shorten(v: &str) -> String { format!("{head} …（中間省略 {} 字）… {tail}", n - VALUE_HEAD - VALUE_TAIL) } +/// The first words inside a node (a row's label). +fn words_inside(n: &Value) -> String { + n["children"].as_array().into_iter().flatten().find_map(|c| { + let w = c["name"].as_str().or_else(|| c["value"].as_str()).unwrap_or("").trim().to_string(); + if w.is_empty() { Some(words_inside(c)).filter(|w| !w.is_empty()) } else { Some(w) } + }).unwrap_or_default() +} + +/// Something inside this node is a control with a name. +fn named_inside(n: &Value) -> bool { + n["children"].as_array().into_iter().flatten().any(|c| (c["ref_id"].is_string() && active(c["role"].as_str().unwrap_or("")) && !c["name"].as_str().unwrap_or("").trim().is_empty()) || named_inside(c)) +} + /// The controls and text of a snapshot, with where each is on the screen (nodes the app gave /// no place are left out: they can't be shown or clicked). pub fn elements_of(tree: &Value, app: &str) -> Vec { @@ -128,14 +141,25 @@ pub fn elements_of(tree: &Value, app: &str) -> Vec { return; } let role = n["role"].as_str().unwrap_or(""); - let name = n["name"].as_str().unwrap_or("").trim().to_string(); + let mut name = n["name"].as_str().unwrap_or("").trim().to_string(); + // A nameless row, cell or list item goes by the words inside it (an icon-only button + // keeps its number without a name: it can still be clicked). + let row_like = matches!(role, "row" | "cell" | "treeitem" | "outlinerow" | "listitem"); + // …unless a named control inside it is what gets clicked (System Settings' sidebar). + let wraps_named = name.is_empty() && named_inside(n); + if name.is_empty() && row_like && !wraps_named { + name = words_inside(n); + } let value = n["value"].as_str().map(str::to_string).or_else(|| n["value"].as_f64().map(|f| f.to_string())).unwrap_or_default(); let b = &n["bounds"]; if let (Some(x), Some(y), Some(w), Some(h)) = (b["x"].as_f64(), b["y"].as_f64(), b["width"].as_f64(), b["height"].as_f64()) { let is_active = n["ref_id"].is_string() && active(role); let has = |k: &str| n["states"].as_array().map(|a| a.iter().any(|x| x == k)).unwrap_or(false) || n[k] == true; - // Hidden or scrolled out of view: nothing a click could reach. - let unseen = has("hidden") || has("offscreen"); + // Hidden or scrolled out of view: nothing a click could reach. A nameless row whose + // own button has the name (System Settings' sidebar) needs no number of its own: a + // list of 「項目」 told the bot nothing. + let empty = name.is_empty() && value.trim().is_empty(); + let unseen = has("hidden") || has("offscreen") || (is_active && wraps_named) || (row_like && empty); if !unseen && w >= 1.0 && h >= 1.0 && x + w > 0.0 && y + h > 0.0 && (is_active || !name.is_empty() || !value.trim().is_empty()) && role != "window" && role != "group" { let mut flags = vec![]; for (k, word) in [("checked", "已勾"), ("selected", "已選"), ("disabled", "停用"), ("focused", "游標在這")] { @@ -188,6 +212,15 @@ mod tests { ]}); let names: Vec = elements_of(&tree, "App").into_iter().map(|e| e.name).collect(); assert_eq!(names, vec!["Close".to_string()]); + // A nameless row around a named button: only the button is numbered. + let rows = serde_json::json!({"role": "window", "children": [ + {"role": "row", "name": "", "ref_id": "r", "bounds": {"x": 0.0, "y": 100.0, "width": 200.0, "height": 30.0}, "children": [ + {"role": "button", "name": "Wi-Fi", "ref_id": "w", "bounds": {"x": 10.0, "y": 105.0, "width": 80.0, "height": 20.0}}]}, + {"role": "row", "name": "", "ref_id": "r2", "bounds": {"x": 0.0, "y": 140.0, "width": 200.0, "height": 30.0}} + ]}); + let kept: Vec<(String, String)> = elements_of(&rows, "App").into_iter().map(|e| (e.kind, e.name)).collect(); + assert_eq!(kept[0].1, "Wi-Fi"); + assert_eq!(kept.len(), 1, "an empty spacer row gets no number: {kept:?}"); } #[test] diff --git a/src/remote.rs b/src/remote.rs index 5878c5f..4a8aa30 100644 --- a/src/remote.rs +++ b/src/remote.rs @@ -298,7 +298,7 @@ pub fn message_json(m: &crate::model::Message, names: &HashMap, pag "wake": m.wake, "author": if m.author == 0 { String::new() } else { names.get(&m.author).cloned().unwrap_or_default() }, "reactions": m.reactions, - "attachments": m.attachments.iter().map(|a| json!({"name": a.name, "image": a.image})).collect::>(), + "attachments": m.attachments.iter().map(|a| json!({"name": a.name, "image": a.image, "path": a.path})).collect::>(), "tools": calls, })) } @@ -466,6 +466,12 @@ pub fn handle(req: Req, via_relay: bool) -> Resp { Err(e) => Resp::error(503, &e), }, // A file shown in a chat (only what AnyBot keeps as attachments). + // A file in a chat (a picture the bot showed, a file it sent, one the person attached), + // for the phone to show or download. Only from the chats' own file folder. + ("GET", ["api", "file"]) => match chat_file(&req.query.get("path").cloned().unwrap_or_default()) { + Some((f, b)) => Resp::bytes(content_type(&f), b), + None => Resp::error(404, "找不到這個檔案"), + }, ("GET", ["api", "sync", "file"]) => { let want = std::path::PathBuf::from(req.query.get("path").cloned().unwrap_or_default()); let root = crate::model::State::path().parent().map(|p| p.join("attachments")).and_then(|p| p.canonicalize().ok()); @@ -521,11 +527,34 @@ pub fn role_allows(role: &str, method: &str, path: &[&str]) -> bool { // 大腦: the owner's other AnyBots showing this one's bots. "mirror" => sync, "owner" | "admin" => !device && !sync, - "member" => matches!((method, path), ("GET", ["api", "state"]) | ("GET", ["api", "bots", _]) | ("GET", ["api", "pages", ..]) | ("POST", ["api", "bots", _, "send" | "read" | "upload"])), + "member" => matches!((method, path), ("GET", ["api", "state"]) | ("GET", ["api", "bots", _]) | ("GET", ["api", "pages", ..]) | ("GET", ["api", "file"]) | ("POST", ["api", "bots", _, "send" | "read" | "upload"])), _ => false, } } +/// A chat file by its path, only when it really is inside the chats' file folder. +fn chat_file(want: &str) -> Option<(std::path::PathBuf, Vec)> { + let root = crate::model::State::path().parent()?.join("attachments").canonicalize().ok()?; + let f = std::path::PathBuf::from(want).canonicalize().ok()?; + if !f.starts_with(&root) || !f.is_file() { + return None; + } + let b = std::fs::read(&f).ok()?; + Some((f, b)) +} + +fn content_type(f: &std::path::Path) -> &'static str { + match f.extension().and_then(|e| e.to_str()).map(|e| e.to_ascii_lowercase()).as_deref() { + Some("jpg" | "jpeg") => "image/jpeg", + Some("png") => "image/png", + Some("gif") => "image/gif", + Some("webp") => "image/webp", + Some("pdf") => "application/pdf", + Some("txt" | "md" | "csv") => "text/plain; charset=utf-8", + _ => "application/octet-stream", + } +} + fn login(req: &Req) -> Resp { // One counter per source address, cleaned as it is touched. let ip = req.peer.split(':').next().unwrap_or(&req.peer).to_string(); @@ -659,6 +688,10 @@ fn device_run(v: &Value, s: &Settings, via_relay: bool, from: &str) -> Resp { let (text, ok) = crate::macapps::run(tool, args); return Resp::json(&json!({"text": text, "ok": ok})); } + // send_file from this computer: the file itself goes to the brain, which puts it in the chat. + if tool == FETCH_FILE { + return fetch_file_here(args); + } if matches!(tool, "shell" | "read_file" | "edit_file" | "list_dir" | "search" | "read_terminal" | "write_terminal") { // The brain passes the bot's work folder inside the tool's args (empty = this home). let dir = crate::tools::args_of(args).get("workdir").and_then(Value::as_str).unwrap_or("").trim().to_string(); @@ -674,6 +707,36 @@ fn device_run(v: &Value, s: &Settings, via_relay: bool, from: &str) -> Resp { Resp::error(400, "這個工具不能遠端使用") } +/// A file on this computer, for the brain's send_file (the brain has no other way to it). +pub const FETCH_FILE: &str = "fetch_file"; +/// Largest file sent that way: the relay carries 30 MB, and base64 adds a third. +const FETCH_FILE_MAX: u64 = 20 * 1024 * 1024; + +fn fetch_file_here(args: &str) -> Resp { + let v = crate::tools::args_of(args); + let path = v["path"].as_str().unwrap_or("").trim().to_string(); + let dir = v["workdir"].as_str().unwrap_or("").trim().to_string(); + let base = crate::tools::expand_home(if dir.is_empty() { "~" } else { &dir }); + let full = crate::tools::resolve(&base, &path); + // Same rule as reading a file: keys, the app's own data and other secrets never leave. + if crate::tools::is_ignored(&full) || crate::tools::is_ignored(std::path::Path::new(&path)) { + return Resp::json(&json!({"text": "這個檔案是機密的，不能傳到聊天。", "ok": false})); + } + match std::fs::metadata(&full) { + Ok(m) if m.is_dir() => Resp::json(&json!({"text": format!("「{path}」是資料夾，請先壓縮成一個檔案。"), "ok": false})), + Ok(m) if m.len() > FETCH_FILE_MAX => Resp::json(&json!({"text": format!("「{path}」太大了（最多 20 MB）。"), "ok": false})), + Ok(_) => match std::fs::read(&full) { + Ok(b) => { + use base64::Engine; + let name = full.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "檔案".into()); + Resp::json(&json!({"ok": true, "text": "", "name": name, "data": base64::engine::general_purpose::STANDARD.encode(&b)})) + } + Err(e) => Resp::json(&json!({"text": format!("讀不到「{path}」：{e}"), "ok": false})), + }, + Err(_) => Resp::json(&json!({"text": format!("這台電腦上找不到「{path}」。"), "ok": false})), + } +} + // ---------- a model on the person's network, for a brain on a server ---------- // A brain in the cloud can't reach a model at a private address (a Qwen server at work, Ollama at // home). Its hands can: the brain hands them the request and gets the whole answer back.