From cbada6bdb12d3831b9f4ff587465c135efbbe404 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=80=A7=E9=A9=8A?= Date: Fri, 6 Nov 2026 23:12:53 +0800 Subject: [PATCH] Bots learn from what they did; the box's browser works with the box's own disk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A job of five or more real steps is looked back on after the answer: the short way that worked becomes a skill (or improves the one it followed); the reply says so. The prompt has the bot check its skills before starting. - Upload, download and PDF on Bot 的電腦's browser look for and save files in the box, not on the brain (a picture in a Threads post took 24 steps). - The bot's browser keeps the six tabs used last (19 Threads tabs had filled the box's 2 GB). - Words name the control, not the box around it: a form's 「Customer name:」, 「Large」 and 「Submit order」 used to land on the

holding them and do nothing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Uirha2RVfSYmrhqNnqx9Z4 --- FEATURES.md | 1 + src/agent.rs | 174 ++++++++++++++++++++++++++++++++++++ src/app.rs | 25 ++++++ src/app/channels_ui.rs | 2 +- src/appctl.rs | 6 +- src/botbrowser.rs | 28 ++++++ src/browser.rs | 60 ++++++++++--- src/cloudbox.rs | 47 ++++++++++ src/memory.rs | 196 ++++++++++++++++++++++++++++++++++++++++- src/providers.rs | 3 + 10 files changed, 521 insertions(+), 21 deletions(-) diff --git a/FEATURES.md b/FEATURES.md index 00e371d..7d0814c 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -1255,6 +1255,7 @@ computer_use_screenshot、click、mouse_move、mouse_button（按住／放開） - **記憶**：跟 Bot 說「記住我喜歡靠窗的座位」，它會用 remember 記下來；「忘記…」用 forget。每個 Bot 最多 200 條、每條 500 字（估計值），每次聊天都帶著。Bot 設定裡的「記得的事」可以一條條刪。 - **技能**：教 Bot 一套做法後請它「存成技能」（save_skill），所有 Bot 共用；系統指示只列名稱和一句說明，Bot 要用時先用 use_skill 讀完整做法。設定 →「連接器」頁的「技能」卡可以刪除。 +- **自己進化**：一件事做了 5 步以上（操作電腦、網頁、指令，不算記憶和設定），回答完後在背景回頭看一次：這是在網站或程式上動手操作的事（填表、發文、回留言、上傳下載、改設定），就把這次真的成功的最短做法存成技能（用畫面上的名字，不寫編號和座標；繞路和失敗寫成「注意」；使用者說過的偏好寫進去）；已經有同一件事的技能就合併成更好的一版，照著做而且很順就不動。回覆下面會多一行「（學會了新做法「…」）」或「（把做法「…」更新了）」。系統指示要 Bot 開始做事前先看技能清單、有符合的先 use_skill。實測（httpbin 訂 pizza，真的 Grok）：第一次 11 步學到技能；第二次先讀技能、10 步 24 秒一次做完，回頭看後判斷不用改。 - **範本**：側邊欄「＋」按右鍵 →「從範本建立」：研究助理、寫作夥伴、電腦助手、客服助理、開發助手，各有角色說明、指示、頭像和顏色。Bot 也能用 create_bot 的 template 參數建立。 ### 保險庫（照原版重做，2026-11-04） diff --git a/src/agent.rs b/src/agent.rs index dd3bef5..c0350d8 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -61,6 +61,8 @@ pub struct Job { pub review: Option, /// 示範教學 this bot has learned (for view_demo_step). pub demos: Vec, + /// The skills there are (a longer job is looked back on against them; see learn). + pub skills: Vec, } #[derive(Clone, Default)] @@ -103,6 +105,37 @@ fn review_action(job: &Job, review: &Review, action: &str, cancel: &AtomicBool) } } +/// 自己進化: after a longer job, the model looks back on its steps and keeps the way that worked +/// as a skill (or improves the one it followed), so the next time takes the short path. It runs +/// after Done: the person isn't kept waiting. +fn learn(job: &Job, history: &[Message], cancel: &AtomicBool, send: &dyn Fn(StreamEvent)) { + let Some(start) = history.iter().rposition(|m| m.role == Role::User && !m.steer) else { return }; + let turn = &history[start..]; + if !crate::memory::worth_learning(turn) { + return; + } + let req = ChatRequest { + system: crate::memory::LEARN_SYSTEM.into(), + history: vec![Message::user(&crate::memory::learn_brief(turn, &job.skills))], + tools: vec![], + ..job.req.clone() + }; + let out = match providers::run_step(&req, cancel, &|_| {}) { + Ok(o) => o, + Err(e) => { + eprintln!("AnyBot：整理做法時模型沒有回應：{e}"); + return; + } + }; + match (crate::memory::learned_skill(&out.text), turn.last()) { + (Some(k), Some(reply)) => { + eprintln!("AnyBot：學到做法「{}」", k.name); + send(StreamEvent::Learned(k, reply.at)); + } + _ => eprintln!("AnyBot：這次的做法沒有存：{}", out.text.split_whitespace().collect::>().join(" ").chars().take(200).collect::()), + } +} + /// Take what the person typed mid-turn and add it to the chat as steering. fn drain_steer(job: &Job, history: &mut Vec, send: &dyn Fn(StreamEvent)) -> bool { let queued: Vec = std::mem::take(&mut *job.steer.lock().unwrap()); @@ -593,6 +626,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< // Something typed while it answered keeps the same turn going. if job.steer.lock().unwrap().is_empty() { send(StreamEvent::Done); + learn(&job, &history, &cancel, &send); return; } continue; @@ -1284,6 +1318,7 @@ mod tests { steer: Arc::new(Mutex::new(vec![])), review: None, demos: vec![], + skills: vec![], } } @@ -1452,6 +1487,141 @@ mod tests { assert!(results[2].contains("盒子裡的測試頁 42"), "{}", results[2]); } + /// The box's browser works with the box's own disk (the brain runs elsewhere): an upload + /// finds the file there, a download and a PDF land there, and old tabs close. Needs Docker + /// and the box: `cargo test the_box_browser_uses_the_box_disk -- --ignored --nocapture`. + #[test] + #[ignore] + fn the_box_browser_uses_the_box_disk() { + let own = crate::model::BOT_COMPUTER_NAME; + let setup = r#"mkdir -p /tmp/anybot-files /tmp/anybot-up && cd /tmp/anybot-files && cat > index.html <<'EOF' +files

檔案測試

+ +

還沒選

下載報表 +EOF +printf 'a,b\n1,2\n' > report.csv && printf 'hello box' > '/tmp/anybot-up/上傳測試.txt' && (setsid python3 -m http.server 8768 --bind 127.0.0.1 >/dev/null 2>&1 &) ; sleep 1; echo served"#; + let check = "ls /home/bot/Downloads/report.csv /home/bot/Downloads/files-*.pdf && head -c 4 /home/bot/Downloads/files-*.pdf && echo"; + let pages = r#"curl -s 127.0.0.1:9222/json/list | python3 -c 'import json,sys; print("pages:", len([t for t in json.load(sys.stdin) if t["type"] == "page"]))'"#; + let clean = r#"pkill -f 'http[.]server 8768'; rm -rf /tmp/anybot-files /tmp/anybot-up /home/bot/Downloads/report.csv /home/bot/Downloads/files-*.pdf; curl -s 127.0.0.1:9222/json/list | python3 -c 'import json,sys,urllib.request; [urllib.request.urlopen("http://127.0.0.1:9222/json/close/" + t["id"]) for t in json.load(sys.stdin) if ":8768" in t["url"]]'; echo cleaned"#; + let mut replies: Vec<(&'static str, serde_json::Value)> = vec![ + ("shell", serde_json::json!({"command": setup})), + ("browser_open", serde_json::json!({"url": "http://127.0.0.1:8768/"})), + ("browser_upload", serde_json::json!({"path": "/tmp/anybot-up/上傳測試.txt"})), + ("browser_read", serde_json::json!({})), + ("browser_upload", serde_json::json!({"path": "/tmp/anybot-up/沒有這個.txt"})), + ("browser_download", serde_json::json!({"text": "下載報表"})), + ("browser_pdf", serde_json::json!({})), + ("shell", serde_json::json!({"command": check})), + ]; + for n in 1..=8 { + replies.push(("browser_open", serde_json::json!({"url": format!("http://127.0.0.1:8768/?n={n}")}))); + } + replies.push(("shell", serde_json::json!({"command": pages}))); + replies.push(("shell", serde_json::json!({"command": clean}))); + replies.push(("", serde_json::json!("好了"))); + let port = fake_model(replies); + let job = Job { + req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("測試盒子裡的檔案")], tools: tools::specs(), session: String::new() }, + computers: vec![tools::Target { name: own.into(), place: tools::Place::Box { dir: crate::cloudbox::bot_dir(1) } }], + default_computer: own.into(), + roles: tools::Roles { own: own.into(), person: None, server: true }, + local_execution: false, + direct: Arc::new(AtomicBool::new(true)), + ..test_job() + }; + let (tx, rx) = channel(); + spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default()); + let mut results = vec![]; + loop { + match rx.recv_timeout(Duration::from_secs(180)).expect("loop stalled") { + (_, StreamEvent::ToolResult(m)) => { + println!("{}: {}\n", m.tool_name, m.text.chars().take(300).collect::()); + results.push(m.text) + } + (_, StreamEvent::Done) => break, + (_, StreamEvent::Error(e)) => panic!("{e}"), + _ => {} + } + } + assert!(results[0].contains("served"), "{}", results[0]); + assert!(results[1].contains("檔案測試"), "{}", results[1]); + assert!(results[2].contains("已把 /tmp/anybot-up/上傳測試.txt 放進上傳欄位"), "{}", results[2]); + assert!(results[3].contains("已選：上傳測試.txt 9"), "{}", results[3]); + assert!(results[4].contains("這台電腦上沒有「/tmp/anybot-up/沒有這個.txt」"), "{}", results[4]); + assert!(results[5].contains("下載好了：/home/bot/Downloads/report.csv"), "{}", results[5]); + assert!(results[6].contains("已存成 PDF：/home/bot/Downloads/files-"), "{}", results[6]); + assert!(results[7].contains("%PDF") && !results[7].contains("No such file"), "{}", results[7]); + let count = results[16].split("pages:").nth(1).and_then(|n| n.trim().parse::().ok()).unwrap_or(99); + assert!(count <= 6, "{}", results[16]); + assert!(results[17].contains("cleaned"), "{}", results[17]); + } + + /// Words reach the control they name on a form laid out like httpbin's (each label and its + /// input inside a

): typing into 「Customer name:」, ticking 「Large」, pressing 「Submit + /// order」. They used to land on the

and do nothing. Needs the box: + /// `cargo test the_words_on_a_form_reach_its_controls -- --ignored --nocapture`. + #[test] + #[ignore] + fn the_words_on_a_form_reach_its_controls() { + let own = crate::model::BOT_COMPUTER_NAME; + let setup = r#"mkdir -p /tmp/anybot-form && cd /tmp/anybot-form && cat > index.html <<'EOF' +form +

+

+

+
Pizza Size +

+

+
Pizza Toppings +

+

+

+
+EOF +echo 'done送出了' > done.html; (setsid python3 -m http.server 8769 --bind 127.0.0.1 >/dev/null 2>&1 &) ; sleep 1; echo served"#; + let clean = r#"pkill -f 'http[.]server 8769'; rm -rf /tmp/anybot-form; curl -s 127.0.0.1:9222/json/list | python3 -c 'import json,sys,urllib.request; [urllib.request.urlopen("http://127.0.0.1:9222/json/close/" + t["id"]) for t in json.load(sys.stdin) if ":8769" in t["url"]]'; echo cleaned"#; + let port = fake_model(vec![ + ("shell", serde_json::json!({"command": setup})), + ("browser_open", serde_json::json!({"url": "http://127.0.0.1:8769/"})), + ("browser_type", serde_json::json!({"text": "Customer name:", "value": "李大華"})), + ("browser_type", serde_json::json!({"text": "Telephone", "value": "0922000111"})), + ("browser_click", serde_json::json!({"text": "Large"})), + ("browser_click", serde_json::json!({"text": "Extra Cheese"})), + ("browser_click", serde_json::json!({"text": "Onion"})), + ("browser_click", serde_json::json!({"text": "Submit order"})), + ("shell", serde_json::json!({"command": clean})), + ("", serde_json::json!("好了")), + ]); + let job = Job { + req: ChatRequest { kind: ProviderKind::Custom, base_url: format!("http://127.0.0.1:{port}/v1"), api_key: String::new(), model: "m".into(), system: String::new(), history: vec![Message::user("填表")], tools: tools::specs(), session: String::new() }, + computers: vec![tools::Target { name: own.into(), place: tools::Place::Box { dir: crate::cloudbox::bot_dir(1) } }], + default_computer: own.into(), + roles: tools::Roles { own: own.into(), person: None, server: true }, + local_execution: false, + direct: Arc::new(AtomicBool::new(true)), + ..test_job() + }; + let (tx, rx) = channel(); + spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default()); + let mut results = vec![]; + loop { + match rx.recv_timeout(Duration::from_secs(180)).expect("loop stalled") { + (_, StreamEvent::ToolResult(m)) => { + println!("{}: {}\n", m.tool_name, m.text.chars().take(240).collect::()); + results.push(m.text) + } + (_, StreamEvent::Done) => break, + (_, StreamEvent::Error(e)) => panic!("{e}"), + _ => {} + } + } + assert!(results[2].contains("欄位現在是「李大華」"), "{}", results[2]); + assert!(results[3].contains("欄位現在是「0922000111」"), "{}", results[3]); + let sent = &results[7]; + assert!(sent.contains("done.html?custname=%E6%9D%8E%E5%A4%A7%E8%8F%AF&custtel=0922000111&size=large&topping=cheese&topping=onion"), "{sent}"); + assert!(results[8].contains("cleaned"), "{}", results[8]); + } + #[test] fn the_same_miss_counts_as_the_same_step() { // Clicks a few pixels apart on the same strip are one step; another computer or target isn't. @@ -1596,6 +1766,7 @@ mod tests { steer: Arc::new(Mutex::new(vec![])), review: None, demos: vec![], + skills: vec![], }; let (tx, rx) = channel(); let cancel = Arc::new(AtomicBool::new(false)); @@ -1668,6 +1839,7 @@ mod tests { steer, review: None, demos: vec![], + skills: vec![], }; let (tx, rx) = channel(); spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default()); @@ -1746,6 +1918,7 @@ mod tests { steer: Arc::new(Mutex::new(vec![])), review: None, demos: vec![], + skills: vec![], }; let (tx, rx) = channel(); spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default()); @@ -1885,6 +2058,7 @@ mod tests { steer: Arc::new(Mutex::new(vec![])), review: None, demos: vec![], + skills: vec![], }; let (tx, rx) = channel(); spawn(job, 1, tx, Arc::new(AtomicBool::new(false)), eframe::egui::Context::default()); diff --git a/src/app.rs b/src/app.rs index 89515e6..5994735 100644 --- a/src/app.rs +++ b/src/app.rs @@ -832,6 +832,7 @@ impl App { steer, review, demos: b.demos.clone(), + skills: self.state.skills.clone(), }; agent::spawn(job, bot_id, self.tx.clone(), cancel, ctx.clone()); // (Its eyes start reading while it works; see face_pose.) @@ -1360,6 +1361,16 @@ impl App { fn drain_events(&mut self, ctx: &egui::Context) { while let Ok((id, ev)) = self.rx.try_recv() { + // 自己進化 arrives after the turn has ended: keep the skill, say so under the reply. + if let StreamEvent::Learned(k, reply_at) = ev { + let name = k.name.clone(); + let fresh = crate::memory::upsert_skill(&mut self.state.skills, k); + if let Some(m) = self.state.bot_mut(id).and_then(|b| b.messages.iter_mut().rev().find(|m| m.role == Role::Assistant && m.at == reply_at)) { + m.text.push_str(&crate::memory::learned_note(&name, fresh)); + } + self.dirty = true; + continue; + } // A stopped request may still deliver a few pieces; ignore them. if !self.streams.contains_key(&id) { continue; @@ -3560,6 +3571,20 @@ impl App { std::thread::sleep(std::time::Duration::from_millis(100)); } let secs = t0.elapsed().as_secs(); + // 自己進化 comes after the turn: wait for it when the turn was long enough to learn from. + let turn_msgs = app.state.bot(id).and_then(|b| b.messages.get(start..)).map(|m| m.to_vec()).unwrap_or_default(); + if crate::memory::worth_learning(&turn_msgs) { + let before = app.state.skills.clone(); + let t1 = Instant::now(); + while app.state.skills == before && t1.elapsed().as_secs() < 180 { + app.drain_events(ctx); + std::thread::sleep(std::time::Duration::from_millis(200)); + } + match app.state.skills.iter().find(|k| !before.contains(k)) { + Some(k) => println!("📚 學到（{} 秒）：{} — {}\n{}\n", t1.elapsed().as_secs(), k.name, k.description, k.instructions), + None => println!("📚 這次沒有存技能（{} 秒）", t1.elapsed().as_secs()), + } + } // The chat can shrink (compaction) between the snapshot and now: slice safely. let msgs = app.state.bot(id).and_then(|b| b.messages.get(start..)).map(|m| m.to_vec()).unwrap_or_default(); let clip = |s: &str, n: usize| -> String { diff --git a/src/app/channels_ui.rs b/src/app/channels_ui.rs index 744fd39..81a3353 100644 --- a/src/app/channels_ui.rs +++ b/src/app/channels_ui.rs @@ -356,7 +356,7 @@ impl App { }); } setting_card(ui, p, |ui| { - setting_title(ui, "技能", "所有 Bot 共用的做法說明。在聊天裡教 Bot 一套做法並請它「存成技能」；Bot 需要時會先讀取再照做。", p); + setting_title(ui, "技能", "所有 Bot 共用的做法。Bot 做完較長的事，會自己把成功的做法整理成技能、下次照著做，做得更好時也會更新；你也可以在聊天裡教它一套做法。", p); if self.state.skills.is_empty() { hint(ui, "還沒有技能。", p); } diff --git a/src/appctl.rs b/src/appctl.rs index a1128d6..988d804 100644 --- a/src/appctl.rs +++ b/src/appctl.rs @@ -425,11 +425,7 @@ pub fn apply(state: &mut State, bot_id: u64, name: &str, args: &str) -> Outcome if name.is_empty() || instructions.trim().is_empty() { return err("技能要有名稱和做法".into()); } - let k = crate::memory::Skill { name: name.clone(), description: s("description").unwrap_or_default(), instructions }; - match state.skills.iter_mut().find(|x| x.name == name) { - Some(x) => *x = k, - None => state.skills.push(k), - } + crate::memory::upsert_skill(&mut state.skills, crate::memory::Skill { name: name.clone(), description: s("description").unwrap_or_default(), instructions }); ok(format!("已存成技能「{name}」，所有 Bot 都能用。")) } "use_skill" => { diff --git a/src/botbrowser.rs b/src/botbrowser.rs index 68c9861..df5cd3e 100644 --- a/src/botbrowser.rs +++ b/src/botbrowser.rs @@ -317,6 +317,7 @@ pub fn open(url: &str, profile: Option<&str>) -> Result { if r.status().as_u16() >= 400 { return Err(format!("{name} 沒辦法開新分頁（{}）", r.status())); } + close_old_tabs(port); } // Back when the page has loaded (not after a fixed 3 s: a slow page was still blank then), // with what it shows — the same as browser_read, one step fewer for every page. @@ -326,6 +327,23 @@ pub fn open(url: &str, profile: Option<&str>) -> Result { Ok(if page.is_empty() { head } else { format!("{head}\n\n{page}") }) } +/// Most tabs the bot's browser keeps. Every browser_open is a new tab and nothing closed them: +/// Bot 的電腦 once had 19 Threads pages open and its whole 2 GB in use (estimated). +const MAX_TABS: usize = 6; + +/// Close the tabs past MAX_TABS that were used longest ago (Chrome lists the latest first). +fn close_old_tabs(port: u16) { + let Ok(list) = local().get(&endpoint(port, "/json/list")).call().map_err(|e| e.to_string()).and_then(|r| r.into_body().read_json::().map_err(|e| e.to_string())) else { return }; + for id in old_tabs(&list) { + let _ = local().get(&endpoint(port, &format!("/json/close/{id}"))).call(); + } +} + +/// The page tabs past MAX_TABS in a /json/list answer (latest-used first). +fn old_tabs(list: &Value) -> Vec { + list.as_array().map(|a| a.iter().filter(|t| t["type"] == "page").skip(MAX_TABS).filter_map(|t| t["id"].as_str().map(String::from)).collect()).unwrap_or_default() +} + /// Longest wait for a page browser_open opened (estimated). const OPEN_WAIT_MS: u64 = 12_000; /// Text of the page handed back by browser_open (browser_read gives more when asked). @@ -431,6 +449,16 @@ mod tests { println!("{page}"); } + #[test] + fn only_the_tabs_used_longest_ago_close() { + let mut list: Vec = (0..9).map(|i| serde_json::json!({"type": "page", "id": format!("p{i}")})).collect(); + list.insert(2, serde_json::json!({"type": "browser_ui", "id": "ui"})); + list.insert(4, serde_json::json!({"type": "service_worker", "id": "sw"})); + assert_eq!(old_tabs(&Value::Array(list.clone())), vec!["p6", "p7", "p8"]); + list.truncate(5); + assert!(old_tabs(&Value::Array(list)).is_empty()); + } + #[test] fn urls_are_encoded_for_new_tabs() { assert_eq!(encode_component("https://www.threads.net/@me?x=1&y=2"), "https://www.threads.net/%40me%3Fx%3D1%26y%3D2"); diff --git a/src/browser.rs b/src/browser.rs index 027a0de..e56c00c 100644 --- a/src/browser.rs +++ b/src/browser.rs @@ -228,7 +228,20 @@ fn find_js(r#ref: &str, text: &str) -> String { const vis = e => {{ const r = e.getBoundingClientRect(); return r.width > 1 && r.height > 1; }}; const all = [...document.querySelectorAll('a,button,[role],input,select,textarea,label,summary,li,td,th,span,div,p,img')].filter(vis); const hit = e => [t(e.innerText), t(e.getAttribute('aria-label')), t(e.getAttribute('placeholder')), t(e.title), t(e.value), e.labels && e.labels[0] ? t(e.labels[0].innerText) : '']; - return all.find(e => hit(e).includes(tx)) || all.filter(e => hit(e).some(h => h.includes(tx))).sort((a, b) => t(a.innerText).length - t(b.innerText).length)[0] || null; + // The control the words name, not the box around it: 「Customer name:」 is a

holding a + // label and its input, 「Submit order」 a

holding the button (its middle is empty page). + const ctl = 'a,button,input,select,textarea,summary,[role=button],[role=link],[role=checkbox],[role=radio],[role=switch],[role=tab],[role=menuitem],[role=option],[role=textbox],[role=combobox],[contenteditable=""],[contenteditable=true]'; + const isCtl = e => e.matches(ctl); + const inner = e => {{ + if (isCtl(e)) return e; + if (e.tagName === 'LABEL' && e.control && vis(e.control)) return e.control; + const c = [...e.querySelectorAll(ctl)].filter(vis); + return c.length === 1 ? c[0] : e; + }}; + const shortest = l => l.slice().sort((a, b) => t(a.innerText).length - t(b.innerText).length)[0]; + const pick = l => l.find(isCtl) || l.map(inner).find(isCtl) || shortest(l) || null; + const exact = all.filter(e => hit(e).includes(tx)); + return exact.length ? pick(exact) : pick(all.filter(e => hit(e).some(h => h.includes(tx)))); }})()"# ) } @@ -704,31 +717,43 @@ pub fn run(name: &str, args: &Value) -> Result<(String, Option), String> } "browser_upload" => { let path = s(args, "path"); - let full = crate::tools::expand_home(&path); - if !full.is_file() { + // The browser opens the file from its own disk: Bot 的電腦's when it runs there. + let in_box = botbrowser::using_box(); + let full = if in_box { crate::cloudbox::box_path(&path) } else { crate::tools::expand_home(&path).display().to_string() }; + let there = if in_box { crate::cloudbox::box_has_file(&full) } else { std::path::Path::new(&full).is_file() }; + if !there { return Err(format!("這台電腦上沒有「{path}」這個檔案")); } let js = format!("(() => {{ let el = {}; if (el && el.tagName !== 'INPUT') el = el.querySelector('input[type=file]') || el.closest('label')?.querySelector('input[type=file]'); return el || document.querySelector('input[type=file]'); }})()", find_js(&rf, &text)); let r = tab.call("Runtime.evaluate", json!({"expression": js, "returnByValue": false}))?; let obj = r["result"]["objectId"].as_str().ok_or("這一頁找不到上傳檔案的欄位")?.to_string(); - tab.call("DOM.setFileInputFiles", json!({"files": [full.display().to_string()], "objectId": obj}))?; + tab.call("DOM.setFileInputFiles", json!({"files": [full], "objectId": obj}))?; tab.eval(&format!("(() => {{ const el = {js}; if (el) el.dispatchEvent(new Event('change', {{bubbles: true}})); return 1; }})()"))?; settle(1000); - Ok((format!("已把 {} 放進上傳欄位。", full.display()), None)) + Ok((format!("已把 {full} 放進上傳欄位。"), None)) } "browser_download" => { - let dir = downloads_dir(); + // The file lands on the browser's own disk: Bot 的電腦's when it runs there. + let in_box = botbrowser::using_box(); + let dir = if in_box { crate::cloudbox::BOX_DOWNLOADS.to_string() } else { downloads_dir().display().to_string() }; + if in_box { + crate::cloudbox::box_mkdir(&dir)?; + } let clicked_at = std::time::SystemTime::now() - Duration::from_secs(1); - tab.call("Browser.setDownloadBehavior", json!({"behavior": "allow", "downloadPath": dir.display().to_string(), "eventsEnabled": false})).or_else(|_| tab.call("Page.setDownloadBehavior", json!({"behavior": "allow", "downloadPath": dir.display().to_string()})))?; + tab.call("Browser.setDownloadBehavior", json!({"behavior": "allow", "downloadPath": dir, "eventsEnabled": false})).or_else(|_| tab.call("Page.setDownloadBehavior", json!({"behavior": "allow", "downloadPath": dir})))?; let (x, y, what) = locate(&mut tab, &rf, &text)?; click_at(&mut tab, x, y, "left", 1)?; let start = Instant::now(); while start.elapsed().as_secs() < 60 { settle(700); // A new file, or one written again since the click (same name replaced). - let now: Vec = std::fs::read_dir(&dir).map(|r| r.flatten().filter(|e| e.metadata().and_then(|m| m.modified()).map(|t| t >= clicked_at).unwrap_or(false)).map(|e| e.path()).collect()).unwrap_or_default(); - if let Some(f) = now.iter().find(|p| !p.extension().map(|e| e == "crdownload" || e == "tmp").unwrap_or(false)) { - return Ok((format!("已點 {what}，下載好了：{}（可以用 read_file 讀它）", f.display()), None)); + let now: Vec = if in_box { + crate::cloudbox::box_new_files(&dir, clicked_at.duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0)) + } else { + std::fs::read_dir(&dir).map(|r| r.flatten().filter(|e| e.metadata().and_then(|m| m.modified()).map(|t| t >= clicked_at).unwrap_or(false)).map(|e| e.path().display().to_string()).collect()).unwrap_or_default() + }; + if let Some(f) = now.iter().find(|p| !(p.ends_with(".crdownload") || p.ends_with(".tmp"))) { + return Ok((format!("已點 {what}，下載好了：{f}（可以用 read_file 讀它）"), None)); } } Err(format!("已點 {what}，但 60 秒內沒有下載到檔案（有些網站要先選格式或日期，或在新分頁產生報表）。")) @@ -739,9 +764,18 @@ pub fn run(name: &str, args: &Value) -> Result<(String, Option), String> let bytes = base64::engine::general_purpose::STANDARD.decode(r["data"].as_str().unwrap_or("")).map_err(|e| e.to_string())?; let (title, _, _) = snapshot(&mut tab); let safe: String = title.chars().filter(|c| !"/\\:*?\"<>|".contains(*c)).take(60).collect(); - let path = downloads_dir().join(format!("{}-{}.pdf", if safe.trim().is_empty() { "網頁".into() } else { safe }, chrono::Local::now().format("%Y%m%d-%H%M%S"))); - std::fs::write(&path, bytes).map_err(|e| e.to_string())?; - Ok((format!("已存成 PDF：{}", path.display()), None)) + let name = format!("{}-{}.pdf", if safe.trim().is_empty() { "網頁".into() } else { safe }, chrono::Local::now().format("%Y%m%d-%H%M%S")); + // Saved where the bot can open it: on Bot 的電腦 when that's the browser's computer. + let path = if botbrowser::using_box() { + let p = format!("{}/{name}", crate::cloudbox::BOX_DOWNLOADS); + crate::cloudbox::box_write(&p, &bytes)?; + p + } else { + let p = downloads_dir().join(name); + std::fs::write(&p, bytes).map_err(|e| e.to_string())?; + p.display().to_string() + }; + Ok((format!("已存成 PDF：{path}"), None)) } "browser_dialog" => { let mut p = json!({"accept": args["accept"].as_bool().unwrap_or(true)}); diff --git a/src/cloudbox.rs b/src/cloudbox.rs index 99445f6..918e9fd 100644 --- a/src/cloudbox.rs +++ b/src/cloudbox.rs @@ -1598,6 +1598,46 @@ fn box_exec(script: &str) -> Result { } } +// ---------- files for the box's browser ---------- +// The brain drives the box's Chromium from outside, but that browser reads and writes the box's +// own disk: an upload, a download or a PDF has to be looked for there, not on the brain (a +// cloud brain once told the bot an attachment sitting in the box "isn't on this computer", and +// the bot spent 24 steps getting one picture into a post). + +/// Where the box's browser keeps downloads and PDFs (the bot reads them there). +pub const BOX_DOWNLOADS: &str = "/home/bot/Downloads"; + +/// A path on Bot 的電腦 as written by the bot: `~` is the bot user's home. +pub fn box_path(p: &str) -> String { + match p.trim() { + "~" => "/home/bot".into(), + t => t.strip_prefix("~/").map(|rest| format!("/home/bot/{rest}")).unwrap_or_else(|| t.to_string()), + } +} + +/// There is a file at `path` on Bot 的電腦. +pub fn box_has_file(path: &str) -> bool { + box_exec(&format!("[ -f {} ] && echo yes", crate::tools::sh_quote(path))).map(|o| o.trim() == "yes").unwrap_or(false) +} + +/// Files in `dir` on Bot 的電腦 written at or after `since` (seconds since 1970). +pub fn box_new_files(dir: &str, since: u64) -> Vec { + let script = format!("find {} -maxdepth 1 -type f -newermt @{since} 2>/dev/null; true", crate::tools::sh_quote(dir)); + box_exec(&script).map(|o| o.lines().map(str::trim).filter(|l| !l.is_empty()).map(String::from).collect()).unwrap_or_default() +} + +/// Make a folder on Bot 的電腦 (as the bot user). +pub fn box_mkdir(dir: &str) -> Result<(), String> { + box_exec(&format!("mkdir -p {}", crate::tools::sh_quote(dir))).map(|_| ()) +} + +/// Write a file on Bot 的電腦 (as the bot user), making its folder first. +pub fn box_write(path: &str, bytes: &[u8]) -> Result<(), String> { + let dir = std::path::Path::new(path).parent().map(|d| d.display().to_string()).unwrap_or_else(|| "/home/bot".into()); + let script = format!("mkdir -p {} && cat > {}", crate::tools::sh_quote(&dir), crate::tools::sh_quote(path)); + dk_with_input(&["exec", "-i", "-u", "bot", container(), "sh", "-c", &script], Some(bytes), 60_000).map(|_| ()).map_err(|e| format!("存不進 Bot 的電腦：{e}")) +} + /// The numbered controls of the box's latest look (centres in desktop pixels). static BOX_SPOTS: Mutex> = Mutex::new(None); /// The latest look's fingerprint (to say when a step changed nothing). @@ -2128,6 +2168,13 @@ mod tests { /// (xterm has no accessibility; clicking a number there used to type into the terminal). #[test] #[ignore] + fn box_paths_are_the_bot_users() { + assert_eq!(box_path("~/bot-6/附件/IMG_0587.jpeg"), "/home/bot/bot-6/附件/IMG_0587.jpeg"); + assert_eq!(box_path(" ~ "), "/home/bot"); + assert_eq!(box_path("/tmp/a.jpg"), "/tmp/a.jpg"); + } + + #[test] fn box_numbers_only_on_what_is_seen() { ensure_running().expect("box starts"); let cancel = AtomicBool::new(false); diff --git a/src/memory.rs b/src/memory.rs index 7b9db8b..30707f3 100644 --- a/src/memory.rs +++ b/src/memory.rs @@ -63,6 +63,121 @@ pub fn forget(list: &mut Vec, what: &str) -> Result { } } +/// Add a skill, or replace the one with the same name. True when it is new. +pub fn upsert_skill(list: &mut Vec, k: Skill) -> bool { + match list.iter_mut().find(|x| x.name == k.name) { + Some(x) => { + *x = k; + false + } + None => { + list.push(k); + true + } + } +} + +// ---------- 自己進化: a job's way that worked becomes a skill ---------- +// The bot forgot everything between chats: a post with a picture on Threads took it 39 steps, +// and the next one would have taken as many. After a longer job the model looks back at the +// steps and writes down the short path that worked (or improves the skill it followed); the +// next time it reads that first. + +/// Steps on computers and pages a job needs before it is worth looking back on (estimated). +pub const LEARN_MIN_STEPS: usize = 5; +/// Longest learned skill (estimated; a skill is read whole before each such job). +const MAX_SKILL_CHARS: usize = 4000; + +/// A turn that did enough real work, and didn't already save a skill itself. +pub fn worth_learning(turn: &[crate::model::Message]) -> bool { + let calls: Vec<&str> = turn.iter().flat_map(|m| m.tool_calls.iter().map(|c| c.name.as_str())).collect(); + !calls.contains(&"save_skill") && calls.iter().filter(|n| !crate::appctl::is_app_tool(n) && **n != "react").count() >= LEARN_MIN_STEPS +} + +pub const LEARN_SYSTEM: &str = "你在幫一個會操作電腦的 Bot 整理經驗，讓它下次做同樣的事更快、不再犯同樣的錯。下面是它剛做完的一件事：使用者的要求、每一步和結果、最後的回覆，還有它已經有的技能。 + +先判斷要不要存。使用者希望它從做過的事學會、下次不用重新摸索：只要這件事是在某個網站或程式上動手操作（填表、發文、回留言、按讚、上傳下載、改設定、跑一套固定的檢查），就要存，就算這次只做了一次。只有閒聊、單純回答問題、只是查資料讀內容，才不用存。這次沒做成功的，不要存成做法；但如果已經有這件事的技能，可以把「這樣做行不通」補進去。 + +要存的話，寫一份「下次照著做就會成功」的技能： +- 寫這次真的成功的那條最短路徑，一步一行，寫出用哪個工具、在哪台電腦、哪個網址或程式（每一步結果開頭 [ ] 裡的名字，就是那一步用的電腦）。 +- 繞路和失敗的嘗試不要寫成步驟；把「哪裡會卡、要怎麼避開」寫成「注意：」。 +- 用畫面上看得到的名字指東西（例如 browser_click 的 text 寫「Attach media」、browser_type 的 text 寫欄位名「Customer name」），不要寫元素編號或座標，那些每次都會變。browser_open 的結果就有這一頁的元素清單，不用另外截圖。 +- 寫通用的做法，不要寫這次的內容（這篇文的文字、這張圖的檔名）；使用者表達過的偏好（語氣、格式、一定要附圖、用哪台電腦）要寫進去。 +- 已經有做同一件事的技能，就用同一個名稱，把舊的和這次學到的合併成更好的一版：舊的對的保留，錯的改掉。這次照著技能做而且很順，就不用再存。 + +只回一個 JSON，不要別的字： +不用存：{\"save\": false, \"why\": \"一句話說為什麼\"} +要存：{\"save\": true, \"name\": \"短名稱\", \"description\": \"一行：什麼時候用\", \"instructions\": \"做法\"}"; + +fn cut(s: &str, n: usize) -> String { + let flat = s.split_whitespace().collect::>().join(" "); + if flat.chars().count() <= n { flat } else { format!("{}…", flat.chars().take(n).collect::()) } +} + +/// What the model looks back on: the request, each step with its result, the answer and the +/// skills there are. +pub fn learn_brief(turn: &[crate::model::Message], skills: &[Skill]) -> String { + use crate::model::Role; + let mut steps: Vec = vec![]; + for (i, m) in turn.iter().enumerate() { + if m.role == Role::User && i > 0 { + steps.push(format!("（使用者中途說：{}）", cut(&m.text, 300))); + } + for c in &m.tool_calls { + let result = turn.iter().find(|r| r.role == Role::Tool && r.tool_call_id == c.id).map(|r| r.text.as_str()).unwrap_or(""); + let failed = if c.status == crate::model::ToolStatus::Failed { "（失敗）" } else { "" }; + steps.push(format!("{} {} → {}{failed}", c.name, cut(&c.args, 160), cut(result, 220))); + } + } + const KEEP: usize = 40; + if steps.len() > KEEP * 2 { + let gone = steps.len() - KEEP * 2; + steps.splice(KEEP..steps.len() - KEEP, [format!("…（中間省略 {gone} 步）…")]); + } + let ask = turn.first().map(|m| cut(&m.text, 800)).unwrap_or_default(); + let answer = turn.iter().rev().find(|m| m.role == crate::model::Role::Assistant && !m.text.trim().is_empty()).map(|m| cut(&m.text, 600)).unwrap_or_default(); + let mut s = format!("使用者的要求：\n{ask}\n\n每一步（工具 參數 → 結果）：\n"); + for (i, st) in steps.iter().enumerate() { + s.push_str(&format!("{}. {st}\n", i + 1)); + } + s.push_str(&format!("\n最後的回覆：\n{answer}\n\n已經有的技能：")); + if skills.is_empty() { + s.push_str("（沒有）"); + } + let mut room = 8000usize; + for k in skills { + let body = cut(&k.instructions, 2000.min(room)); + room = room.saturating_sub(body.chars().count()); + s.push_str(&format!("\n- {}：{}\n 做法：{body}", k.name, k.description)); + } + s +} + +/// The skill in the model's answer, when it chose to save one. +pub fn learned_skill(text: &str) -> Option { + let t = text.trim(); + let json = t.find('{').and_then(|a| t.rfind('}').filter(|b| *b >= a).map(|b| &t[a..=b]))?; + let v: serde_json::Value = serde_json::from_str(json).ok()?; + if v["save"].as_bool() != Some(true) { + return None; + } + let name: String = v["name"].as_str()?.trim().chars().take(40).collect(); + let instructions: String = v["instructions"].as_str()?.trim().chars().take(MAX_SKILL_CHARS).collect(); + if name.is_empty() || instructions.is_empty() { + return None; + } + Some(Skill { name, description: v["description"].as_str().unwrap_or("").trim().chars().take(120).collect(), instructions }) +} + +/// The line under the bot's reply that says it learned (or improved) a skill. +pub fn learned_note(name: &str, fresh: bool) -> String { + if fresh { + format!("\n\n（學會了新做法「{name}」，下次做這件事會照著做。）") + } else { + format!("\n\n（把做法「{name}」更新了，下次會照新的做。）") + } +} + pub fn prompt_section(memories: &[String], skills: &[Skill]) -> String { let mut s = String::new(); if !memories.is_empty() { @@ -74,7 +189,7 @@ pub fn prompt_section(memories: &[String], skills: &[Skill]) -> String { s.push_str("\n\n使用者說「記住…」「以後都…」時，用 remember 記下來；說「忘記…」時用 forget。"); } if !skills.is_empty() { - s.push_str("\n\n可用的技能（做法說明；需要時先用 use_skill 讀取完整內容再照做）：\n"); + s.push_str("\n\n你會的技能（以前做成功的做法）。開始做一件事之前先看這裡：有符合的，先用 use_skill 讀出完整做法再照著做，不要重新摸索：\n"); for k in skills { s.push_str(&format!("- {}：{}\n", k.name, k.description)); } @@ -102,6 +217,83 @@ mod tests { assert_eq!(m.len(), MAX_MEMORIES); assert_eq!(m[0], "事 5"); let p = prompt_section(&m[..1], &[Skill { name: "週報".into(), description: "整理每週進度".into(), instructions: "…".into() }]); - assert!(p.contains("1. 事 5") && p.contains("- 週報：整理每週進度")); + assert!(p.contains("1. 事 5") && p.contains("- 週報：整理每週進度") && p.contains("use_skill")); + } + + fn call(id: &str, name: &str, args: &str, failed: bool) -> crate::model::ToolCall { + crate::model::ToolCall { id: id.into(), name: name.into(), args: args.into(), status: if failed { crate::model::ToolStatus::Failed } else { crate::model::ToolStatus::Done } } + } + + fn result(id: &str, text: &str) -> crate::model::Message { + crate::model::Message { role: crate::model::Role::Tool, tool_call_id: id.into(), text: text.into(), ..Default::default() } + } + + /// A post with a picture: five real steps, one of them failed, the person corrects it midway. + fn post_turn() -> Vec { + use crate::model::Message; + let step = |c: crate::model::ToolCall| Message { tool_calls: vec![c], ..Message::assistant("", "b") }; + vec![ + Message::user("幫我發一篇 Threads，記得附圖"), + step(call("1", "browser_open", r#"{"url":"https://www.threads.com/"}"#, false)), + result("1", "已打開 https://www.threads.com/\n[13] 按鈕「Create」"), + step(call("2", "browser_click", r#"{"ref":"13"}"#, false)), + result("2", "已點 [13]。新出現的文字：New thread"), + Message { steer: true, ..Message::user("語氣要輕鬆一點") }, + step(call("3", "browser_type", r#"{"ref":"12","value":"今天天氣好"}"#, false)), + result("3", "已在 [12] 輸入。"), + step(call("4", "browser_upload", r#"{"path":"/home/bot/x.jpg"}"#, true)), + result("4", "這台電腦上沒有這個檔案"), + step(call("5", "browser_upload", r#"{"path":"/home/bot/bot-6/附件/x.jpg"}"#, false)), + result("5", "已把 /home/bot/bot-6/附件/x.jpg 放進上傳欄位。"), + step(call("6", "remember", r#"{"fact":"x"}"#, false)), + result("6", "記住了"), + Message::assistant("發好了：https://www.threads.com/@me/post/1", "b"), + ] + } + + #[test] + fn a_long_enough_job_is_looked_back_on() { + let turn = post_turn(); + assert!(worth_learning(&turn)); + // App steps (remember) don't count; four real steps aren't enough. + assert!(!worth_learning(&turn[..10])); + // A job that saved a skill itself is left alone. + let mut saved = turn.clone(); + saved[12].tool_calls[0].name = "save_skill".into(); + assert!(!worth_learning(&saved)); + } + + #[test] + fn the_look_back_has_every_step_and_the_skills() { + let skills = [Skill { name: "Threads 發文".into(), description: "在 Threads 發文".into(), instructions: "1. browser_open https://www.threads.com/".into() }]; + let b = learn_brief(&post_turn(), &skills); + assert!(b.starts_with("使用者的要求：\n幫我發一篇 Threads，記得附圖"), "{b}"); + assert!(b.contains("1. browser_open {\"url\":\"https://www.threads.com/\"} → 已打開 https://www.threads.com/ [13] 按鈕「Create」"), "{b}"); + assert!(b.contains("（使用者中途說：語氣要輕鬆一點）"), "{b}"); + assert!(b.contains("browser_upload {\"path\":\"/home/bot/x.jpg\"} → 這台電腦上沒有這個檔案（失敗）"), "{b}"); + assert!(b.contains("最後的回覆：\n發好了"), "{b}"); + assert!(b.contains("- Threads 發文：在 Threads 發文\n 做法：1. browser_open"), "{b}"); + // A very long job keeps its start and its end. + let mut long = vec![crate::model::Message::user("做很多事")]; + for i in 0..200 { + long.push(crate::model::Message { tool_calls: vec![call(&i.to_string(), "browser_click", "{}", false)], ..crate::model::Message::assistant("", "b") }); + } + let b = learn_brief(&long, &[]); + assert!(b.contains("（中間省略 120 步）") && b.contains("81. browser_click") && !b.contains("82. "), "{b}"); + } + + #[test] + fn a_learned_skill_is_read_from_the_answer() { + let k = learned_skill("好的\n{\"save\": true, \"name\": \" Threads 發文附圖 \", \"description\": \"在 Threads 發一篇附圖的文\", \"instructions\": \"1. browser_open https://www.threads.com/\"}").unwrap(); + assert_eq!((k.name.as_str(), k.description.as_str()), ("Threads 發文附圖", "在 Threads 發一篇附圖的文")); + assert!(learned_skill(r#"{"save": false}"#).is_none()); + assert!(learned_skill(r#"{"save": true, "name": "", "instructions": "x"}"#).is_none()); + assert!(learned_skill("不用存").is_none()); + assert!(learned_skill("} {").is_none()); + let mut list = vec![]; + assert!(upsert_skill(&mut list, k.clone())); + assert!(!upsert_skill(&mut list, Skill { instructions: "新的".into(), ..k })); + assert_eq!((list.len(), list[0].instructions.as_str()), (1, "新的")); + assert!(learned_note("Threads 發文附圖", true).contains("學會了新做法「Threads 發文附圖」")); } } diff --git a/src/providers.rs b/src/providers.rs index 8c96983..f331df6 100644 --- a/src/providers.rs +++ b/src/providers.rs @@ -199,6 +199,9 @@ pub enum StreamEvent { React(String), /// The bot sent a file into the chat (a file card under its reply). Attach(crate::attach::Attachment), + /// 自己進化: after the turn, the way that worked was kept as a skill (with the time of the + /// reply it learned from, which gets a line saying so). + Learned(crate::memory::Skill, i64), Done, Error(String), }