AnyBot/src/memory.rs

404 lines
24 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// 記憶、技能、範本 (the original: user bot memories, agent skills and template skills/bots).
// - Memories belong to a bot: facts it was asked to keep across chats.
// - Skills are shared, named procedures; the bot sees their names and loads one when it needs it.
// - Templates start a new bot with a ready role and instructions.
use serde::{Deserialize, Serialize};
/// Most memories a bot keeps; the oldest go first (estimated).
pub const MAX_MEMORIES: usize = 200;
/// Longest memory (estimated).
pub const MAX_MEMORY_CHARS: usize = 500;
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq)]
pub struct Skill {
pub name: String,
pub description: String,
pub instructions: String,
/// The steps that worked, to do again as they were (replay_skill): no model in between,
/// each control found again by what it is, the first mismatch hands back to the bot.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub steps: Vec<ReplayStep>,
}
/// One step of a skill to do again: the tool and its arguments, and for a step that acted on a
/// numbered control, what that control was (「按鈕「Close」」), since numbers change from look to
/// look. `targets` has one entry per action of a computer_use_batch ("" where none).
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Default)]
pub struct ReplayStep {
pub tool: String,
pub args: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub targets: Vec<String>,
}
pub struct Template {
pub name: &'static str,
pub description: &'static str,
pub instructions: &'static str,
pub shape: &'static str,
pub color: &'static str,
}
pub const TEMPLATES: [Template; 5] = [
Template { name: "研究助理", description: "上網找資料、整理重點、附上來源", instructions: "先釐清問題,再用 web_fetch 或瀏覽器找至少兩個可靠來源。回答先給結論,再列重點,每個重點附來源網址。不確定的地方直接說不確定。", shape: "circle", color: "green" },
Template { name: "寫作夥伴", description: "幫你構思、改寫和潤稿", instructions: "先問清楚讀者和語氣。改稿時保留作者的意思,列出改了什麼和為什麼。用繁體中文、台灣用語。", shape: "cloud", color: "purple" },
Template { name: "電腦助手", description: "在自己的電腦上跑指令、整理檔案、操作網頁", instructions: "預設在你自己的電腦上做事。動手前先看清楚(list_dir、read_file、截圖),每一步確認結果。要動到使用者的 Mac 時先說明要做什麼。", shape: "gem", color: "blue" },
Template { name: "客服助理", description: "在 Slack、Discord、Telegram 或 Teams 回覆客人", instructions: "回答簡短、有禮貌。不確定的事不要編,說會請同事確認。不要透露內部資料或使用者的私人資訊。", shape: "pill", color: "orange" },
Template { name: "開發助手", description: "看程式、跑測試、追 GitHub 上的事", instructions: "改程式前先讀相關檔案和測試。每次修改後跑測試,回報結果。PR 或 CI 的事件來時,先看紀錄再下結論。", shape: "hexagon", color: "black" },
];
/// Add a memory (no duplicates; the oldest drop off past the limit).
pub fn remember(list: &mut Vec<String>, fact: &str) -> Result<String, String> {
let f: String = fact.trim().chars().take(MAX_MEMORY_CHARS).collect();
if f.is_empty() {
return Err("要記住什麼?".into());
}
if list.iter().any(|m| m == &f) {
return Ok(format!("已經記得:{f}"));
}
list.push(f.clone());
let n = list.len();
if n > MAX_MEMORIES {
list.drain(..n - MAX_MEMORIES);
}
Ok(format!("記住了:{f}"))
}
/// Forget by number (1-based) or by text it contains.
pub fn forget(list: &mut Vec<String>, what: &str) -> Result<String, String> {
let w = what.trim();
let idx = match w.parse::<usize>() {
Ok(n) if n >= 1 && n <= list.len() => Some(n - 1),
_ => list.iter().position(|m| !w.is_empty() && m.contains(w)),
};
match idx {
Some(i) => Ok(format!("忘記了:{}", list.remove(i))),
None => Err(format!("沒有記得「{w}」這件事")),
}
}
/// Add a skill, or replace the one with the same name. True when it is new.
pub fn upsert_skill(list: &mut Vec<Skill>, k: Skill) -> bool {
match list.iter_mut().find(|x| x.name == k.name) {
Some(x) => {
*x = k;
false
}
None => {
list.push(k);
true
}
}
}
// ---------- 自己進化: a job's way that worked becomes a skill ----------
// The bot forgot everything between chats: a post with a picture on Threads took it 39 steps,
// and the next one would have taken as many. After a longer job the model looks back at the
// steps and writes down the short path that worked (or improves the skill it followed); the
// next time it reads that first.
/// Steps on computers and pages a job needs before it is worth looking back on (estimated).
pub const LEARN_MIN_STEPS: usize = 5;
/// Longest learned skill (estimated; a skill is read whole before each such job).
const MAX_SKILL_CHARS: usize = 4000;
/// A turn that did enough real work, and didn't already save a skill itself.
pub fn worth_learning(turn: &[crate::model::Message]) -> bool {
let calls: Vec<&str> = turn.iter().flat_map(|m| m.tool_calls.iter().map(|c| c.name.as_str())).collect();
!calls.contains(&"save_skill") && calls.iter().filter(|n| !crate::appctl::is_app_tool(n) && **n != "react").count() >= LEARN_MIN_STEPS
}
pub const LEARN_SYSTEM: &str = "你在幫一個會操作電腦的 Bot 整理經驗,讓它下次做同樣的事更快、不再犯同樣的錯。下面是它剛做完的一件事:使用者的要求、每一步和結果、最後的回覆,還有它已經有的技能。
先判斷要不要存。使用者希望它從做過的事學會、下次不用重新摸索:只要這件事是在某個網站或程式上動手操作(填表、發文、回留言、按讚、上傳下載、改設定、跑一套固定的檢查),就要存,就算這次只做了一次。只有閒聊、單純回答問題、只是查資料讀內容,才不用存。這次沒做成功的,不要存成做法;但如果已經有這件事的技能,可以把「這樣做行不通」補進去。
要存的話,寫一份「下次照著做就會成功」的技能:
- 寫這次真的成功的那條最短路徑,一步一行,寫出用哪個工具、在哪台電腦、哪個網址或程式(每一步結果開頭 [ ] 裡的名字,就是那一步用的電腦)。
- 繞路和失敗的嘗試不要寫成步驟;把「哪裡會卡、要怎麼避開」寫成「注意:」。
- 用畫面上看得到的名字指東西(例如 browser_click 的 text 寫「Attach media」、browser_type 的 text 寫欄位名「Customer name」),不要寫元素編號或座標,那些每次都會變。browser_open 的結果就有這一頁的元素清單,不用另外截圖。
- 寫通用的做法,不要寫這次的內容(這篇文的文字、這張圖的檔名);使用者表達過的偏好(語氣、格式、一定要附圖、用哪台電腦)要寫進去。
- 已經有做同一件事的技能,就用同一個名稱,把舊的和這次學到的合併成更好的一版:舊的對的保留,錯的改掉。這次照著技能做而且很順,就不用再存。
只回一個 JSON,不要別的字:
不用存:{\"save\": false, \"why\": \"一句話說為什麼\"}
要存:{\"save\": true, \"name\": \"短名稱\", \"description\": \"一行:什麼時候用\", \"instructions\": \"做法\", \"replay\": [步驟編號]}
replay:下次做同一件事時可以原樣照做的步驟編號(上面清單的號碼,照順序)。只列這次真的成功、必要、會改變東西的步驟(不含截圖、讀取這類只是看的步驟,不含失敗或繞路的步驟);內容每次會不一樣的工作(發文的文字、要回覆的留言、檔名)不要列,給空陣列 []。";
fn cut(s: &str, n: usize) -> String {
let flat = s.split_whitespace().collect::<Vec<_>>().join(" ");
if flat.chars().count() <= n { flat } else { format!("{}…", flat.chars().take(n).collect::<String>()) }
}
/// What the model looks back on: the request, each step with its result, the answer and the
/// skills there are.
/// The numbered steps the model looks back on, each with the call it was (None for words the
/// person typed in between, and for the "…省略…" line).
fn brief_steps(turn: &[crate::model::Message]) -> Vec<(String, Option<crate::model::ToolCall>)> {
use crate::model::Role;
let mut steps: Vec<(String, Option<crate::model::ToolCall>)> = vec![];
for (i, m) in turn.iter().enumerate() {
if m.role == Role::User && i > 0 {
steps.push((format!("(使用者中途說:{})", cut(&m.text, 300)), None));
}
for c in &m.tool_calls {
let result = turn.iter().find(|r| r.role == Role::Tool && r.tool_call_id == c.id).map(|r| r.text.as_str()).unwrap_or("");
let failed = if c.status == crate::model::ToolStatus::Failed { "(失敗)" } else { "" };
steps.push((format!("{} {} → {}{failed}", c.name, cut(&c.args, 160), cut(result, 220)), Some(c.clone())));
}
}
const KEEP: usize = 40;
if steps.len() > KEEP * 2 {
let gone = steps.len() - KEEP * 2;
steps.splice(KEEP..steps.len() - KEEP, [(format!("…(中間省略 {gone} 步)…"), None)]);
}
steps
}
pub fn learn_brief(turn: &[crate::model::Message], skills: &[Skill]) -> String {
let steps: Vec<String> = brief_steps(turn).into_iter().map(|(s, _)| s).collect();
let ask = turn.first().map(|m| cut(&m.text, 800)).unwrap_or_default();
let answer = turn.iter().rev().find(|m| m.role == crate::model::Role::Assistant && !m.text.trim().is_empty()).map(|m| cut(&m.text, 600)).unwrap_or_default();
let mut s = format!("使用者的要求:\n{ask}\n\n每一步(工具 參數 → 結果):\n");
for (i, st) in steps.iter().enumerate() {
s.push_str(&format!("{}. {st}\n", i + 1));
}
s.push_str(&format!("\n最後的回覆:\n{answer}\n\n已經有的技能:"));
if skills.is_empty() {
s.push_str("(沒有)");
}
let mut room = 8000usize;
for k in skills {
let body = cut(&k.instructions, 2000.min(room));
room = room.saturating_sub(body.chars().count());
s.push_str(&format!("\n- {}:{}\n 做法:{body}", k.name, k.description));
}
s
}
/// The skill in the model's answer, when it chose to save one.
pub fn learned_skill(text: &str) -> Option<Skill> {
let t = text.trim();
let json = t.find('{').and_then(|a| t.rfind('}').filter(|b| *b >= a).map(|b| &t[a..=b]))?;
let v: serde_json::Value = serde_json::from_str(json).ok()?;
if v["save"].as_bool() != Some(true) {
return None;
}
let name: String = v["name"].as_str()?.trim().chars().take(40).collect();
let instructions: String = v["instructions"].as_str()?.trim().chars().take(MAX_SKILL_CHARS).collect();
if name.is_empty() || instructions.is_empty() {
return None;
}
Some(Skill { name, description: v["description"].as_str().unwrap_or("").trim().chars().take(120).collect(), instructions, steps: vec![] })
}
/// The step numbers the model picked to do again as they were (its "replay").
pub fn learned_replay(text: &str) -> Vec<usize> {
let t = text.trim();
let Some(json) = t.find('{').and_then(|a| t.rfind('}').filter(|b| *b >= a).map(|b| &t[a..=b])) else { return vec![] };
let v: serde_json::Value = serde_json::from_str(json).unwrap_or_default();
v["replay"].as_array().map(|a| a.iter().filter_map(|n| n.as_u64().map(|n| n as usize)).collect()).unwrap_or_default()
}
/// The control a numbered step acted on, in words, from the newest list before it in the turn
/// ("[12] 按鈕「Close」" → 按鈕「Close」).
fn control_named(turn: &[crate::model::Message], before_call: &str, r: &str) -> String {
use crate::model::Role;
let n = r.trim().trim_matches(['[', ']', '#', 'e']);
if n.is_empty() {
return String::new();
}
let upto = turn.iter().position(|m| m.tool_calls.iter().any(|c| c.id == before_call)).unwrap_or(turn.len());
let prefix = format!("[{n}] ");
turn[..upto].iter().rev().filter(|m| m.role == Role::Tool).find_map(|m| m.text.lines().find_map(|l| l.trim_start().strip_prefix(&prefix).map(|d| d.trim().to_string()))).unwrap_or_default()
}
/// The skill's steps from the model's picks: each step's call, with the controls it acted on by
/// number written down as what they were. Steps that only look are left out whatever was picked.
pub fn replay_steps(turn: &[crate::model::Message], picks: &[usize]) -> Vec<ReplayStep> {
let steps = brief_steps(turn);
let mut out = vec![];
for &k in picks {
let Some((_, Some(c))) = k.checked_sub(1).and_then(|i| steps.get(i)) else { continue };
if c.status == crate::model::ToolStatus::Failed || matches!(c.name.as_str(), "computer_use_screenshot" | "computer_use_zoom" | "browser_read" | "browser_screenshot" | "browser_elements" | "use_skill" | "replay_skill") {
continue;
}
let v: serde_json::Value = serde_json::from_str(&c.args).unwrap_or_default();
let targets: Vec<String> = if c.name == "computer_use_batch" {
v["actions"].as_array().map(|a| a.iter().map(|x| x["ref"].as_str().map(|r| control_named(turn, &c.id, r)).unwrap_or_default()).collect()).unwrap_or_default()
} else {
match v["ref"].as_str().map(String::from).or_else(|| v["ref"].as_u64().map(|n| n.to_string())) {
Some(r) => vec![control_named(turn, &c.id, &r)],
None => vec![],
}
};
// A numbered step whose control can't be named can't be found again: not a step to replay.
if targets.iter().zip(v["actions"].as_array().map(|a| a.iter().map(|x| !x["ref"].is_null()).collect::<Vec<_>>()).unwrap_or_else(|| vec![!v["ref"].is_null()])).any(|(t, had)| had && t.is_empty()) {
return vec![];
}
out.push(ReplayStep { tool: c.name.clone(), args: c.args.clone(), targets: if targets.iter().all(String::is_empty) { vec![] } else { targets } });
}
out
}
/// The line under the bot's reply that says it learned (or improved) a skill.
pub fn learned_note(name: &str, fresh: bool) -> String {
if fresh {
format!("\n\n(學會了新做法「{name}」,下次做這件事會照著做。)")
} else {
format!("\n\n(把做法「{name}」更新了,下次會照新的做。)")
}
}
pub fn prompt_section(memories: &[String], skills: &[Skill]) -> String {
let mut s = String::new();
if !memories.is_empty() {
s.push_str("\n\n你記得的事(使用者要你記住的;用 remember 新增、forget 刪除):\n");
for (i, m) in memories.iter().enumerate() {
s.push_str(&format!("{}. {m}\n", i + 1));
}
} else {
s.push_str("\n\n使用者說「記住…」「以後都…」時,用 remember 記下來;說「忘記…」時用 forget。");
}
if !skills.is_empty() {
s.push_str("\n\n你會的技能(以前做成功的做法)。開始做一件事之前先看這裡:有符合的,先用 use_skill 讀出完整做法再照著做,不要重新摸索:\n");
for k in skills {
let replay = if k.steps.is_empty() { String::new() } else { format!("(可重播,{} 步)", k.steps.len()) };
s.push_str(&format!("- {}:{}{replay}\n", k.name, k.description));
}
if skills.iter().any(|k| !k.steps.is_empty()) {
s.push_str("標著「可重播」的技能:要做的事和技能一模一樣(同一個網站或程式、同樣的設定,沒有每次不同的內容)時,直接用 replay_skill,AnyBot 會照存下的步驟一步一步做,每一步先在畫面上找到同一個東西,對不上就停下來交給你;有一點不同就照做法自己做。\n");
}
}
s.push_str("\n使用者教你一套可以重複用的做法時,可以用 save_skill 存成技能,所有 Bot 都能用。");
s
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_steps_that_worked_are_kept_with_what_they_acted_on() {
use crate::model::{Message, Role, ToolCall, ToolStatus};
let call = |id: &str, name: &str, args: &str| Message { role: Role::Assistant, tool_calls: vec![ToolCall { id: id.into(), name: name.into(), args: args.into(), status: ToolStatus::Done }], ..Default::default() };
let turn = vec![
Message::user("把自動更新打開"),
call("a", "computer_use_screenshot", "{}"),
Message::tool_result("a", "computer_use_screenshot", "螢幕截圖。\n[3] 按鈕「Close」\n[5] 勾選框「自動更新」:0"),
call("b", "computer_use_click", r#"{"ref":"5"}"#),
Message::tool_result("b", "computer_use_click", "已點 #5(勾選框「自動更新」:0)。"),
call("c", "shell", r#"{"command":"defaults read x"}"#),
Message::tool_result("c", "shell", "1"),
];
let brief = learn_brief(&turn, &[]);
assert!(brief.contains("1. computer_use_screenshot") && brief.contains("2. computer_use_click") && brief.contains("3. shell"), "{brief}");
let steps = replay_steps(&turn, &[1, 2, 3]);
// The look is left out; the click remembers what it clicked.
assert_eq!(steps.len(), 2);
assert_eq!(steps[0].tool, "computer_use_click");
assert_eq!(steps[0].targets, vec!["勾選框「自動更新」:0".to_string()]);
assert!(steps[1].targets.is_empty());
assert_eq!(learned_replay(r#"{"save": true, "name": "x", "instructions": "y", "replay": [2, 3]}"#), vec![2, 3]);
assert!(learned_replay(r#"{"save": true, "name": "x", "instructions": "y"}"#).is_empty());
// A numbered step whose control can't be named: nothing to replay safely.
assert!(replay_steps(&turn[..1].iter().cloned().chain([call("d", "computer_use_click", r#"{"ref":"77"}"#)]).collect::<Vec<_>>(), &[1]).is_empty());
}
#[test]
fn remember_and_forget() {
let mut m = vec![];
assert!(remember(&mut m, "我喜歡靠窗的座位").unwrap().starts_with("記住了"));
assert!(remember(&mut m, "我喜歡靠窗的座位").unwrap().starts_with("已經記得"));
remember(&mut m, "週五不開會").unwrap();
assert_eq!(forget(&mut m, "1").unwrap(), "忘記了:我喜歡靠窗的座位");
assert!(forget(&mut m, "不存在").is_err());
assert_eq!(forget(&mut m, "週五").unwrap(), "忘記了:週五不開會");
for i in 0..(MAX_MEMORIES + 5) {
remember(&mut m, &format!("事 {i}")).unwrap();
}
assert_eq!(m.len(), MAX_MEMORIES);
assert_eq!(m[0], "事 5");
let p = prompt_section(&m[..1], &[Skill { name: "週報".into(), description: "整理每週進度".into(), instructions: "…".into(), steps: vec![] }]);
assert!(p.contains("1. 事 5") && p.contains("- 週報:整理每週進度") && p.contains("use_skill"));
}
fn call(id: &str, name: &str, args: &str, failed: bool) -> crate::model::ToolCall {
crate::model::ToolCall { id: id.into(), name: name.into(), args: args.into(), status: if failed { crate::model::ToolStatus::Failed } else { crate::model::ToolStatus::Done } }
}
fn result(id: &str, text: &str) -> crate::model::Message {
crate::model::Message { role: crate::model::Role::Tool, tool_call_id: id.into(), text: text.into(), ..Default::default() }
}
/// A post with a picture: five real steps, one of them failed, the person corrects it midway.
fn post_turn() -> Vec<crate::model::Message> {
use crate::model::Message;
let step = |c: crate::model::ToolCall| Message { tool_calls: vec![c], ..Message::assistant("", "b") };
vec![
Message::user("幫我發一篇 Threads,記得附圖"),
step(call("1", "browser_open", r#"{"url":"https://www.threads.com/"}"#, false)),
result("1", "已打開 https://www.threads.com/\n[13] 按鈕「Create」"),
step(call("2", "browser_click", r#"{"ref":"13"}"#, false)),
result("2", "已點 [13]。新出現的文字:New thread"),
Message { steer: true, ..Message::user("語氣要輕鬆一點") },
step(call("3", "browser_type", r#"{"ref":"12","value":"今天天氣好"}"#, false)),
result("3", "已在 [12] 輸入。"),
step(call("4", "browser_upload", r#"{"path":"/home/bot/x.jpg"}"#, true)),
result("4", "這台電腦上沒有這個檔案"),
step(call("5", "browser_upload", r#"{"path":"/home/bot/bot-6/附件/x.jpg"}"#, false)),
result("5", "已把 /home/bot/bot-6/附件/x.jpg 放進上傳欄位。"),
step(call("6", "remember", r#"{"fact":"x"}"#, false)),
result("6", "記住了"),
Message::assistant("發好了:https://www.threads.com/@me/post/1", "b"),
]
}
#[test]
fn a_long_enough_job_is_looked_back_on() {
let turn = post_turn();
assert!(worth_learning(&turn));
// App steps (remember) don't count; four real steps aren't enough.
assert!(!worth_learning(&turn[..10]));
// A job that saved a skill itself is left alone.
let mut saved = turn.clone();
saved[12].tool_calls[0].name = "save_skill".into();
assert!(!worth_learning(&saved));
}
#[test]
fn the_look_back_has_every_step_and_the_skills() {
let skills = [Skill { name: "Threads 發文".into(), description: "在 Threads 發文".into(), instructions: "1. browser_open https://www.threads.com/".into(), steps: vec![] }];
let b = learn_brief(&post_turn(), &skills);
assert!(b.starts_with("使用者的要求:\n幫我發一篇 Threads,記得附圖"), "{b}");
assert!(b.contains("1. browser_open {\"url\":\"https://www.threads.com/\"} → 已打開 https://www.threads.com/ [13] 按鈕「Create」"), "{b}");
assert!(b.contains("(使用者中途說:語氣要輕鬆一點)"), "{b}");
assert!(b.contains("browser_upload {\"path\":\"/home/bot/x.jpg\"} → 這台電腦上沒有這個檔案(失敗)"), "{b}");
assert!(b.contains("最後的回覆:\n發好了"), "{b}");
assert!(b.contains("- Threads 發文:在 Threads 發文\n 做法:1. browser_open"), "{b}");
// A very long job keeps its start and its end.
let mut long = vec![crate::model::Message::user("做很多事")];
for i in 0..200 {
long.push(crate::model::Message { tool_calls: vec![call(&i.to_string(), "browser_click", "{}", false)], ..crate::model::Message::assistant("", "b") });
}
let b = learn_brief(&long, &[]);
assert!(b.contains("(中間省略 120 步)") && b.contains("81. browser_click") && !b.contains("82. "), "{b}");
}
#[test]
fn a_learned_skill_is_read_from_the_answer() {
let k = learned_skill("好的\n{\"save\": true, \"name\": \" Threads 發文附圖 \", \"description\": \"在 Threads 發一篇附圖的文\", \"instructions\": \"1. browser_open https://www.threads.com/\"}").unwrap();
assert_eq!((k.name.as_str(), k.description.as_str()), ("Threads 發文附圖", "在 Threads 發一篇附圖的文"));
assert!(learned_skill(r#"{"save": false}"#).is_none());
assert!(learned_skill(r#"{"save": true, "name": "", "instructions": "x"}"#).is_none());
assert!(learned_skill("不用存").is_none());
assert!(learned_skill("} {").is_none());
let mut list = vec![];
assert!(upsert_skill(&mut list, k.clone()));
assert!(!upsert_skill(&mut list, Skill { instructions: "新的".into(), ..k }));
assert_eq!((list.len(), list[0].instructions.as_str()), (1, "新的"));
assert!(learned_note("Threads 發文附圖", true).contains("學會了新做法「Threads 發文附圖」"));
}
}