From bbd677b0503d8df3f70b4ebd8962a590f0ed8688 Mon Sep 17 00:00:00 2001 From: AnyBot Dev Date: Mon, 9 Nov 2026 18:15:24 +0800 Subject: [PATCH] Memory, searched not carried: FTS index + event log (P6, step 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - store.rs grows a memories table + FTS5 (trigram) index: bot facts and the person's shared lines (bot 0), no cap — state.json's 200-item list stays the source that mirrors sync and the 設定 list; the index is its searchable superset, best-effort as always - queries answer in Chinese: phrase hits (FTS), then words inside lines — CJK queries cut into two-character windows, lines scored by how many they contain (「有没有靠窗」 finds 「我喜歡靠窗的座位」) - every finished turn writes one event line (ask → answer) nobody's asked to write: 「我上個月說的那個報表」 answers from months back - the turn's prompt carries the 14 memories that answer *this* request (no index yet: the list itself, the old behavior); remember/forget and the UI's 忘記 keep the index in step with the list - embedding (Ollama) and document indexing land on this same table next --- src/agent.rs | 17 ++++++ src/app.rs | 12 ++-- src/appctl.rs | 14 ++++- src/memory.rs | 27 ++++++++- src/store.rs | 165 +++++++++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 227 insertions(+), 8 deletions(-) diff --git a/src/agent.rs b/src/agent.rs index 85f0d56..4a566f1 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -179,6 +179,22 @@ fn parse_verdict(text: &str) -> ReviewVerdict { /// 自己進化: after a longer job, the model looks back on its steps and keeps the way that worked /// as a skill (or improves the one it followed), so the next time takes the short path. It runs /// after Done: the person isn't kept waiting. +/// The event log (P6): one line per finished turn — who asked what, what came out — written +/// without anyone's asking, so "我上個月說的那個報表" answers from the memory index months later. +fn memory_note(job: &Job, history: &[Message]) { + let Some(start) = history.iter().rposition(|m| m.role == Role::User && !m.steer) else { return }; + let ask = history[start].text.trim(); + if ask.is_empty() { + return; + } + let cut = |s: &str, n: usize| -> String { s.split_whitespace().collect::>().join(" ").chars().take(n).collect() }; + let ans = history.iter().rev().find(|m| m.role == Role::Assistant && !m.text.trim().is_empty()).map(|m| cut(&m.text, 120)).unwrap_or_default(); + if let Some(bot) = job.req.session.strip_prefix("anybot-bot-").and_then(|s| s.parse::().ok()) { + let day = chrono::Local::now().format("%m-%d"); + crate::store::memory_event(bot, &format!("{day}：{} → {ans}", cut(ask, 100))); + } +} + fn learn(job: &Job, history: &[Message], cancel: &AtomicBool, send: &dyn Fn(StreamEvent)) { let Some(start) = history.iter().rposition(|m| m.role == Role::User && !m.steer) else { return }; let turn = &history[start..]; @@ -1099,6 +1115,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc< closing.close(crate::oplog::Outcome::Completed); send(StreamEvent::Done); browsers.close_tabs(); + memory_note(&job, &history); learn(&job, &history, &cancel, &send); return; } diff --git a/src/app.rs b/src/app.rs index 7eab7d8..c14472b 100644 --- a/src/app.rs +++ b/src/app.rs @@ -843,7 +843,10 @@ impl App { api_key: providers::resolve_key(b.provider, &ps.api_key), model: b.model.clone(), system: { - let mut sys = system_prompt(&b, &self.state, &computers, &default_computer, &roles, missing_computer.as_deref(), &self.computer_lines(&computers, &roles), mac_apps); + // P6: not the whole shelf — the memories that answer this request ride along. + let ask = conv.messages.iter().rev().find(|m| m.role == crate::model::Role::User && !m.text.trim().is_empty()).map(|m| m.text.clone()).unwrap_or_default(); + let mems = crate::memory::recall(b.id, &ask, &b.memories, 14); + let mut sys = system_prompt(&b, &self.state, &computers, &default_computer, &roles, missing_computer.as_deref(), &self.computer_lines(&computers, &roles), mac_apps, &mems); if speaker.is_some() { let names: Vec = conv.members.iter().filter_map(|m| self.state.bot(*m)).map(|x| x.name.clone()).collect(); sys.push_str(&format!("\n\n你在群組聊天「{}」裡，成員依序回答：{}。其他成員的發言以【名字】標示。只回答你擅長的部分，補充不同的觀點，不要重複別人說過的；沒有要補充就簡短說一句。", conv.name, names.join("、"))); @@ -3116,7 +3119,8 @@ impl App { }); } if let Some(i) = remove { - b.memories.remove(i); + let gone = b.memories.remove(i); + crate::store::memory_del(b.id, &gone); } }); }); @@ -4181,7 +4185,7 @@ fn transcript_markdown(b: &Bot) -> String { /// What the bot is told about itself, the app it lives in and the computers it can use. #[allow(clippy::too_many_arguments)] -fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_computer: &str, roles: &tools::Roles, missing: Option<&str>, lines: &[String], mac_apps: bool) -> String { +fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_computer: &str, roles: &tools::Roles, missing: Option<&str>, lines: &[String], mac_apps: bool, memories: &[String]) -> String { let mut s = format!("你是「{}」。", b.name); if !b.description.trim().is_empty() { s.push_str(&format!("你的角色：{}。", b.description.trim())); @@ -4275,7 +4279,7 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu if !chans.is_empty() { s.push_str(&format!("\n\n你也在這些訊息管道回覆別人：{}。以「［訊息管道］」開頭的訊息來自那裡，你這一回合最後的回答會自動傳回給對方，所以要簡潔、直接回答對方，不要用表格，也不要提到 AnyBot 的內部操作。對方不是使用者本人：不要透露使用者的私人資料，要動到使用者的電腦或帳號前先確認。", chans.join("、"))); } - s.push_str(&crate::memory::prompt_section(&b.memories, &st.skills)); + s.push_str(&crate::memory::prompt_section(memories, &st.skills)); let mcp_notes = crate::mcp::instructions(&st.mcp_servers); let mcp_tools = crate::mcp::status(); if !mcp_tools.is_empty() { diff --git a/src/appctl.rs b/src/appctl.rs index 7d10b53..8ae06ad 100644 --- a/src/appctl.rs +++ b/src/appctl.rs @@ -412,14 +412,24 @@ pub fn apply(state: &mut State, bot_id: u64, name: &str, args: &str) -> Outcome "remember" => { let Some(b) = state.bot_mut(bot_id) else { return err("找不到這個 Bot".into()) }; match crate::memory::remember(&mut b.memories, &s("fact").unwrap_or_default()) { - Ok(t) => ok(t), + Ok(t) => { + if let Some(f) = t.strip_prefix("記住了：").or_else(|| t.strip_prefix("已經記得：")) { + crate::store::memory_add(bot_id, f); + } + ok(t) + } Err(e) => err(e), } } "forget" => { let Some(b) = state.bot_mut(bot_id) else { return err("找不到這個 Bot".into()) }; match crate::memory::forget(&mut b.memories, &s("what").unwrap_or_default()) { - Ok(t) => ok(t), + Ok(t) => { + if let Some(f) = t.strip_prefix("忘記了：") { + crate::store::memory_del(bot_id, f); + } + ok(t) + } Err(e) => err(e), } } diff --git a/src/memory.rs b/src/memory.rs index 1457e8a..3f9934b 100644 --- a/src/memory.rs +++ b/src/memory.rs @@ -251,10 +251,23 @@ pub fn learned_note(name: &str, fresh: bool) -> String { } } +/// What to carry into a turn's prompt (P6): the memories that answer *this* request, from the +/// SQLite index (FTS + LIKE, the person's shared preferences included), not the whole shelf. +/// No index (fresh install, no database): the newest of the list, like the old days. +pub fn recall(bot: u64, query: &str, list: &[String], limit: usize) -> Vec { + crate::store::memory_index(bot, list); + let hits = crate::store::recall(bot, query, limit); + if hits.is_empty() { + let n = list.len().min(20); + return list[list.len() - n..].to_vec(); + } + hits +} + pub fn prompt_section(memories: &[String], skills: &[Skill]) -> String { let mut s = String::new(); if !memories.is_empty() { - s.push_str("\n\n你記得的事（使用者要你記住的；用 remember 新增、forget 刪除）：\n"); + s.push_str("\n\n你記得的事（和這件事最相關的幾條；完整清單在「設定 › 記得的事」，用 remember 新增、forget 刪除）：\n"); for (i, m) in memories.iter().enumerate() { s.push_str(&format!("{}. {m}\n", i + 1)); } @@ -279,6 +292,18 @@ pub fn prompt_section(memories: &[String], skills: &[Skill]) -> String { mod tests { use super::*; + #[test] + fn the_shelf_is_searched_not_carried_whole() { + let list = vec!["我喜歡靠窗的座位".to_string(), "週五下午不開會".to_string()]; + // A bot that has heard of 座位 gets just the line that answers. + let r = recall(4244, "有没有靠窗", &list, 10); + assert!(r.contains(&list[0]) && !r.contains(&list[1]), "{r:?}"); + // No index to answer (fresh bot): the list itself, like the old days. + assert_eq!(recall(5556, "完全不相干的查詢", &[], 10), Vec::::new()); + let r = recall(5557, "完全不相干的查詢", &list, 10); + assert_eq!(r, list, "fallback carries the list"); + } + #[test] fn the_steps_that_worked_are_kept_with_what_they_acted_on() { use crate::model::{Message, Role, ToolCall, ToolStatus}; diff --git a/src/store.rs b/src/store.rs index f0395b0..ea29af8 100644 --- a/src/store.rs +++ b/src/store.rs @@ -53,7 +53,24 @@ fn open() -> Option { call_id TEXT ); CREATE INDEX IF NOT EXISTS audit_bot_at ON audit(bot, at DESC); - CREATE INDEX IF NOT EXISTS audit_call ON audit(bot, op, call_id);", + CREATE INDEX IF NOT EXISTS audit_call ON audit(bot, op, call_id); + + CREATE TABLE IF NOT EXISTS memories( + id INTEGER PRIMARY KEY, + bot INTEGER NOT NULL, + text TEXT NOT NULL, + kind TEXT NOT NULL DEFAULT 'fact', + created INTEGER NOT NULL, + UNIQUE(bot, text) + ); + CREATE INDEX IF NOT EXISTS memories_bot ON memories(bot, created DESC); + CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(text, content='memories', content_rowid='id', tokenize='trigram'); + CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN + INSERT INTO memories_fts(rowid, text) VALUES (new.id, new.text); + END; + CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN + INSERT INTO memories_fts(memories_fts, rowid, text) VALUES ('delete', old.id, old.text); + END;", ) .ok()?; let cutoff = crate::model::now_ms() - KEEP_DAYS * 86_400_000; @@ -211,6 +228,29 @@ mod tests { use super::*; use crate::oplog::{Prompt, Replay}; + #[test] + fn the_memory_index_answers_in_chinese() { + // (One database per test process, one bot id: 4243 is ours here.) + let bot = 4243u64; + memory_add(bot, "我喜歡靠窗的座位，長途航班也是"); + memory_add(bot, "週五下午不開會"); + memory_add(9999, "別的 Bot 的事：預算 42 萬"); + memory_event(bot, "3-8：幫我排週五的會 → 已排在 15:00"); + let hits = recall(bot, "靠窗的座位", 10); + assert!(hits.iter().any(|m| m.contains("靠窗")), "{hits:?}"); + // A two-word query finds lines that merely contain the words (LIKE path). + let hits = recall(bot, "週五", 10); + assert!(hits.iter().any(|m| m.contains("週五下午")), "{hits:?}"); + // Other bots' memories stay theirs. + assert!(!recall(bot, "預算 42 萬", 10).iter().any(|m| m.contains("預算")), "leaked another bot's memory"); + // Events answer the "上個月說過的那個" questions. + let hits = recall(bot, "排週五的會", 10); + assert!(hits.iter().any(|m| m.contains("已排在 15:00")), "{hits:?}"); + // Forgetting leaves the index with the list. + memory_del(bot, "週五下午不開會"); + assert!(!recall(bot, "週五下午不開會", 3).contains(&"週五下午不開會".to_string())); + } + #[test] fn the_trail_keeps_the_story_after_the_turn_log_cleans_up() { // (Tests share the temp dir of this process; one database, one bot id. @@ -244,3 +284,126 @@ mod tests { assert_eq!(ts[1]["steps"][2]["state"], "consumed"); } } + +// ---------- 記憶的索引（P6）---------- +// `state.json`'s bot.memories stays the source that mirrors sync between devices, and the +// list the person manages in 設定 › 記得的事. The index is its searchable side: a superset +// (the JSON caps at 200, the index does not) plus *events* — one line per finished turn, +// nobody's asked to write them — so "我上個月說的那個…" answers across months. +// Best-effort in, best-effort out: with no index the caller falls back to the JSON list. + +/// Index what a bot already remembers (startup / first use; INSERT OR IGNORE, cheap). +pub fn memory_index(bot: u64, texts: &[String]) { + let Some(s) = store() else { return }; + let Ok(c) = s.conn.lock() else { return }; + let now = crate::model::now_ms(); + for t in texts { + let _ = c.execute("INSERT OR IGNORE INTO memories(bot,text,kind,created) VALUES(?1,?2,'fact',?3)", params![bot as i64, t, now]); + } +} + +/// One new memory (the model's `remember`, or the person's 記住…). +pub fn memory_add(bot: u64, text: &str) { + let Some(s) = store() else { return }; + let Ok(c) = s.conn.lock() else { return }; + let _ = c.execute("INSERT OR IGNORE INTO memories(bot,text,kind,created) VALUES(?1,?2,'fact',?3)", params![bot as i64, text, crate::model::now_ms()]); +} + +/// A forgotten memory leaves the index with the list. +pub fn memory_del(bot: u64, text: &str) { + let Some(s) = store() else { return }; + let Ok(c) = s.conn.lock() else { return }; + let _ = c.execute("DELETE FROM memories WHERE bot=?1 AND text=?2", params![bot as i64, text]); +} + +/// One finished turn as a line of the event log: 「3-8：幫我發一篇 Threads… → 發好了…」. +pub fn memory_event(bot: u64, text: &str) { + if text.trim().is_empty() { + return; + } + let Some(s) = store() else { return }; + let Ok(c) = s.conn.lock() else { return }; + let _ = c.execute("INSERT INTO memories(bot,text,kind,created) VALUES(?1,?2,'event',?3)", params![bot as i64, text, crate::model::now_ms()]); +} + +/// The memories that answer a request: phrase hits first (FTS trigram, good with Chinese), +/// words inside lines next (LIKE, for short queries), newest facts last. The person's own +/// shared preferences ride along (bot 0). Empty means "no index" — the caller decides. +pub fn recall(bot: u64, query: &str, limit: usize) -> Vec { + let mut out: Vec = vec![]; + let Some(s) = store() else { return out }; + let Ok(c) = s.conn.lock() else { return out }; + let q: String = query.chars().filter(|ch| *ch != '"').collect::().split_whitespace().collect::>().join(" "); + let bot = bot as i64; + if q.chars().count() >= 3 { + // Trigram wants at least three characters; a quoted phrase keeps it literal. + let phrase = format!("\"{q}\""); + if let Ok(rows) = c.prepare("SELECT m.text FROM memories m JOIN memories_fts f ON f.rowid=m.id WHERE memories_fts MATCH ?1 AND m.bot IN (?2, 0) ORDER BY bm25(memories_fts) LIMIT ?3").and_then(|mut st| { + let it = st.query_map(params![phrase, bot, limit as i64], |r| r.get::<_, String>(0))?; + let mut v = vec![]; + for x in it.flatten() { + v.push(x); + } + Ok(v) + }) { + out.extend(rows); + } + } + // Words inside lines next. Chinese queries are cut into two-character windows + // ("有没有靠窗" → 有没/没有/有靠/靠窗) and lines score by how many windows they + // contain: a shelf line answering the question outranks a fresh one. + let mut segs: Vec = vec![]; + for w in q.split_whitespace() { + let ch: Vec = w.chars().collect(); + if w.is_ascii() { + if ch.len() >= 2 { + segs.push(w.to_lowercase()); + } + } else if ch.len() >= 2 { + for i in 0..ch.len() - 1 { + segs.push(format!("{}{}", ch[i], ch[i + 1])); + } + } + } + segs.sort(); + segs.dedup(); + segs.truncate(12); + if !segs.is_empty() { + let mut scored: Vec<(usize, i64, String)> = vec![]; // (windows hit, freshness, text) + for seg in &segs { + let _ = c.prepare("SELECT id, created, text FROM memories WHERE bot IN (?1, 0) AND text LIKE '%'||?2||'%' LIMIT 60").and_then(|mut st| { + let it = st.query_map(params![bot, seg], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?, r.get::<_, String>(2)?)))?; + for x in it.flatten() { + let (id, created, text) = x; + match scored.iter_mut().find(|(_, _, t)| t == &text) { + Some(e) => e.0 += 1, + None => scored.push((1, created, text)), + } + } + Ok(()) + }); + } + scored.sort_by(|a, b| b.0.cmp(&a.0).then(b.1.cmp(&a.1))); + for (_, _, text) in scored { + if !out.contains(&text) { + out.push(text); + } + } + } + if out.is_empty() { + // Nothing matched: the freshest facts, so "記住…" works in the very next chat even + // with no words to match. + let want = limit; + let _ = c.prepare("SELECT text FROM memories WHERE bot IN (?1, 0) AND kind='fact' ORDER BY created DESC LIMIT ?2").and_then(|mut st| { + let it = st.query_map(params![bot, want as i64], |r| r.get::<_, String>(0))?; + for x in it.flatten() { + if !out.contains(&x) { + out.push(x); + } + } + Ok(()) + }); + } + out.truncate(limit); + out +}