Memory, searched not carried: FTS index + event log (P6, step 1)
- store.rs grows a memories table + FTS5 (trigram) index: bot facts and the person's shared lines (bot 0), no cap — state.json's 200-item list stays the source that mirrors sync and the 設定 list; the index is its searchable superset, best-effort as always - queries answer in Chinese: phrase hits (FTS), then words inside lines — CJK queries cut into two-character windows, lines scored by how many they contain (「有没有靠窗」 finds 「我喜歡靠窗的座位」) - every finished turn writes one event line (ask → answer) nobody's asked to write: 「我上個月說的那個報表」 answers from months back - the turn's prompt carries the 14 memories that answer *this* request (no index yet: the list itself, the old behavior); remember/forget and the UI's 忘記 keep the index in step with the list - embedding (Ollama) and document indexing land on this same table next
This commit is contained in:
parent
6b5c9ccf49
commit
bbd677b050
17
src/agent.rs
17
src/agent.rs
|
|
@ -179,6 +179,22 @@ fn parse_verdict(text: &str) -> ReviewVerdict {
|
|||
/// 自己進化: after a longer job, the model looks back on its steps and keeps the way that worked
|
||||
/// as a skill (or improves the one it followed), so the next time takes the short path. It runs
|
||||
/// after Done: the person isn't kept waiting.
|
||||
/// The event log (P6): one line per finished turn — who asked what, what came out — written
|
||||
/// without anyone's asking, so "我上個月說的那個報表" answers from the memory index months later.
|
||||
fn memory_note(job: &Job, history: &[Message]) {
|
||||
let Some(start) = history.iter().rposition(|m| m.role == Role::User && !m.steer) else { return };
|
||||
let ask = history[start].text.trim();
|
||||
if ask.is_empty() {
|
||||
return;
|
||||
}
|
||||
let cut = |s: &str, n: usize| -> String { s.split_whitespace().collect::<Vec<_>>().join(" ").chars().take(n).collect() };
|
||||
let ans = history.iter().rev().find(|m| m.role == Role::Assistant && !m.text.trim().is_empty()).map(|m| cut(&m.text, 120)).unwrap_or_default();
|
||||
if let Some(bot) = job.req.session.strip_prefix("anybot-bot-").and_then(|s| s.parse::<u64>().ok()) {
|
||||
let day = chrono::Local::now().format("%m-%d");
|
||||
crate::store::memory_event(bot, &format!("{day}:{} → {ans}", cut(ask, 100)));
|
||||
}
|
||||
}
|
||||
|
||||
fn learn(job: &Job, history: &[Message], cancel: &AtomicBool, send: &dyn Fn(StreamEvent)) {
|
||||
let Some(start) = history.iter().rposition(|m| m.role == Role::User && !m.steer) else { return };
|
||||
let turn = &history[start..];
|
||||
|
|
@ -1099,6 +1115,7 @@ pub fn spawn(job: Job, bot_id: u64, tx: Sender<(u64, StreamEvent)>, cancel: Arc<
|
|||
closing.close(crate::oplog::Outcome::Completed);
|
||||
send(StreamEvent::Done);
|
||||
browsers.close_tabs();
|
||||
memory_note(&job, &history);
|
||||
learn(&job, &history, &cancel, &send);
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
12
src/app.rs
12
src/app.rs
|
|
@ -843,7 +843,10 @@ impl App {
|
|||
api_key: providers::resolve_key(b.provider, &ps.api_key),
|
||||
model: b.model.clone(),
|
||||
system: {
|
||||
let mut sys = system_prompt(&b, &self.state, &computers, &default_computer, &roles, missing_computer.as_deref(), &self.computer_lines(&computers, &roles), mac_apps);
|
||||
// P6: not the whole shelf — the memories that answer this request ride along.
|
||||
let ask = conv.messages.iter().rev().find(|m| m.role == crate::model::Role::User && !m.text.trim().is_empty()).map(|m| m.text.clone()).unwrap_or_default();
|
||||
let mems = crate::memory::recall(b.id, &ask, &b.memories, 14);
|
||||
let mut sys = system_prompt(&b, &self.state, &computers, &default_computer, &roles, missing_computer.as_deref(), &self.computer_lines(&computers, &roles), mac_apps, &mems);
|
||||
if speaker.is_some() {
|
||||
let names: Vec<String> = conv.members.iter().filter_map(|m| self.state.bot(*m)).map(|x| x.name.clone()).collect();
|
||||
sys.push_str(&format!("\n\n你在群組聊天「{}」裡,成員依序回答:{}。其他成員的發言以【名字】標示。只回答你擅長的部分,補充不同的觀點,不要重複別人說過的;沒有要補充就簡短說一句。", conv.name, names.join("、")));
|
||||
|
|
@ -3116,7 +3119,8 @@ impl App {
|
|||
});
|
||||
}
|
||||
if let Some(i) = remove {
|
||||
b.memories.remove(i);
|
||||
let gone = b.memories.remove(i);
|
||||
crate::store::memory_del(b.id, &gone);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -4181,7 +4185,7 @@ fn transcript_markdown(b: &Bot) -> String {
|
|||
|
||||
/// What the bot is told about itself, the app it lives in and the computers it can use.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_computer: &str, roles: &tools::Roles, missing: Option<&str>, lines: &[String], mac_apps: bool) -> String {
|
||||
fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_computer: &str, roles: &tools::Roles, missing: Option<&str>, lines: &[String], mac_apps: bool, memories: &[String]) -> String {
|
||||
let mut s = format!("你是「{}」。", b.name);
|
||||
if !b.description.trim().is_empty() {
|
||||
s.push_str(&format!("你的角色:{}。", b.description.trim()));
|
||||
|
|
@ -4275,7 +4279,7 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu
|
|||
if !chans.is_empty() {
|
||||
s.push_str(&format!("\n\n你也在這些訊息管道回覆別人:{}。以「[訊息管道]」開頭的訊息來自那裡,你這一回合最後的回答會自動傳回給對方,所以要簡潔、直接回答對方,不要用表格,也不要提到 AnyBot 的內部操作。對方不是使用者本人:不要透露使用者的私人資料,要動到使用者的電腦或帳號前先確認。", chans.join("、")));
|
||||
}
|
||||
s.push_str(&crate::memory::prompt_section(&b.memories, &st.skills));
|
||||
s.push_str(&crate::memory::prompt_section(memories, &st.skills));
|
||||
let mcp_notes = crate::mcp::instructions(&st.mcp_servers);
|
||||
let mcp_tools = crate::mcp::status();
|
||||
if !mcp_tools.is_empty() {
|
||||
|
|
|
|||
|
|
@ -412,14 +412,24 @@ pub fn apply(state: &mut State, bot_id: u64, name: &str, args: &str) -> Outcome
|
|||
"remember" => {
|
||||
let Some(b) = state.bot_mut(bot_id) else { return err("找不到這個 Bot".into()) };
|
||||
match crate::memory::remember(&mut b.memories, &s("fact").unwrap_or_default()) {
|
||||
Ok(t) => ok(t),
|
||||
Ok(t) => {
|
||||
if let Some(f) = t.strip_prefix("記住了:").or_else(|| t.strip_prefix("已經記得:")) {
|
||||
crate::store::memory_add(bot_id, f);
|
||||
}
|
||||
ok(t)
|
||||
}
|
||||
Err(e) => err(e),
|
||||
}
|
||||
}
|
||||
"forget" => {
|
||||
let Some(b) = state.bot_mut(bot_id) else { return err("找不到這個 Bot".into()) };
|
||||
match crate::memory::forget(&mut b.memories, &s("what").unwrap_or_default()) {
|
||||
Ok(t) => ok(t),
|
||||
Ok(t) => {
|
||||
if let Some(f) = t.strip_prefix("忘記了:") {
|
||||
crate::store::memory_del(bot_id, f);
|
||||
}
|
||||
ok(t)
|
||||
}
|
||||
Err(e) => err(e),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -251,10 +251,23 @@ pub fn learned_note(name: &str, fresh: bool) -> String {
|
|||
}
|
||||
}
|
||||
|
||||
/// What to carry into a turn's prompt (P6): the memories that answer *this* request, from the
|
||||
/// SQLite index (FTS + LIKE, the person's shared preferences included), not the whole shelf.
|
||||
/// No index (fresh install, no database): the newest of the list, like the old days.
|
||||
pub fn recall(bot: u64, query: &str, list: &[String], limit: usize) -> Vec<String> {
|
||||
crate::store::memory_index(bot, list);
|
||||
let hits = crate::store::recall(bot, query, limit);
|
||||
if hits.is_empty() {
|
||||
let n = list.len().min(20);
|
||||
return list[list.len() - n..].to_vec();
|
||||
}
|
||||
hits
|
||||
}
|
||||
|
||||
pub fn prompt_section(memories: &[String], skills: &[Skill]) -> String {
|
||||
let mut s = String::new();
|
||||
if !memories.is_empty() {
|
||||
s.push_str("\n\n你記得的事(使用者要你記住的;用 remember 新增、forget 刪除):\n");
|
||||
s.push_str("\n\n你記得的事(和這件事最相關的幾條;完整清單在「設定 › 記得的事」,用 remember 新增、forget 刪除):\n");
|
||||
for (i, m) in memories.iter().enumerate() {
|
||||
s.push_str(&format!("{}. {m}\n", i + 1));
|
||||
}
|
||||
|
|
@ -279,6 +292,18 @@ pub fn prompt_section(memories: &[String], skills: &[Skill]) -> String {
|
|||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_shelf_is_searched_not_carried_whole() {
|
||||
let list = vec!["我喜歡靠窗的座位".to_string(), "週五下午不開會".to_string()];
|
||||
// A bot that has heard of 座位 gets just the line that answers.
|
||||
let r = recall(4244, "有没有靠窗", &list, 10);
|
||||
assert!(r.contains(&list[0]) && !r.contains(&list[1]), "{r:?}");
|
||||
// No index to answer (fresh bot): the list itself, like the old days.
|
||||
assert_eq!(recall(5556, "完全不相干的查詢", &[], 10), Vec::<String>::new());
|
||||
let r = recall(5557, "完全不相干的查詢", &list, 10);
|
||||
assert_eq!(r, list, "fallback carries the list");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_steps_that_worked_are_kept_with_what_they_acted_on() {
|
||||
use crate::model::{Message, Role, ToolCall, ToolStatus};
|
||||
|
|
|
|||
165
src/store.rs
165
src/store.rs
|
|
@ -53,7 +53,24 @@ fn open() -> Option<Store> {
|
|||
call_id TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS audit_bot_at ON audit(bot, at DESC);
|
||||
CREATE INDEX IF NOT EXISTS audit_call ON audit(bot, op, call_id);",
|
||||
CREATE INDEX IF NOT EXISTS audit_call ON audit(bot, op, call_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memories(
|
||||
id INTEGER PRIMARY KEY,
|
||||
bot INTEGER NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
kind TEXT NOT NULL DEFAULT 'fact',
|
||||
created INTEGER NOT NULL,
|
||||
UNIQUE(bot, text)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS memories_bot ON memories(bot, created DESC);
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(text, content='memories', content_rowid='id', tokenize='trigram');
|
||||
CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
|
||||
INSERT INTO memories_fts(rowid, text) VALUES (new.id, new.text);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
|
||||
INSERT INTO memories_fts(memories_fts, rowid, text) VALUES ('delete', old.id, old.text);
|
||||
END;",
|
||||
)
|
||||
.ok()?;
|
||||
let cutoff = crate::model::now_ms() - KEEP_DAYS * 86_400_000;
|
||||
|
|
@ -211,6 +228,29 @@ mod tests {
|
|||
use super::*;
|
||||
use crate::oplog::{Prompt, Replay};
|
||||
|
||||
#[test]
|
||||
fn the_memory_index_answers_in_chinese() {
|
||||
// (One database per test process, one bot id: 4243 is ours here.)
|
||||
let bot = 4243u64;
|
||||
memory_add(bot, "我喜歡靠窗的座位,長途航班也是");
|
||||
memory_add(bot, "週五下午不開會");
|
||||
memory_add(9999, "別的 Bot 的事:預算 42 萬");
|
||||
memory_event(bot, "3-8:幫我排週五的會 → 已排在 15:00");
|
||||
let hits = recall(bot, "靠窗的座位", 10);
|
||||
assert!(hits.iter().any(|m| m.contains("靠窗")), "{hits:?}");
|
||||
// A two-word query finds lines that merely contain the words (LIKE path).
|
||||
let hits = recall(bot, "週五", 10);
|
||||
assert!(hits.iter().any(|m| m.contains("週五下午")), "{hits:?}");
|
||||
// Other bots' memories stay theirs.
|
||||
assert!(!recall(bot, "預算 42 萬", 10).iter().any(|m| m.contains("預算")), "leaked another bot's memory");
|
||||
// Events answer the "上個月說過的那個" questions.
|
||||
let hits = recall(bot, "排週五的會", 10);
|
||||
assert!(hits.iter().any(|m| m.contains("已排在 15:00")), "{hits:?}");
|
||||
// Forgetting leaves the index with the list.
|
||||
memory_del(bot, "週五下午不開會");
|
||||
assert!(!recall(bot, "週五下午不開會", 3).contains(&"週五下午不開會".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_trail_keeps_the_story_after_the_turn_log_cleans_up() {
|
||||
// (Tests share the temp dir of this process; one database, one bot id.
|
||||
|
|
@ -244,3 +284,126 @@ mod tests {
|
|||
assert_eq!(ts[1]["steps"][2]["state"], "consumed");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- 記憶的索引(P6)----------
|
||||
// `state.json`'s bot.memories stays the source that mirrors sync between devices, and the
|
||||
// list the person manages in 設定 › 記得的事. The index is its searchable side: a superset
|
||||
// (the JSON caps at 200, the index does not) plus *events* — one line per finished turn,
|
||||
// nobody's asked to write them — so "我上個月說的那個…" answers across months.
|
||||
// Best-effort in, best-effort out: with no index the caller falls back to the JSON list.
|
||||
|
||||
/// Index what a bot already remembers (startup / first use; INSERT OR IGNORE, cheap).
|
||||
pub fn memory_index(bot: u64, texts: &[String]) {
|
||||
let Some(s) = store() else { return };
|
||||
let Ok(c) = s.conn.lock() else { return };
|
||||
let now = crate::model::now_ms();
|
||||
for t in texts {
|
||||
let _ = c.execute("INSERT OR IGNORE INTO memories(bot,text,kind,created) VALUES(?1,?2,'fact',?3)", params![bot as i64, t, now]);
|
||||
}
|
||||
}
|
||||
|
||||
/// One new memory (the model's `remember`, or the person's 記住…).
|
||||
pub fn memory_add(bot: u64, text: &str) {
|
||||
let Some(s) = store() else { return };
|
||||
let Ok(c) = s.conn.lock() else { return };
|
||||
let _ = c.execute("INSERT OR IGNORE INTO memories(bot,text,kind,created) VALUES(?1,?2,'fact',?3)", params![bot as i64, text, crate::model::now_ms()]);
|
||||
}
|
||||
|
||||
/// A forgotten memory leaves the index with the list.
|
||||
pub fn memory_del(bot: u64, text: &str) {
|
||||
let Some(s) = store() else { return };
|
||||
let Ok(c) = s.conn.lock() else { return };
|
||||
let _ = c.execute("DELETE FROM memories WHERE bot=?1 AND text=?2", params![bot as i64, text]);
|
||||
}
|
||||
|
||||
/// One finished turn as a line of the event log: 「3-8:幫我發一篇 Threads… → 發好了…」.
|
||||
pub fn memory_event(bot: u64, text: &str) {
|
||||
if text.trim().is_empty() {
|
||||
return;
|
||||
}
|
||||
let Some(s) = store() else { return };
|
||||
let Ok(c) = s.conn.lock() else { return };
|
||||
let _ = c.execute("INSERT INTO memories(bot,text,kind,created) VALUES(?1,?2,'event',?3)", params![bot as i64, text, crate::model::now_ms()]);
|
||||
}
|
||||
|
||||
/// The memories that answer a request: phrase hits first (FTS trigram, good with Chinese),
|
||||
/// words inside lines next (LIKE, for short queries), newest facts last. The person's own
|
||||
/// shared preferences ride along (bot 0). Empty means "no index" — the caller decides.
|
||||
pub fn recall(bot: u64, query: &str, limit: usize) -> Vec<String> {
|
||||
let mut out: Vec<String> = vec![];
|
||||
let Some(s) = store() else { return out };
|
||||
let Ok(c) = s.conn.lock() else { return out };
|
||||
let q: String = query.chars().filter(|ch| *ch != '"').collect::<String>().split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
let bot = bot as i64;
|
||||
if q.chars().count() >= 3 {
|
||||
// Trigram wants at least three characters; a quoted phrase keeps it literal.
|
||||
let phrase = format!("\"{q}\"");
|
||||
if let Ok(rows) = c.prepare("SELECT m.text FROM memories m JOIN memories_fts f ON f.rowid=m.id WHERE memories_fts MATCH ?1 AND m.bot IN (?2, 0) ORDER BY bm25(memories_fts) LIMIT ?3").and_then(|mut st| {
|
||||
let it = st.query_map(params![phrase, bot, limit as i64], |r| r.get::<_, String>(0))?;
|
||||
let mut v = vec![];
|
||||
for x in it.flatten() {
|
||||
v.push(x);
|
||||
}
|
||||
Ok(v)
|
||||
}) {
|
||||
out.extend(rows);
|
||||
}
|
||||
}
|
||||
// Words inside lines next. Chinese queries are cut into two-character windows
|
||||
// ("有没有靠窗" → 有没/没有/有靠/靠窗) and lines score by how many windows they
|
||||
// contain: a shelf line answering the question outranks a fresh one.
|
||||
let mut segs: Vec<String> = vec![];
|
||||
for w in q.split_whitespace() {
|
||||
let ch: Vec<char> = w.chars().collect();
|
||||
if w.is_ascii() {
|
||||
if ch.len() >= 2 {
|
||||
segs.push(w.to_lowercase());
|
||||
}
|
||||
} else if ch.len() >= 2 {
|
||||
for i in 0..ch.len() - 1 {
|
||||
segs.push(format!("{}{}", ch[i], ch[i + 1]));
|
||||
}
|
||||
}
|
||||
}
|
||||
segs.sort();
|
||||
segs.dedup();
|
||||
segs.truncate(12);
|
||||
if !segs.is_empty() {
|
||||
let mut scored: Vec<(usize, i64, String)> = vec![]; // (windows hit, freshness, text)
|
||||
for seg in &segs {
|
||||
let _ = c.prepare("SELECT id, created, text FROM memories WHERE bot IN (?1, 0) AND text LIKE '%'||?2||'%' LIMIT 60").and_then(|mut st| {
|
||||
let it = st.query_map(params![bot, seg], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?, r.get::<_, String>(2)?)))?;
|
||||
for x in it.flatten() {
|
||||
let (id, created, text) = x;
|
||||
match scored.iter_mut().find(|(_, _, t)| t == &text) {
|
||||
Some(e) => e.0 += 1,
|
||||
None => scored.push((1, created, text)),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
scored.sort_by(|a, b| b.0.cmp(&a.0).then(b.1.cmp(&a.1)));
|
||||
for (_, _, text) in scored {
|
||||
if !out.contains(&text) {
|
||||
out.push(text);
|
||||
}
|
||||
}
|
||||
}
|
||||
if out.is_empty() {
|
||||
// Nothing matched: the freshest facts, so "記住…" works in the very next chat even
|
||||
// with no words to match.
|
||||
let want = limit;
|
||||
let _ = c.prepare("SELECT text FROM memories WHERE bot IN (?1, 0) AND kind='fact' ORDER BY created DESC LIMIT ?2").and_then(|mut st| {
|
||||
let it = st.query_map(params![bot, want as i64], |r| r.get::<_, String>(0))?;
|
||||
for x in it.flatten() {
|
||||
if !out.contains(&x) {
|
||||
out.push(x);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
out.truncate(limit);
|
||||
out
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue