AnyBot/src/memory.rs

642 lines
34 KiB
Rust
Raw Normal View History

// 記憶、技能、範本 (the original: user bot memories, agent skills and template skills/bots).
// - Memories belong to a bot: facts it was asked to keep across chats.
// - Skills are shared, named procedures; the bot sees their names and loads one when it needs it.
// - Templates start a new bot with a ready role and instructions.
use serde::{Deserialize, Serialize};
/// Most memories a bot keeps; the oldest go first (estimated).
pub const MAX_MEMORIES: usize = 200;
/// Longest memory (estimated).
pub const MAX_MEMORY_CHARS: usize = 500;
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq)]
pub struct Skill {
pub name: String,
pub description: String,
pub instructions: String,
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
/// The steps that worked, to do again as they were (replay_skill): no model in between,
/// each control found again by what it is, the first mismatch hands back to the bot.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub steps: Vec<ReplayStep>,
}
/// One step of a skill to do again: the tool and its arguments, and for a step that acted on a
/// numbered control, what that control was (「按鈕「Close」」), since numbers change from look to
/// look. `targets` has one entry per action of a computer_use_batch ("" where none).
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Default)]
pub struct ReplayStep {
pub tool: String,
pub args: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub targets: Vec<String>,
}
pub struct Template {
pub name: &'static str,
pub description: &'static str,
pub instructions: &'static str,
pub shape: &'static str,
pub color: &'static str,
}
pub const TEMPLATES: [Template; 5] = [
Template { name: "研究助理", description: "上網找資料、整理重點、附上來源", instructions: "先釐清問題,再用 web_fetch 或瀏覽器找至少兩個可靠來源。回答先給結論,再列重點,每個重點附來源網址。不確定的地方直接說不確定。", shape: "circle", color: "green" },
Template { name: "寫作夥伴", description: "幫你構思、改寫和潤稿", instructions: "先問清楚讀者和語氣。改稿時保留作者的意思,列出改了什麼和為什麼。用繁體中文、台灣用語。", shape: "cloud", color: "purple" },
Template { name: "電腦助手", description: "在自己的電腦上跑指令、整理檔案、操作網頁", instructions: "預設在你自己的電腦上做事。動手前先看清楚(list_dir、read_file、截圖),每一步確認結果。要動到使用者的 Mac 時先說明要做什麼。", shape: "gem", color: "blue" },
Template { name: "客服助理", description: "在 Slack、Discord、Telegram 或 Teams 回覆客人", instructions: "回答簡短、有禮貌。不確定的事不要編,說會請同事確認。不要透露內部資料或使用者的私人資訊。", shape: "pill", color: "orange" },
Template { name: "開發助手", description: "看程式、跑測試、追 GitHub 上的事", instructions: "改程式前先讀相關檔案和測試。每次修改後跑測試,回報結果。PR 或 CI 的事件來時,先看紀錄再下結論。", shape: "hexagon", color: "black" },
];
/// ---- Skill files (P7): a skill you can keep in a file, share, and install ----
///
/// A `.md` file in `<data>/skills/`: front matter (`name:`, `description:`), the instructions
/// as the body, and an optional ```steps fence (the ReplayStep list a replay follows).
pub fn skill_dir() -> std::path::PathBuf {
crate::oplog::dir().join("skills")
}
fn slug_of(name: &str) -> String {
let mut o: String = name
.chars()
.map(|c| if c.is_alphanumeric() { c.to_ascii_lowercase() } else { '-' })
.collect();
while o.contains("--") {
o = o.replace("--", "-");
}
let o: String = o.trim_matches('-').chars().take(40).collect();
if o.is_empty() {
"skill".into()
} else {
o
}
}
pub fn skill_to_file(s: &Skill) -> String {
let mut o = format!("---\nname: {}\ndescription: {}\n---\n\n{}\n", s.name, s.description, s.instructions);
if !s.steps.is_empty() {
if let Ok(j) = serde_json::to_string(&s.steps) {
o.push_str(&format!("\n```steps\n{j}\n```\n"));
}
}
o
}
pub fn parse_skill_file(text: &str) -> Option<Skill> {
let t = text.trim_start();
let mut name = String::new();
let mut description = String::new();
let mut rest = t;
if let Some(body) = t.strip_prefix("---") {
if let Some(end) = body.find("\n---") {
for line in body[..end].lines() {
if let Some(v) = line.strip_prefix("name:") {
name = v.trim().to_string();
} else if let Some(v) = line.strip_prefix("description:") {
description = v.trim().to_string();
}
}
rest = body[end + 3..].trim_start_matches('\n');
}
}
let mut steps: Vec<ReplayStep> = vec![];
let mut instructions = rest.to_string();
if let Some(start) = rest.find("```steps") {
if let Some(close) = rest[start..].find("```") {
// close is at the fence start inside rest[start..]; find the *second* fence
if let Some(end) = rest[start + 8..].find("```") {
let json = &rest[start + 8..start + 8 + end];
steps = serde_json::from_str(json.trim()).unwrap_or_default();
instructions = format!("{}\n{}", &rest[..start], &rest[start + 8 + end + 3..]).trim().to_string();
let _ = close;
}
}
}
if name.is_empty() {
return None;
}
Some(Skill { name, description, instructions, steps })
}
/// Save one skill file (the text is validated first). Returns the skill's name.
pub fn install_skill_text(text: &str) -> Result<String, String> {
let s = parse_skill_file(text).ok_or("這不是一張技能檔(開頭要有 name:)")?;
std::fs::create_dir_all(skill_dir()).map_err(|e| format!("開不了技能資料夾:{e}"))?;
std::fs::write(skill_dir().join(format!("{}.md", slug_of(&s.name))), text).map_err(|e| e.to_string())?;
Ok(s.name)
}
pub fn load_skill_files() -> Vec<Skill> {
let mut out = vec![];
if let Ok(rd) = std::fs::read_dir(skill_dir()) {
for f in rd.flatten() {
let p = f.path();
if p.extension().and_then(|e| e.to_str()) == Some("md") {
if let Ok(text) = std::fs::read_to_string(&p) {
if let Some(s) = parse_skill_file(&text) {
out.push(s);
}
}
}
}
}
out.sort_by(|a, b| a.name.cmp(&b.name));
out
}
/// Files win: a skill file replaces the same-named skill in the bot's head at every start.
pub fn merge_skill_files(skills: &mut Vec<Skill>) -> usize {
let mut n = 0;
for f in load_skill_files() {
match skills.iter_mut().find(|s| s.name == f.name) {
Some(old) => {
*old = f;
}
None => skills.push(f),
}
n += 1;
}
n
}
/// C8: 示範即教學 — turn the most recent finished turn that answered `prompt` into a skill,
/// keeping the steps that worked and threw nothing away. Looking steps (screenshots, reads) are
/// dropped: a replay looks for itself. Steps that point at a number on the screen (ref) are
/// dropped too — numbers only live as long as the look that made them.
pub fn skillify(prompt: &str, recs: &[crate::oplog::Record]) -> Option<Skill> {
let key: String = prompt.chars().take(40).collect();
let mut op = String::new();
for r in recs.iter().rev() {
if let crate::oplog::Record::OperationStarted { op: o, prompt: Some(p), .. } = r {
if p.text.chars().take(40).collect::<String>() == key {
op = o.clone();
break;
}
}
}
if op.is_empty() {
return None;
}
let mut done: Vec<(String, bool)> = vec![]; // index-parallel with steps: (call_id, finished ok)
let mut steps: Vec<(String, String)> = vec![];
for r in recs {
match r {
crate::oplog::Record::ToolStarted { op: o, call_id, name, args, .. } if o == &op => {
let a: serde_json::Value = serde_json::from_str(args).unwrap_or_default();
let looks = matches!(
name.as_str(),
"computer_use_screenshot" | "computer_use_zoom" | "computer_use_look" | "browser_screenshot" | "browser_read" | "browser_elements"
) || (name == "computer" && matches!(a["action"].as_str().unwrap_or(""), "screenshot" | "zoom" | "look" | "scroll" | "move"));
let numbered = a["ref"].is_number() || !a["actions"].is_null();
if !looks && !numbered {
steps.push((name.clone(), args.clone()));
done.push((call_id.clone(), true));
}
}
crate::oplog::Record::ToolFinished { op: o, call_id, status, .. } if o == &op => {
if status != "done" {
if let Some(e) = done.iter_mut().find(|(c, _)| c == call_id) {
e.1 = false;
}
}
}
_ => {}
}
}
let kept: Vec<ReplayStep> = steps
.into_iter()
.enumerate()
.filter(|(i, _)| done.get(*i).is_some_and(|(_, ok)| *ok))
.map(|(_, (tool, args))| ReplayStep { tool, args, targets: vec![] })
.take(24)
.collect();
if kept.is_empty() {
return None;
}
let name: String = prompt.chars().take(24).collect();
Some(Skill {
name,
description: format!("示範學會:{prompt}"),
instructions: format!("照著做過一次的那條路做完这件事:{prompt}"),
steps: kept,
})
}
/// Add a memory (no duplicates; the oldest drop off past the limit).
pub fn remember(list: &mut Vec<String>, fact: &str) -> Result<String, String> {
let f: String = fact.trim().chars().take(MAX_MEMORY_CHARS).collect();
if f.is_empty() {
return Err("要記住什麼?".into());
}
if list.iter().any(|m| m == &f) {
return Ok(format!("已經記得:{f}"));
}
list.push(f.clone());
let n = list.len();
if n > MAX_MEMORIES {
list.drain(..n - MAX_MEMORIES);
}
Ok(format!("記住了:{f}"))
}
/// Forget by number (1-based) or by text it contains.
pub fn forget(list: &mut Vec<String>, what: &str) -> Result<String, String> {
let w = what.trim();
let idx = match w.parse::<usize>() {
Ok(n) if n >= 1 && n <= list.len() => Some(n - 1),
_ => list.iter().position(|m| !w.is_empty() && m.contains(w)),
};
match idx {
Some(i) => Ok(format!("忘記了:{}", list.remove(i))),
None => Err(format!("沒有記得「{w}」這件事")),
}
}
/// Add a skill, or replace the one with the same name. True when it is new.
pub fn upsert_skill(list: &mut Vec<Skill>, k: Skill) -> bool {
match list.iter_mut().find(|x| x.name == k.name) {
Some(x) => {
*x = k;
false
}
None => {
list.push(k);
true
}
}
}
// ---------- 自己進化: a job's way that worked becomes a skill ----------
// The bot forgot everything between chats: a post with a picture on Threads took it 39 steps,
// and the next one would have taken as many. After a longer job the model looks back at the
// steps and writes down the short path that worked (or improves the skill it followed); the
// next time it reads that first.
/// Steps on computers and pages a job needs before it is worth looking back on (estimated).
pub const LEARN_MIN_STEPS: usize = 5;
/// Longest learned skill (estimated; a skill is read whole before each such job).
const MAX_SKILL_CHARS: usize = 4000;
/// A turn that did enough real work, and didn't already save a skill itself.
pub fn worth_learning(turn: &[crate::model::Message]) -> bool {
let calls: Vec<&str> = turn.iter().flat_map(|m| m.tool_calls.iter().map(|c| c.name.as_str())).collect();
!calls.contains(&"save_skill") && calls.iter().filter(|n| !crate::appctl::is_app_tool(n) && **n != "react").count() >= LEARN_MIN_STEPS
}
pub const LEARN_SYSTEM: &str = "你在幫一個會操作電腦的 Bot 整理經驗,讓它下次做同樣的事更快、不再犯同樣的錯。下面是它剛做完的一件事:使用者的要求、每一步和結果、最後的回覆,還有它已經有的技能。
先判斷要不要存。使用者希望它從做過的事學會、下次不用重新摸索:只要這件事是在某個網站或程式上動手操作(填表、發文、回留言、按讚、上傳下載、改設定、跑一套固定的檢查),就要存,就算這次只做了一次。只有閒聊、單純回答問題、只是查資料讀內容,才不用存。這次沒做成功的,不要存成做法;但如果已經有這件事的技能,可以把「這樣做行不通」補進去。
要存的話,寫一份「下次照著做就會成功」的技能:
- 寫這次真的成功的那條最短路徑,一步一行,寫出用哪個工具、在哪台電腦、哪個網址或程式(每一步結果開頭 [ ] 裡的名字,就是那一步用的電腦)。
- 繞路和失敗的嘗試不要寫成步驟;把「哪裡會卡、要怎麼避開」寫成「注意:」。
- 用畫面上看得到的名字指東西(例如 browser_click 的 text 寫「Attach media」、browser_type 的 text 寫欄位名「Customer name」),不要寫元素編號或座標,那些每次都會變。browser_open 的結果就有這一頁的元素清單,不用另外截圖。
- 寫通用的做法,不要寫這次的內容(這篇文的文字、這張圖的檔名);使用者表達過的偏好(語氣、格式、一定要附圖、用哪台電腦)要寫進去。
- 已經有做同一件事的技能,就用同一個名稱,把舊的和這次學到的合併成更好的一版:舊的對的保留,錯的改掉。這次照著技能做而且很順,就不用再存。
只回一個 JSON,不要別的字:
不用存:{\"save\": false, \"why\": \"一句話說為什麼\"}
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
要存:{\"save\": true, \"name\": \"短名稱\", \"description\": \"一行:什麼時候用\", \"instructions\": \"做法\", \"replay\": [步驟編號]}
replay:下次做同一件事時可以原樣照做的步驟編號(上面清單的號碼,照順序)。只列這次真的成功、必要、會改變東西的步驟(不含截圖、讀取這類只是看的步驟,不含失敗或繞路的步驟);內容每次會不一樣的工作(發文的文字、要回覆的留言、檔名)不要列,給空陣列 []。";
fn cut(s: &str, n: usize) -> String {
let flat = s.split_whitespace().collect::<Vec<_>>().join(" ");
if flat.chars().count() <= n { flat } else { format!("{}…", flat.chars().take(n).collect::<String>()) }
}
/// What the model looks back on: the request, each step with its result, the answer and the
/// skills there are.
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
/// The numbered steps the model looks back on, each with the call it was (None for words the
/// person typed in between, and for the "…省略…" line).
fn brief_steps(turn: &[crate::model::Message]) -> Vec<(String, Option<crate::model::ToolCall>)> {
use crate::model::Role;
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
let mut steps: Vec<(String, Option<crate::model::ToolCall>)> = vec![];
for (i, m) in turn.iter().enumerate() {
if m.role == Role::User && i > 0 {
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
steps.push((format!("(使用者中途說:{})", cut(&m.text, 300)), None));
}
for c in &m.tool_calls {
let result = turn.iter().find(|r| r.role == Role::Tool && r.tool_call_id == c.id).map(|r| r.text.as_str()).unwrap_or("");
let failed = if c.status == crate::model::ToolStatus::Failed { "(失敗)" } else { "" };
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
steps.push((format!("{} {} → {}{failed}", c.name, cut(&c.args, 160), cut(result, 220)), Some(c.clone())));
}
}
const KEEP: usize = 40;
if steps.len() > KEEP * 2 {
let gone = steps.len() - KEEP * 2;
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
steps.splice(KEEP..steps.len() - KEEP, [(format!("…(中間省略 {gone} 步)…"), None)]);
}
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
steps
}
pub fn learn_brief(turn: &[crate::model::Message], skills: &[Skill]) -> String {
let steps: Vec<String> = brief_steps(turn).into_iter().map(|(s, _)| s).collect();
let ask = turn.first().map(|m| cut(&m.text, 800)).unwrap_or_default();
let answer = turn.iter().rev().find(|m| m.role == crate::model::Role::Assistant && !m.text.trim().is_empty()).map(|m| cut(&m.text, 600)).unwrap_or_default();
let mut s = format!("使用者的要求:\n{ask}\n\n每一步(工具 參數 → 結果):\n");
for (i, st) in steps.iter().enumerate() {
s.push_str(&format!("{}. {st}\n", i + 1));
}
s.push_str(&format!("\n最後的回覆:\n{answer}\n\n已經有的技能:"));
if skills.is_empty() {
s.push_str("(沒有)");
}
let mut room = 8000usize;
for k in skills {
let body = cut(&k.instructions, 2000.min(room));
room = room.saturating_sub(body.chars().count());
s.push_str(&format!("\n- {}:{}\n 做法:{body}", k.name, k.description));
}
s
}
/// The skill in the model's answer, when it chose to save one.
pub fn learned_skill(text: &str) -> Option<Skill> {
let t = text.trim();
let json = t.find('{').and_then(|a| t.rfind('}').filter(|b| *b >= a).map(|b| &t[a..=b]))?;
let v: serde_json::Value = serde_json::from_str(json).ok()?;
if v["save"].as_bool() != Some(true) {
return None;
}
let name: String = v["name"].as_str()?.trim().chars().take(40).collect();
let instructions: String = v["instructions"].as_str()?.trim().chars().take(MAX_SKILL_CHARS).collect();
if name.is_empty() || instructions.is_empty() {
return None;
}
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
Some(Skill { name, description: v["description"].as_str().unwrap_or("").trim().chars().take(120).collect(), instructions, steps: vec![] })
}
/// The step numbers the model picked to do again as they were (its "replay").
pub fn learned_replay(text: &str) -> Vec<usize> {
let t = text.trim();
let Some(json) = t.find('{').and_then(|a| t.rfind('}').filter(|b| *b >= a).map(|b| &t[a..=b])) else { return vec![] };
let v: serde_json::Value = serde_json::from_str(json).unwrap_or_default();
v["replay"].as_array().map(|a| a.iter().filter_map(|n| n.as_u64().map(|n| n as usize)).collect()).unwrap_or_default()
}
/// The control a numbered step acted on, in words, from the newest list before it in the turn
/// ("[12] 按鈕「Close」" → 按鈕「Close」).
fn control_named(turn: &[crate::model::Message], before_call: &str, r: &str) -> String {
use crate::model::Role;
let n = r.trim().trim_matches(['[', ']', '#', 'e']);
if n.is_empty() {
return String::new();
}
let upto = turn.iter().position(|m| m.tool_calls.iter().any(|c| c.id == before_call)).unwrap_or(turn.len());
let prefix = format!("[{n}] ");
turn[..upto].iter().rev().filter(|m| m.role == Role::Tool).find_map(|m| m.text.lines().find_map(|l| l.trim_start().strip_prefix(&prefix).map(|d| d.trim().to_string()))).unwrap_or_default()
}
/// The skill's steps from the model's picks: each step's call, with the controls it acted on by
/// number written down as what they were. Steps that only look are left out whatever was picked.
pub fn replay_steps(turn: &[crate::model::Message], picks: &[usize]) -> Vec<ReplayStep> {
let steps = brief_steps(turn);
let mut out = vec![];
for &k in picks {
let Some((_, Some(c))) = k.checked_sub(1).and_then(|i| steps.get(i)) else { continue };
if c.status == crate::model::ToolStatus::Failed || matches!(c.name.as_str(), "computer_use_screenshot" | "computer_use_zoom" | "browser_read" | "browser_screenshot" | "browser_elements" | "use_skill" | "replay_skill") {
continue;
}
let v: serde_json::Value = serde_json::from_str(&c.args).unwrap_or_default();
let targets: Vec<String> = if c.name == "computer_use_batch" {
v["actions"].as_array().map(|a| a.iter().map(|x| x["ref"].as_str().map(|r| control_named(turn, &c.id, r)).unwrap_or_default()).collect()).unwrap_or_default()
} else {
match v["ref"].as_str().map(String::from).or_else(|| v["ref"].as_u64().map(|n| n.to_string())) {
Some(r) => vec![control_named(turn, &c.id, &r)],
None => vec![],
}
};
// A numbered step whose control can't be named can't be found again: not a step to replay.
if targets.iter().zip(v["actions"].as_array().map(|a| a.iter().map(|x| !x["ref"].is_null()).collect::<Vec<_>>()).unwrap_or_else(|| vec![!v["ref"].is_null()])).any(|(t, had)| had && t.is_empty()) {
return vec![];
}
out.push(ReplayStep { tool: c.name.clone(), args: c.args.clone(), targets: if targets.iter().all(String::is_empty) { vec![] } else { targets } });
}
out
}
/// The line under the bot's reply that says it learned (or improved) a skill.
pub fn learned_note(name: &str, fresh: bool) -> String {
if fresh {
format!("\n\n(學會了新做法「{name}」,下次做這件事會照著做。)")
} else {
format!("\n\n(把做法「{name}」更新了,下次會照新的做。)")
}
}
/// What to carry into a turn's prompt (P6): the memories that answer *this* request, from the
/// SQLite index (FTS + LIKE, the person's shared preferences included), not the whole shelf.
/// No index (fresh install, no database): the newest of the list, like the old days.
pub fn recall(bot: u64, query: &str, list: &[String], limit: usize) -> Vec<String> {
crate::store::memory_index(bot, list);
let hits = crate::store::recall(bot, query, limit);
if hits.is_empty() {
let n = list.len().min(20);
return list[list.len() - n..].to_vec();
}
hits
}
pub fn prompt_section(memories: &[String], skills: &[Skill]) -> String {
let mut s = String::new();
if !memories.is_empty() {
s.push_str("\n\n你記得的事(和這件事最相關的幾條;完整清單在「設定 › 記得的事」,用 remember 新增、forget 刪除):\n");
for (i, m) in memories.iter().enumerate() {
s.push_str(&format!("{}. {m}\n", i + 1));
}
} else {
s.push_str("\n\n使用者說「記住…」「以後都…」時,用 remember 記下來;說「忘記…」時用 forget。");
}
if !skills.is_empty() {
s.push_str("\n\n你會的技能(以前做成功的做法)。開始做一件事之前先看這裡:有符合的,先用 use_skill 讀出完整做法再照著做,不要重新摸索:\n");
for k in skills {
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
let replay = if k.steps.is_empty() { String::new() } else { format!("(可重播,{} 步)", k.steps.len()) };
s.push_str(&format!("- {}:{}{replay}\n", k.name, k.description));
}
if skills.iter().any(|k| !k.steps.is_empty()) {
s.push_str("標著「可重播」的技能:要做的事和技能一模一樣(同一個網站或程式、同樣的設定,沒有每次不同的內容)時,直接用 replay_skill,AnyBot 會照存下的步驟一步一步做,每一步先在畫面上找到同一個東西,對不上就停下來交給你;有一點不同就照做法自己做。\n");
}
}
s.push_str("\n使用者教你一套可以重複用的做法時,可以用 save_skill 存成技能,所有 Bot 都能用。");
s
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_shelf_is_searched_not_carried_whole() {
let list = vec!["我喜歡靠窗的座位".to_string(), "週五下午不開會".to_string()];
// A bot that has heard of 座位 gets just the line that answers.
let r = recall(4244, "有没有靠窗", &list, 10);
assert!(r.contains(&list[0]) && !r.contains(&list[1]), "{r:?}");
// No index to answer (fresh bot): the list itself, like the old days.
assert_eq!(recall(5556, "完全不相干的查詢", &[], 10), Vec::<String>::new());
let r = recall(5557, "完全不相干的查詢", &list, 10);
assert_eq!(r, list, "fallback carries the list");
}
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
#[test]
fn the_steps_that_worked_are_kept_with_what_they_acted_on() {
use crate::model::{Message, Role, ToolCall, ToolStatus};
let call = |id: &str, name: &str, args: &str| Message { role: Role::Assistant, tool_calls: vec![ToolCall { id: id.into(), name: name.into(), args: args.into(), status: ToolStatus::Done }], ..Default::default() };
let turn = vec![
Message::user("把自動更新打開"),
call("a", "computer_use_screenshot", "{}"),
Message::tool_result("a", "computer_use_screenshot", "螢幕截圖。\n[3] 按鈕「Close」\n[5] 勾選框「自動更新」:0"),
call("b", "computer_use_click", r#"{"ref":"5"}"#),
Message::tool_result("b", "computer_use_click", "已點 #5(勾選框「自動更新」:0)。"),
call("c", "shell", r#"{"command":"defaults read x"}"#),
Message::tool_result("c", "shell", "1"),
];
let brief = learn_brief(&turn, &[]);
assert!(brief.contains("1. computer_use_screenshot") && brief.contains("2. computer_use_click") && brief.contains("3. shell"), "{brief}");
let steps = replay_steps(&turn, &[1, 2, 3]);
// The look is left out; the click remembers what it clicked.
assert_eq!(steps.len(), 2);
assert_eq!(steps[0].tool, "computer_use_click");
assert_eq!(steps[0].targets, vec!["勾選框「自動更新」:0".to_string()]);
assert!(steps[1].targets.is_empty());
assert_eq!(learned_replay(r#"{"save": true, "name": "x", "instructions": "y", "replay": [2, 3]}"#), vec![2, 3]);
assert!(learned_replay(r#"{"save": true, "name": "x", "instructions": "y"}"#).is_empty());
// A numbered step whose control can't be named: nothing to replay safely.
assert!(replay_steps(&turn[..1].iter().cloned().chain([call("d", "computer_use_click", r#"{"ref":"77"}"#)]).collect::<Vec<_>>(), &[1]).is_empty());
}
#[test]
fn remember_and_forget() {
let mut m = vec![];
assert!(remember(&mut m, "我喜歡靠窗的座位").unwrap().starts_with("記住了"));
assert!(remember(&mut m, "我喜歡靠窗的座位").unwrap().starts_with("已經記得"));
remember(&mut m, "週五不開會").unwrap();
assert_eq!(forget(&mut m, "1").unwrap(), "忘記了:我喜歡靠窗的座位");
assert!(forget(&mut m, "不存在").is_err());
assert_eq!(forget(&mut m, "週五").unwrap(), "忘記了:週五不開會");
for i in 0..(MAX_MEMORIES + 5) {
remember(&mut m, &format!("事 {i}")).unwrap();
}
assert_eq!(m.len(), MAX_MEMORIES);
assert_eq!(m[0], "事 5");
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
let p = prompt_section(&m[..1], &[Skill { name: "週報".into(), description: "整理每週進度".into(), instructions: "…".into(), steps: vec![] }]);
assert!(p.contains("1. 事 5") && p.contains("- 週報:整理每週進度") && p.contains("use_skill"));
}
fn call(id: &str, name: &str, args: &str, failed: bool) -> crate::model::ToolCall {
crate::model::ToolCall { id: id.into(), name: name.into(), args: args.into(), status: if failed { crate::model::ToolStatus::Failed } else { crate::model::ToolStatus::Done } }
}
fn result(id: &str, text: &str) -> crate::model::Message {
crate::model::Message { role: crate::model::Role::Tool, tool_call_id: id.into(), text: text.into(), ..Default::default() }
}
/// A post with a picture: five real steps, one of them failed, the person corrects it midway.
fn post_turn() -> Vec<crate::model::Message> {
use crate::model::Message;
let step = |c: crate::model::ToolCall| Message { tool_calls: vec![c], ..Message::assistant("", "b") };
vec![
Message::user("幫我發一篇 Threads,記得附圖"),
step(call("1", "browser_open", r#"{"url":"https://www.threads.com/"}"#, false)),
result("1", "已打開 https://www.threads.com/\n[13] 按鈕「Create」"),
step(call("2", "browser_click", r#"{"ref":"13"}"#, false)),
result("2", "已點 [13]。新出現的文字:New thread"),
Message { steer: true, ..Message::user("語氣要輕鬆一點") },
step(call("3", "browser_type", r#"{"ref":"12","value":"今天天氣好"}"#, false)),
result("3", "已在 [12] 輸入。"),
step(call("4", "browser_upload", r#"{"path":"/home/bot/x.jpg"}"#, true)),
result("4", "這台電腦上沒有這個檔案"),
step(call("5", "browser_upload", r#"{"path":"/home/bot/bot-6/附件/x.jpg"}"#, false)),
result("5", "已把 /home/bot/bot-6/附件/x.jpg 放進上傳欄位。"),
step(call("6", "remember", r#"{"fact":"x"}"#, false)),
result("6", "記住了"),
Message::assistant("發好了:https://www.threads.com/@me/post/1", "b"),
]
}
#[test]
fn a_long_enough_job_is_looked_back_on() {
let turn = post_turn();
assert!(worth_learning(&turn));
// App steps (remember) don't count; four real steps aren't enough.
assert!(!worth_learning(&turn[..10]));
// A job that saved a skill itself is left alone.
let mut saved = turn.clone();
saved[12].tool_calls[0].name = "save_skill".into();
assert!(!worth_learning(&saved));
}
#[test]
fn the_look_back_has_every_step_and_the_skills() {
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before; 16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s. Knowing the machine - facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager, tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the instructions at the start of a turn. - The box's sudo is the bot's: only what follows sudo is checked against the always-denied list (it was refused outright, while the instructions said it could sudo), and commands with sudo run as the box's root (setuid does nothing under gVisor). - Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE / ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one. - Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the sandbox broke says so, instead of looking like a broken program. Reaching what it sees - The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside for the system's own), computer_use_zoom looks closer at full sharpness. - Numbers stay with their controls from look to look (they were reassigned every look, so a step's second click landed elsewhere); the box's front window is listed first; a control can be named instead of numbered (name "4"). - Web page numbering walks open shadow DOM (Chromium settings, web components). - A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry without it came up with no network). Working like a person - Instructions: pick the route, know what done looks like, one route, check once, say what's stuck instead of faking it with a substitute, ask before cutting the connection. - StepPlan: the bot's model takes the first step, the step after trouble (failed, refused, repeated, nothing changed on screen, the quick model switching routes), one in four, and the final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does the planned steps in between, falling back to the bot's model if it can't be used. - Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does them again with no model in between, finding each control again, handing back on a mismatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
let skills = [Skill { name: "Threads 發文".into(), description: "在 Threads 發文".into(), instructions: "1. browser_open https://www.threads.com/".into(), steps: vec![] }];
let b = learn_brief(&post_turn(), &skills);
assert!(b.starts_with("使用者的要求:\n幫我發一篇 Threads,記得附圖"), "{b}");
assert!(b.contains("1. browser_open {\"url\":\"https://www.threads.com/\"} → 已打開 https://www.threads.com/ [13] 按鈕「Create」"), "{b}");
assert!(b.contains("(使用者中途說:語氣要輕鬆一點)"), "{b}");
assert!(b.contains("browser_upload {\"path\":\"/home/bot/x.jpg\"} → 這台電腦上沒有這個檔案(失敗)"), "{b}");
assert!(b.contains("最後的回覆:\n發好了"), "{b}");
assert!(b.contains("- Threads 發文:在 Threads 發文\n 做法:1. browser_open"), "{b}");
// A very long job keeps its start and its end.
let mut long = vec![crate::model::Message::user("做很多事")];
for i in 0..200 {
long.push(crate::model::Message { tool_calls: vec![call(&i.to_string(), "browser_click", "{}", false)], ..crate::model::Message::assistant("", "b") });
}
let b = learn_brief(&long, &[]);
assert!(b.contains("(中間省略 120 步)") && b.contains("81. browser_click") && !b.contains("82. "), "{b}");
}
#[test]
fn a_learned_skill_is_read_from_the_answer() {
let k = learned_skill("好的\n{\"save\": true, \"name\": \" Threads 發文附圖 \", \"description\": \"在 Threads 發一篇附圖的文\", \"instructions\": \"1. browser_open https://www.threads.com/\"}").unwrap();
assert_eq!((k.name.as_str(), k.description.as_str()), ("Threads 發文附圖", "在 Threads 發一篇附圖的文"));
assert!(learned_skill(r#"{"save": false}"#).is_none());
assert!(learned_skill(r#"{"save": true, "name": "", "instructions": "x"}"#).is_none());
assert!(learned_skill("不用存").is_none());
assert!(learned_skill("} {").is_none());
let mut list = vec![];
assert!(upsert_skill(&mut list, k.clone()));
assert!(!upsert_skill(&mut list, Skill { instructions: "新的".into(), ..k }));
assert_eq!((list.len(), list[0].instructions.as_str()), (1, "新的"));
assert!(learned_note("Threads 發文附圖", true).contains("學會了新做法「Threads 發文附圖」"));
}
#[test]
fn a_skill_round_trips_through_a_file() {
let sk = Skill {
name: "每日打卡".into(),
description: "在下載資料夾記一天".into(),
instructions: "打開終端機記個日期".into(),
steps: vec![ReplayStep { tool: "shell".into(), args: "{\"cmd\":\"date +%F\"}".into(), targets: vec![] }],
};
let text = skill_to_file(&sk);
let back = parse_skill_file(&text).unwrap();
assert_eq!(back.name, "每日打卡");
assert_eq!(back.steps, sk.steps);
assert!(parse_skill_file("no front matter at all").is_none());
}
#[test]
fn a_finished_turn_can_become_a_skill() {
use crate::oplog::{Prompt, Record, Replay};
let recs = vec![
Record::OperationStarted { op: "o1".into(), at: 1, source_leaf: 0, prompt: Some(Prompt { at: 1, text: "每天記一下日期".into() }) },
Record::ToolStarted { op: "o1".into(), call_id: "c1".into(), name: "computer".into(), args: "{\"action\":\"screenshot\"}".into(), replay: Replay::Safe, at: 2 },
Record::ToolFinished { op: "o1".into(), call_id: "c1".into(), status: "done".into(), at: 3 },
Record::ToolStarted { op: "o1".into(), call_id: "c2".into(), name: "shell".into(), args: "{\"cmd\":\"date\"}".into(), replay: Replay::Safe, at: 4 },
Record::ToolFinished { op: "o1".into(), call_id: "c2".into(), status: "done".into(), at: 5 },
Record::ToolStarted { op: "o1".into(), call_id: "c3".into(), name: "shell".into(), args: "{\"cmd\":\"rm\"}".into(), replay: Replay::Safe, at: 6 },
Record::ToolFinished { op: "o1".into(), call_id: "c3".into(), status: "failed".into(), at: 7 },
Record::ToolStarted { op: "o1".into(), call_id: "c4".into(), name: "computer".into(), args: "{\"action\":\"click\",\"ref\":3}".into(), replay: Replay::Safe, at: 8 },
Record::ToolFinished { op: "o1".into(), call_id: "c4".into(), status: "done".into(), at: 9 },
];
let sk = skillify("每天記一下日期", &recs).unwrap();
// only the shell date step survives: looks and numbered clicks are dropped, failures too
assert_eq!(sk.steps.len(), 1);
assert_eq!(sk.steps[0].tool, "shell");
assert!(sk.steps[0].args.contains("date"));
assert!(skillify("沒這回合", &recs).is_none());
}
}