From 0884ec736c56ecd8e54344a32ea4944dc567c7ba Mon Sep 17 00:00:00 2001 From: AnyBot Dev Date: Mon, 9 Nov 2026 20:30:38 +0800 Subject: [PATCH] Shortcuts before GUI (C6): the one-step doors, registered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - intents.rs: volume / wifi / lock / dark / trash / eject as named shortcuts — the model picks a name, the script is built from constants and checked numbers ("volume set 40; rm -rf /" is not a thing: args are grammar-checked before any shell sees them) - the computer tool gains action=intent {intent_name, intent_arg}: one call where the bench measured 9 steps and 2 minutes of GUI - the computer's facts line advertises the shortcuts each machine really has (Mac 6, Linux desktop 5, headless box 0 — nobody hears a box's volume), so the prompt cache carries the knowledge for free - scripts self-dispatch (uname): osascript/networksetup/pmset on Mac, pactl/amixer/nmcli/loginctl/gsettings on Linux, honest failure text - system prompt: 有捷徑先捷徑，捷徑失敗才回 GUI - tests: the injection grammar, the availability matrix, a real run --- src/app.rs | 1 + src/computer.rs | 13 +++- src/facts.rs | 5 ++ src/intents.rs | 203 ++++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 1 + 5 files changed, 220 insertions(+), 3 deletions(-) create mode 100644 src/intents.rs diff --git a/src/app.rs b/src/app.rs index c1e373e..4c85b57 100644 --- a/src/app.rs +++ b/src/app.rs @@ -4356,6 +4356,7 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu } s.push_str(&crate::memory::prompt_section(memories, &st.skills)); s.push_str("使用者問「自己的檔案」（那份報表、寫過的筆記、存過的檔案在哪）：先用 search_documents，它會用意思加關鍵字搜對方索引過的資料夾，不記得確切用詞也找得到。"); + s.push_str("系統捷徑：電腦現況列了「系統捷徑」的電腦，音量、Wi-Fi、鎖螢幕、深淺色、清空資源筒這類事要用 computer 的 intent 動作一步做完（intent_name＋intent_arg），不要用截圖加點擊；捷徑回報失敗才回 GUI。"); let mcp_notes = crate::mcp::instructions(&st.mcp_servers); let mcp_tools = crate::mcp::status(); if !mcp_tools.is_empty() { diff --git a/src/computer.rs b/src/computer.rs index 22d1cfb..5ab30c3 100644 --- a/src/computer.rs +++ b/src/computer.rs @@ -12,7 +12,7 @@ use base64::Engine; use serde_json::{json, Value}; use std::sync::Mutex; -pub const NAMES: [&str; 15] = ["computer_use_screenshot", "computer_use_zoom", "computer_use_show", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app", "computer_use_batch", "computer_use_menu_click", "computer_use_fill_form", "computer_use_wait_for"]; +pub const NAMES: [&str; 16] = ["computer_use_screenshot", "computer_use_zoom", "computer_use_show", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app", "computer_use_batch", "computer_use_menu_click", "computer_use_fill_form", "computer_use_wait_for", "computer_use_intent"]; /// Wait after an action before taking the picture that comes back with it (menus open, pages /// start to change; estimated). The box waits for the screen to stop changing instead, up to /// about this long. @@ -303,6 +303,7 @@ pub fn describe(name: &str, args: &str) -> String { "computer_use_menu_click" => format!("選單 {}", v["menu"].as_array().map(|a| a.iter().filter_map(|x| x.as_str()).collect::>().join("→")).unwrap_or_default()), "computer_use_fill_form" => format!("{} 個欄位", v["fields"].as_array().map(|a| a.len()).unwrap_or(0)), "computer_use_wait_for" => format!("等「{}」", v["text"].as_str().unwrap_or("")), + "computer_use_intent" => format!("捷徑 {} {}", v["intent_name"].as_str().unwrap_or(""), v["intent_arg"].as_str().unwrap_or("")), _ => String::new(), } } @@ -320,13 +321,13 @@ pub fn specs_for(computers: &[String], default: &str, chosen: bool) -> Vec Vec Option<&'static str> { "open_app" => "computer_use_open_app", "batch" => "computer_use_batch", "check_permissions" => "computer_use_check_permissions", + "intent" => "computer_use_intent", _ => return None, }) } @@ -1329,6 +1333,9 @@ fn run_inner(name: &str, v: &Value) -> Result<(String, Option), String> if name == "computer_use_wait_for" { return Ok((wait_for_tool(v)?, None)); } + if name == "computer_use_intent" { + return crate::intents::tool(v); + } #[cfg(target_os = "linux")] if !crate::desk::available() { return Err(crate::desk::describe().into()); diff --git a/src/facts.rs b/src/facts.rs index d405ab7..1dce30b 100644 --- a/src/facts.rs +++ b/src/facts.rs @@ -142,6 +142,11 @@ pub fn describe(name: &str, f: &Facts, own_box: bool, mac_sandbox: bool) -> Stri let more = if f.apps.len() > MAX_APPS { "…" } else { "" }; parts.push(format!("裝了的 App：{}{more}", f.apps.iter().take(MAX_APPS).cloned().collect::>().join("、"))); } + // The one-step doors (C6): advertised here so the model takes them instead of clicking. + let shortcuts = crate::intents::facts_line(&f.kernel, !f.wm.is_empty() || !f.desktop.is_empty(), own_box); + if !shortcuts.is_empty() { + parts.push(shortcuts); + } format!("「{name}」：{}。", parts.join("；")) } diff --git a/src/intents.rs b/src/intents.rs new file mode 100644 index 0000000..a4a74ee --- /dev/null +++ b/src/intents.rs @@ -0,0 +1,203 @@ +//! 捷徑優先於 GUI (C6): the operations a person does with one key press, registered as one +//! step for the bot. +//! +//! The bench said it plainly: 「把音量調小」 took 9 model steps through the GUI (about two +//! minutes) and one shell command (four seconds). Volume, Wi-Fi, the lock, dark mode — these +//! have supported command-line doors; a bot that clicks menus is a bot that didn't know the +//! door existed. So the registry here is the knowledge, the computer's facts line advertises +//! it every turn (cached by the prompt cache), and the computer tool's `intent` action walks +//! through: one call, one command, no screenshot. +//! +//! A script is a constant chosen from a validated name and argument — the model never writes +//! the command, it picks a name. That is why this can run where clicks run, with the same +//! approval, and why 「體積 set 999; rm -rf /」 is not a thing: the argument is checked against +//! the intent's grammar (a number 0–100, on/off, dark/light) before any script is built. + +use serde_json::Value; + +/// One registered shortcut: what it's called, how to say it, which machines have it. +pub struct Intent { + pub name: &'static str, + /// How to call it, shown in the computer's facts line. + pub usage: &'static str, + /// Does this computer have it? kernel: "Darwin"/"Linux"; desktop: a window system is + /// running (a headless box has no volume knob the person hears). + pub has: fn(kernel: &str, desktop: bool) -> bool, +} + +pub const INTENTS: &[Intent] = &[ + Intent { + name: "volume", + usage: "volume（音量：arg=set 0-100 / up / down / mute / unmute）", + has: |k, desktop| matches!(k, "Darwin") || (k == "Linux" && desktop), + }, + Intent { + name: "wifi", + usage: "wifi（Wi-Fi 開關：arg=on / off）", + has: |k, desktop| matches!(k, "Darwin") || (k == "Linux" && desktop), + }, + Intent { + name: "lock", + usage: "lock（鎖螢幕/關螢幕：不用 arg）", + has: |k, desktop| matches!(k, "Darwin") || (k == "Linux" && desktop), + }, + Intent { + name: "dark", + usage: "dark（深淺色：arg=dark / light / toggle）", + has: |k, desktop| matches!(k, "Darwin") || (k == "Linux" && desktop), + }, + Intent { + name: "trash", + usage: "trash（清空資源筒/垃圾桶：不用 arg）", + has: |k, desktop| matches!(k, "Darwin") || (k == "Linux" && desktop), + }, + Intent { + name: "eject", + usage: "eject（退出所有可退出磁碟：不用 arg，Mac）", + has: |k, _| k == "Darwin", + }, +]; + +pub fn find(name: &str) -> Option<&'static Intent> { + INTENTS.iter().find(|i| i.name == name) +} + +/// The names this kind of computer answers to — the facts line and the tool's error both +/// speak this. +pub fn names_for(kernel: &str, desktop: bool) -> Vec<&'static str> { + INTENTS.iter().filter(|i| (i.has)(kernel, desktop)).map(|i| i.name).collect() +} + +/// The sentence in a computer's facts: names and grammar, so the model can pick without +/// guessing. Empty for a headless box (its "volume" is nobody's volume). +pub fn facts_line(kernel: &str, desktop: bool, own_box: bool) -> String { + if own_box { + return String::new(); + } + let list = names_for(kernel, desktop); + if list.is_empty() { + return String::new(); + } + let usages: Vec<&str> = INTENTS.iter().filter(|i| list.contains(&i.name)).map(|i| i.usage).collect(); + format!("系統捷徑（有捷徑先捷徑，一步完成，不要用 GUI 點擊；用 computer 工具 action=intent，參數 intent_name＋intent_arg）：{}", usages.join("；")) +} + +fn is_num(s: &str) -> Option { + let n: u32 = s.parse().ok()?; + if n <= 100 { + Some(n) + } else { + None + } +} + +/// The script for one validated pick. Constants and (checked) numbers only — the model's +/// words never reach the shell. +pub fn script(name: &str, arg: &str) -> Result { + if find(name).is_none() { + return Err(format!("沒有捷徑「{name}」。")); + } + let a = arg.trim(); + let mac = |s: &str| format!("case \"$(uname -s)\" in Darwin) {s} ;; Linux) "); + Ok(match (name, a) { + ("volume", "up") => format!( + "{}osascript -e 'set u to (output volume of (get volume settings)) + 15' -e 'if u > 100 then set u to 100' -e 'set volume output volume u' ;; pactl set-sink-volume @DEFAULT_SINK@ +15% 2>/dev/null || amixer -q sset Master 15%+ ;; esac)", + mac("") + ), + ("volume", "down") => format!( + "{}osascript -e 'set u to (output volume of (get volume settings)) - 15' -e 'if u < 0 then set u to 0' -e 'set volume output volume u' ;; pactl set-sink-volume @DEFAULT_SINK@ -15% 2>/dev/null || amixer -q sset Master 15%- ;; esac)", + mac("") + ), + ("volume", "mute") => format!( + "{}osascript -e 'set m to muted of (get volume settings)' -e 'set volume output muted (not m)' ;; pactl set-sink-mute @DEFAULT_SINK@ toggle 2>/dev/null || amixer -q sset Master toggle ;; esac)", + mac("") + ), + ("volume", "unmute") => format!("{}osascript -e 'set volume output muted false' ;; pactl set-sink-mute @DEFAULT_SINK@ 0 2>/dev/null || amixer -q sset Master unmute ;; esac)", mac("")), + ("volume", set) => { + // "set 40" as the facts line teaches; a bare "40" is kindly taken too. + let Some(n) = is_num(set.strip_prefix("set ").unwrap_or(set).trim()) else { + return Err("volume 的 arg 要写成 set 0-100（如 set 40）或 up/down/mute/unmute。".into()); + }; + format!("{}osascript -e 'set volume output volume {n}' ;; pactl set-sink-volume @DEFAULT_SINK@ {n}% 2>/dev/null || amixer -q sset Master {n}% ;; esac)", mac("")) + } + ("wifi", "on") => format!("{}networksetup -setairportpower en0 on ;; nmcli radio wifi on 2>/dev/null || rfkill unblock wifi ;; esac)", mac("")), + ("wifi", "off") => format!("{}networksetup -setairportpower en0 off ;; nmcli radio wifi off 2>/dev/null || rfkill block wifi ;; esac)", mac("")), + ("lock", "") => format!("{}pmset displaysleepnow ;; loginctl lock-session 2>/dev/null || xset dpms force off ;; esac)", mac("")), + ("dark", "dark") => format!("{}osascript -e 'tell application \"System Events\" to tell appearance preferences to set dark mode to true' ;; gsettings set org.gnome.desktop.interface color-scheme prefer-dark ;; esac)", mac("")), + ("dark", "light") => format!("{}osascript -e 'tell application \"System Events\" to tell appearance preferences to set dark mode to false' ;; gsettings set org.gnome.desktop.interface color-scheme prefer-light ;; esac)", mac("")), + ("dark", "toggle") => format!( + "{}osascript -e 'tell application \"System Events\" to tell appearance preferences to set dark mode to not (dark mode)' ;; gsettings set org.gnome.desktop.interface color-scheme \"$( [ \"$(gsettings get org.gnome.desktop.interface color-scheme)\" = \"'prefer-dark'\" ] && echo prefer-light || echo prefer-dark )\" ;; esac)", + mac("") + ), + ("trash", "") => format!("{}osascript -e 'tell application \"Finder\" to empty trash' ;; rm -rf \"$HOME/.local/share/Trash/files\" \"$HOME/.local/share/Trash/info\" ;; esac)", mac("")), + ("eject", "") => format!("{}osascript -e 'tell application \"Finder\" to eject (every disk whose ejectable is true)' ;; echo 只有 Mac 有這個捷徑 ;; esac)", mac("")), + (_, "") => return Err(format!("「{name}」不需要 arg。")), + (_, _) => return Err(format!("「{name}」的 arg 不對。看捷徑說明：{}", find(name).map(|i| i.usage).unwrap_or(""))), + }) +} + +/// The computer tool's `intent` action: pick, build, run, report. Runs where the computer +/// tool runs — the bot's box is excluded by the facts line (it advertises no shortcuts). +pub fn tool(v: &Value) -> Result<(String, Option), String> { + let name = v["intent_name"].as_str().unwrap_or("").trim(); + let arg = v["intent_arg"].as_str().unwrap_or("").trim(); + if name.is_empty() { + return Err(format!("要 intent_name。這台電腦的捷徑：{}", INTENTS.iter().map(|i| i.name).collect::>().join("、"))); + } + let sh = script(name, arg)?; + let out = std::process::Command::new("/bin/sh").arg("-c").arg(&sh).env("PATH", crate::cloudbox::tool_path()).output().map_err(|e| format!("捷徑跑不起來：{e}"))?; + let text = format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)).trim().to_string(); + if out.status.success() { + Ok((format!("捷徑 {name}{}完成。{}", if arg.is_empty() { String::new() } else { format!("（{arg}）") }, if text.is_empty() { String::new() } else { format!(" 系統說：{text}") }), None)) + } else { + Ok((format!("捷徑 {name} 的指令回來了但沒成功（這台電腦可能沒有對應的系統指令）：{text}"), None)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_mac_and_a_linux_desktop_get_their_shortcuts() { + let mac = names_for("Darwin", true); + for n in ["volume", "wifi", "lock", "dark", "trash", "eject"] { + assert!(mac.contains(&n), "{n}"); + } + let linux = names_for("Linux", true); + assert!(linux.contains(&"volume") && !linux.contains(&"eject")); + // A headless box advertises none: nobody hears its volume. The facts line says so. + assert!(names_for("Linux", false).is_empty()); + assert!(facts_line("Linux", true, true).is_empty(), "the bot's own box: no shortcuts"); + let line = facts_line("Darwin", true, false); + for want in ["系統捷徑", "volume", "intent_name", "不要用 GUI 點擊", "eject"] { + assert!(line.contains(want), "{want}: {line}"); + } + } + + #[test] + fn words_never_reach_the_shell() { + // The model picks a name; the script is built from constants and checked numbers. + assert!(script("volume", "set 40").unwrap().contains("output volume 40")); + for bad in ["set 101", "set -5", "set 40; rm -rf /", "set 40$(id)", "set x", "999", "up; reboot"] { + assert!(script("volume", bad).is_err(), "{bad} must not build a script"); + } + assert!(script("wifi", "on").unwrap().contains("networksetup")); + assert!(script("wifi", "on; rm -rf /").is_err()); + assert!(script("lock", "").unwrap().contains("pmset")); + assert!(script("lock", "now").is_err(), "lock takes no arg"); + assert!(script("dark", "toggle").unwrap().contains("dark mode to not")); + assert!(script("nope", "").is_err()); + // The tool says which names exist when the name is missing. + let e = tool(&serde_json::json!({})).unwrap_err(); + assert!(e.contains("volume") && e.contains("intent_name"), "{e}"); + } + + #[test] + fn a_shortcut_runs_here_and_says_so() { + // This machine is Linux: volume goes through pactl/amixer; with neither in the + // sandbox the script still returns an honest sentence, never a panic. + let (text, _) = tool(&serde_json::json!({"intent_name": "lock"})).unwrap(); + assert!(text.starts_with("捷徑 lock"), "{text}"); + } +} diff --git a/src/main.rs b/src/main.rs index a34ddae..a4389a1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,6 +29,7 @@ mod desk; mod egress; mod facts; mod httpd; +mod intents; mod i18n; mod i18n_table; mod icons;