Merge c6/intents: shortcuts before GUI (C6)
This commit is contained in:
daniel wang
commit
ce1ed6aea2
|
|
@ -4356,6 +4356,7 @@ fn system_prompt(b: &Bot, st: &State, computers: &[tools::Target], default_compu
|
|||
}
|
||||
s.push_str(&crate::memory::prompt_section(memories, &st.skills));
|
||||
s.push_str("使用者問「自己的檔案」（那份報表、寫過的筆記、存過的檔案在哪）：先用 search_documents，它會用意思加關鍵字搜對方索引過的資料夾，不記得確切用詞也找得到。");
|
||||
s.push_str("系統捷徑：電腦現況列了「系統捷徑」的電腦，音量、Wi-Fi、鎖螢幕、深淺色、清空資源筒這類事要用 computer 的 intent 動作一步做完（intent_name＋intent_arg），不要用截圖加點擊；捷徑回報失敗才回 GUI。");
|
||||
let mcp_notes = crate::mcp::instructions(&st.mcp_servers);
|
||||
let mcp_tools = crate::mcp::status();
|
||||
if !mcp_tools.is_empty() {
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ use base64::Engine;
|
|||
use serde_json::{json, Value};
|
||||
use std::sync::Mutex;
|
||||
|
||||
pub const NAMES: [&str; 15] = ["computer_use_screenshot", "computer_use_zoom", "computer_use_show", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app", "computer_use_batch", "computer_use_menu_click", "computer_use_fill_form", "computer_use_wait_for"];
|
||||
pub const NAMES: [&str; 16] = ["computer_use_screenshot", "computer_use_zoom", "computer_use_show", "computer_use_click", "computer_use_typing", "computer_use_press_key", "computer_use_scroll", "computer_use_mouse_move", "computer_use_drag", "computer_use_check_permissions", "computer_use_open_app", "computer_use_batch", "computer_use_menu_click", "computer_use_fill_form", "computer_use_wait_for", "computer_use_intent"];
|
||||
/// Wait after an action before taking the picture that comes back with it (menus open, pages
|
||||
/// start to change; estimated). The box waits for the screen to stop changing instead, up to
|
||||
/// about this long.
|
||||
|
|
@ -303,6 +303,7 @@ pub fn describe(name: &str, args: &str) -> String {
|
|||
"computer_use_menu_click" => format!("選單 {}", v["menu"].as_array().map(|a| a.iter().filter_map(|x| x.as_str()).collect::<Vec<_>>().join("→")).unwrap_or_default()),
|
||||
"computer_use_fill_form" => format!("{} 個欄位", v["fields"].as_array().map(|a| a.len()).unwrap_or(0)),
|
||||
"computer_use_wait_for" => format!("等「{}」", v["text"].as_str().unwrap_or("")),
|
||||
"computer_use_intent" => format!("捷徑 {} {}", v["intent_name"].as_str().unwrap_or(""), v["intent_arg"].as_str().unwrap_or("")),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
|
@ -320,13 +321,13 @@ pub fn specs_for(computers: &[String], default: &str, chosen: bool) -> Vec<Value
|
|||
vec![json!({
|
||||
"name": TOOL,
|
||||
"description": format!(
|
||||
"用滑鼠鍵盤操作一台電腦的螢幕（{list}）。action 選一種：screenshot（看畫面：回傳截圖＋編號控件清單，結果頭兩行是座標對應，x、y 要用那張圖的座標）; zoom（看小字：給 x,y,width,height 框一块）; show（把畫面放進聊天給人看）; click（ref=截圖編號、name=控件名字【優先：如 name=「4」、name=「=」】、或 x,y）; typing（value 必填；name=先點那個欄位、append=true 接在原文後面、submit=true 按 Enter）; press_key（key 如 Return/Escape/Tab/cmd+s，一次一個組合鍵；要按住給 hold_duration_ms）; scroll（direction=down/up、amount 約幾格、可給 x,y）; mouse_move（移滑鼠並顯示游標）; drag（path=至少兩點 [{{x,y}}…] 拖過的路徑）; open_app（app：Finder/系統設定/計算機/文字編輯/Safari）; batch（你已經從畫面看懂的連續 2-16 步一次做完，中途不能看畫面；每項 {{action:…}} 加各自參數；要跟上一步結果的步不要放）; check_permissions。macOS 一律優先 ref/name，x、y 只給沒有編號的東西。每次看畫面用一句話說畫面顯示什麼、這步成功與否；結果說「畫面和上一步完全一樣」就是沒點到，換編號換方法，不要重複同一步。",
|
||||
"用滑鼠鍵盤操作一台電腦的螢幕（{list}）。action 選一種：screenshot（看畫面：回傳截圖＋編號控件清單，結果頭兩行是座標對應，x、y 要用那張圖的座標）; zoom（看小字：給 x,y,width,height 框一块）; show（把畫面放進聊天給人看）; click（ref=截圖編號、name=控件名字【優先：如 name=「4」、name=「=」】、或 x,y）; typing（value 必填；name=先點那個欄位、append=true 接在原文後面、submit=true 按 Enter）; press_key（key 如 Return/Escape/Tab/cmd+s，一次一個組合鍵；要按住給 hold_duration_ms）; scroll（direction=down/up、amount 約幾格、可給 x,y）; mouse_move（移滑鼠並顯示游標）; drag（path=至少兩點 [{{x,y}}…] 拖過的路徑）; open_app（app：Finder/系統設定/計算機/文字編輯/Safari）; batch（你已經從畫面看懂的連續 2-16 步一次做完，中途不能看畫面；每項 {{action:…}} 加各自參數；要跟上一步結果的步不要放）; intent（系統捷徑：音量/Wi-Fi/鎖螢幕/深淺色/清空資源筒这类事一步做到，電腦現況有列「系統捷徑」就先用它，勝過 GUI 點擊；參數 intent_name＋intent_arg）; check_permissions。macOS 一律優先 ref/name，x、y 只給沒有編號的東西。每次看畫面用一句話說畫面顯示什麼、這步成功與否；結果說「畫面和上一步完全一樣」就是沒點到，換編號換方法，不要重複同一步。",
|
||||
list = list
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": { "type": "string", "enum": ["screenshot", "zoom", "show", "click", "typing", "press_key", "scroll", "mouse_move", "drag", "open_app", "batch", "check_permissions"] },
|
||||
"action": { "type": "string", "enum": ["screenshot", "zoom", "show", "click", "typing", "press_key", "scroll", "mouse_move", "drag", "open_app", "batch", "check_permissions", "intent"] },
|
||||
"computer": { "type": "string", "description": "哪一台電腦（列表裡的名字）。" },
|
||||
"ref": { "type": "string", "description": "截圖左邊紅框的編號。" },
|
||||
"name": { "type": "string", "description": "控件的名字（優先於座標）。" },
|
||||
|
|
@ -345,6 +346,8 @@ pub fn specs_for(computers: &[String], default: &str, chosen: bool) -> Vec<Value
|
|||
"amount": { "type": "integer", "minimum": 1, "description": "scroll 約幾格。" },
|
||||
"path": { "type": "array", "items": { "type": "object", "properties": { "x": { "type": "number" }, "y": { "type": "number" } }, "required": ["x", "y"] }, "description": "drag 的點陣（至少兩點，用截圖座標）。" },
|
||||
"app": { "type": "string", "description": "open_app 的軟體名。" },
|
||||
"intent_name": { "type": "string", "description": "intent 的捷徑名（看電腦現況的「系統捷徑」清單）。" },
|
||||
"intent_arg": { "type": "string", "description": "intent 的 arg（照清單寫法，如 set 40、on、toggle；有的不用）。" },
|
||||
"actions": { "type": "array", "items": { "type": "object",
|
||||
"properties": {
|
||||
"action": { "type": "string", "enum": ["click", "typing", "press_key", "scroll", "mouse_move", "drag", "open_app"] },
|
||||
|
|
@ -385,6 +388,7 @@ pub fn top_action_tool(action: &str) -> Option<&'static str> {
|
|||
"open_app" => "computer_use_open_app",
|
||||
"batch" => "computer_use_batch",
|
||||
"check_permissions" => "computer_use_check_permissions",
|
||||
"intent" => "computer_use_intent",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
|
@ -1329,6 +1333,9 @@ fn run_inner(name: &str, v: &Value) -> Result<(String, Option<String>), String>
|
|||
if name == "computer_use_wait_for" {
|
||||
return Ok((wait_for_tool(v)?, None));
|
||||
}
|
||||
if name == "computer_use_intent" {
|
||||
return crate::intents::tool(v);
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
if !crate::desk::available() {
|
||||
return Err(crate::desk::describe().into());
|
||||
|
|
|
|||
|
|
@ -142,6 +142,11 @@ pub fn describe(name: &str, f: &Facts, own_box: bool, mac_sandbox: bool) -> Stri
|
|||
let more = if f.apps.len() > MAX_APPS { "…" } else { "" };
|
||||
parts.push(format!("裝了的 App：{}{more}", f.apps.iter().take(MAX_APPS).cloned().collect::<Vec<_>>().join("、")));
|
||||
}
|
||||
// The one-step doors (C6): advertised here so the model takes them instead of clicking.
|
||||
let shortcuts = crate::intents::facts_line(&f.kernel, !f.wm.is_empty() || !f.desktop.is_empty(), own_box);
|
||||
if !shortcuts.is_empty() {
|
||||
parts.push(shortcuts);
|
||||
}
|
||||
format!("「{name}」：{}。", parts.join("；"))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,203 @@
|
|||
//! 捷徑優先於 GUI (C6): the operations a person does with one key press, registered as one
|
||||
//! step for the bot.
|
||||
//!
|
||||
//! The bench said it plainly: 「把音量調小」 took 9 model steps through the GUI (about two
|
||||
//! minutes) and one shell command (four seconds). Volume, Wi-Fi, the lock, dark mode — these
|
||||
//! have supported command-line doors; a bot that clicks menus is a bot that didn't know the
|
||||
//! door existed. So the registry here is the knowledge, the computer's facts line advertises
|
||||
//! it every turn (cached by the prompt cache), and the computer tool's `intent` action walks
|
||||
//! through: one call, one command, no screenshot.
|
||||
//!
|
||||
//! A script is a constant chosen from a validated name and argument — the model never writes
|
||||
//! the command, it picks a name. That is why this can run where clicks run, with the same
|
||||
//! approval, and why 「體積 set 999; rm -rf /」 is not a thing: the argument is checked against
|
||||
//! the intent's grammar (a number 0–100, on/off, dark/light) before any script is built.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
/// One registered shortcut: what it's called, how to say it, which machines have it.
|
||||
pub struct Intent {
|
||||
pub name: &'static str,
|
||||
/// How to call it, shown in the computer's facts line.
|
||||
pub usage: &'static str,
|
||||
/// Does this computer have it? kernel: "Darwin"/"Linux"; desktop: a window system is
|
||||
/// running (a headless box has no volume knob the person hears).
|
||||
pub has: fn(kernel: &str, desktop: bool) -> bool,
|
||||
}
|
||||
|
||||
pub const INTENTS: &[Intent] = &[
|
||||
Intent {
|
||||
name: "volume",
|
||||
usage: "volume（音量：arg=set 0-100 / up / down / mute / unmute）",
|
||||
has: |k, desktop| matches!(k, "Darwin") || (k == "Linux" && desktop),
|
||||
},
|
||||
Intent {
|
||||
name: "wifi",
|
||||
usage: "wifi（Wi-Fi 開關：arg=on / off）",
|
||||
has: |k, desktop| matches!(k, "Darwin") || (k == "Linux" && desktop),
|
||||
},
|
||||
Intent {
|
||||
name: "lock",
|
||||
usage: "lock（鎖螢幕/關螢幕：不用 arg）",
|
||||
has: |k, desktop| matches!(k, "Darwin") || (k == "Linux" && desktop),
|
||||
},
|
||||
Intent {
|
||||
name: "dark",
|
||||
usage: "dark（深淺色：arg=dark / light / toggle）",
|
||||
has: |k, desktop| matches!(k, "Darwin") || (k == "Linux" && desktop),
|
||||
},
|
||||
Intent {
|
||||
name: "trash",
|
||||
usage: "trash（清空資源筒/垃圾桶：不用 arg）",
|
||||
has: |k, desktop| matches!(k, "Darwin") || (k == "Linux" && desktop),
|
||||
},
|
||||
Intent {
|
||||
name: "eject",
|
||||
usage: "eject（退出所有可退出磁碟：不用 arg，Mac）",
|
||||
has: |k, _| k == "Darwin",
|
||||
},
|
||||
];
|
||||
|
||||
pub fn find(name: &str) -> Option<&'static Intent> {
|
||||
INTENTS.iter().find(|i| i.name == name)
|
||||
}
|
||||
|
||||
/// The names this kind of computer answers to — the facts line and the tool's error both
|
||||
/// speak this.
|
||||
pub fn names_for(kernel: &str, desktop: bool) -> Vec<&'static str> {
|
||||
INTENTS.iter().filter(|i| (i.has)(kernel, desktop)).map(|i| i.name).collect()
|
||||
}
|
||||
|
||||
/// The sentence in a computer's facts: names and grammar, so the model can pick without
|
||||
/// guessing. Empty for a headless box (its "volume" is nobody's volume).
|
||||
pub fn facts_line(kernel: &str, desktop: bool, own_box: bool) -> String {
|
||||
if own_box {
|
||||
return String::new();
|
||||
}
|
||||
let list = names_for(kernel, desktop);
|
||||
if list.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
let usages: Vec<&str> = INTENTS.iter().filter(|i| list.contains(&i.name)).map(|i| i.usage).collect();
|
||||
format!("系統捷徑（有捷徑先捷徑，一步完成，不要用 GUI 點擊；用 computer 工具 action=intent，參數 intent_name＋intent_arg）：{}", usages.join("；"))
|
||||
}
|
||||
|
||||
fn is_num(s: &str) -> Option<u32> {
|
||||
let n: u32 = s.parse().ok()?;
|
||||
if n <= 100 {
|
||||
Some(n)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// The script for one validated pick. Constants and (checked) numbers only — the model's
|
||||
/// words never reach the shell.
|
||||
pub fn script(name: &str, arg: &str) -> Result<String, String> {
|
||||
if find(name).is_none() {
|
||||
return Err(format!("沒有捷徑「{name}」。"));
|
||||
}
|
||||
let a = arg.trim();
|
||||
let mac = |s: &str| format!("case \"$(uname -s)\" in Darwin) {s} ;; Linux) ");
|
||||
Ok(match (name, a) {
|
||||
("volume", "up") => format!(
|
||||
"{}osascript -e 'set u to (output volume of (get volume settings)) + 15' -e 'if u > 100 then set u to 100' -e 'set volume output volume u' ;; pactl set-sink-volume @DEFAULT_SINK@ +15% 2>/dev/null || amixer -q sset Master 15%+ ;; esac)",
|
||||
mac("")
|
||||
),
|
||||
("volume", "down") => format!(
|
||||
"{}osascript -e 'set u to (output volume of (get volume settings)) - 15' -e 'if u < 0 then set u to 0' -e 'set volume output volume u' ;; pactl set-sink-volume @DEFAULT_SINK@ -15% 2>/dev/null || amixer -q sset Master 15%- ;; esac)",
|
||||
mac("")
|
||||
),
|
||||
("volume", "mute") => format!(
|
||||
"{}osascript -e 'set m to muted of (get volume settings)' -e 'set volume output muted (not m)' ;; pactl set-sink-mute @DEFAULT_SINK@ toggle 2>/dev/null || amixer -q sset Master toggle ;; esac)",
|
||||
mac("")
|
||||
),
|
||||
("volume", "unmute") => format!("{}osascript -e 'set volume output muted false' ;; pactl set-sink-mute @DEFAULT_SINK@ 0 2>/dev/null || amixer -q sset Master unmute ;; esac)", mac("")),
|
||||
("volume", set) => {
|
||||
// "set 40" as the facts line teaches; a bare "40" is kindly taken too.
|
||||
let Some(n) = is_num(set.strip_prefix("set ").unwrap_or(set).trim()) else {
|
||||
return Err("volume 的 arg 要写成 set 0-100（如 set 40）或 up/down/mute/unmute。".into());
|
||||
};
|
||||
format!("{}osascript -e 'set volume output volume {n}' ;; pactl set-sink-volume @DEFAULT_SINK@ {n}% 2>/dev/null || amixer -q sset Master {n}% ;; esac)", mac(""))
|
||||
}
|
||||
("wifi", "on") => format!("{}networksetup -setairportpower en0 on ;; nmcli radio wifi on 2>/dev/null || rfkill unblock wifi ;; esac)", mac("")),
|
||||
("wifi", "off") => format!("{}networksetup -setairportpower en0 off ;; nmcli radio wifi off 2>/dev/null || rfkill block wifi ;; esac)", mac("")),
|
||||
("lock", "") => format!("{}pmset displaysleepnow ;; loginctl lock-session 2>/dev/null || xset dpms force off ;; esac)", mac("")),
|
||||
("dark", "dark") => format!("{}osascript -e 'tell application \"System Events\" to tell appearance preferences to set dark mode to true' ;; gsettings set org.gnome.desktop.interface color-scheme prefer-dark ;; esac)", mac("")),
|
||||
("dark", "light") => format!("{}osascript -e 'tell application \"System Events\" to tell appearance preferences to set dark mode to false' ;; gsettings set org.gnome.desktop.interface color-scheme prefer-light ;; esac)", mac("")),
|
||||
("dark", "toggle") => format!(
|
||||
"{}osascript -e 'tell application \"System Events\" to tell appearance preferences to set dark mode to not (dark mode)' ;; gsettings set org.gnome.desktop.interface color-scheme \"$( [ \"$(gsettings get org.gnome.desktop.interface color-scheme)\" = \"'prefer-dark'\" ] && echo prefer-light || echo prefer-dark )\" ;; esac)",
|
||||
mac("")
|
||||
),
|
||||
("trash", "") => format!("{}osascript -e 'tell application \"Finder\" to empty trash' ;; rm -rf \"$HOME/.local/share/Trash/files\" \"$HOME/.local/share/Trash/info\" ;; esac)", mac("")),
|
||||
("eject", "") => format!("{}osascript -e 'tell application \"Finder\" to eject (every disk whose ejectable is true)' ;; echo 只有 Mac 有這個捷徑 ;; esac)", mac("")),
|
||||
(_, "") => return Err(format!("「{name}」不需要 arg。")),
|
||||
(_, _) => return Err(format!("「{name}」的 arg 不對。看捷徑說明：{}", find(name).map(|i| i.usage).unwrap_or(""))),
|
||||
})
|
||||
}
|
||||
|
||||
/// The computer tool's `intent` action: pick, build, run, report. Runs where the computer
|
||||
/// tool runs — the bot's box is excluded by the facts line (it advertises no shortcuts).
|
||||
pub fn tool(v: &Value) -> Result<(String, Option<String>), String> {
|
||||
let name = v["intent_name"].as_str().unwrap_or("").trim();
|
||||
let arg = v["intent_arg"].as_str().unwrap_or("").trim();
|
||||
if name.is_empty() {
|
||||
return Err(format!("要 intent_name。這台電腦的捷徑：{}", INTENTS.iter().map(|i| i.name).collect::<Vec<_>>().join("、")));
|
||||
}
|
||||
let sh = script(name, arg)?;
|
||||
let out = std::process::Command::new("/bin/sh").arg("-c").arg(&sh).env("PATH", crate::cloudbox::tool_path()).output().map_err(|e| format!("捷徑跑不起來：{e}"))?;
|
||||
let text = format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)).trim().to_string();
|
||||
if out.status.success() {
|
||||
Ok((format!("捷徑 {name}{}完成。{}", if arg.is_empty() { String::new() } else { format!("（{arg}）") }, if text.is_empty() { String::new() } else { format!(" 系統說：{text}") }), None))
|
||||
} else {
|
||||
Ok((format!("捷徑 {name} 的指令回來了但沒成功（這台電腦可能沒有對應的系統指令）：{text}"), None))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_mac_and_a_linux_desktop_get_their_shortcuts() {
|
||||
let mac = names_for("Darwin", true);
|
||||
for n in ["volume", "wifi", "lock", "dark", "trash", "eject"] {
|
||||
assert!(mac.contains(&n), "{n}");
|
||||
}
|
||||
let linux = names_for("Linux", true);
|
||||
assert!(linux.contains(&"volume") && !linux.contains(&"eject"));
|
||||
// A headless box advertises none: nobody hears its volume. The facts line says so.
|
||||
assert!(names_for("Linux", false).is_empty());
|
||||
assert!(facts_line("Linux", true, true).is_empty(), "the bot's own box: no shortcuts");
|
||||
let line = facts_line("Darwin", true, false);
|
||||
for want in ["系統捷徑", "volume", "intent_name", "不要用 GUI 點擊", "eject"] {
|
||||
assert!(line.contains(want), "{want}: {line}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn words_never_reach_the_shell() {
|
||||
// The model picks a name; the script is built from constants and checked numbers.
|
||||
assert!(script("volume", "set 40").unwrap().contains("output volume 40"));
|
||||
for bad in ["set 101", "set -5", "set 40; rm -rf /", "set 40$(id)", "set x", "999", "up; reboot"] {
|
||||
assert!(script("volume", bad).is_err(), "{bad} must not build a script");
|
||||
}
|
||||
assert!(script("wifi", "on").unwrap().contains("networksetup"));
|
||||
assert!(script("wifi", "on; rm -rf /").is_err());
|
||||
assert!(script("lock", "").unwrap().contains("pmset"));
|
||||
assert!(script("lock", "now").is_err(), "lock takes no arg");
|
||||
assert!(script("dark", "toggle").unwrap().contains("dark mode to not"));
|
||||
assert!(script("nope", "").is_err());
|
||||
// The tool says which names exist when the name is missing.
|
||||
let e = tool(&serde_json::json!({})).unwrap_err();
|
||||
assert!(e.contains("volume") && e.contains("intent_name"), "{e}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_shortcut_runs_here_and_says_so() {
|
||||
// This machine is Linux: volume goes through pactl/amixer; with neither in the
|
||||
// sandbox the script still returns an honest sentence, never a panic.
|
||||
let (text, _) = tool(&serde_json::json!({"intent_name": "lock"})).unwrap();
|
||||
assert!(text.starts_with("捷徑 lock"), "{text}");
|
||||
}
|
||||
}
|
||||
|
|
@ -29,6 +29,7 @@ mod desk;
|
|||
mod egress;
|
||||
mod facts;
|
||||
mod httpd;
|
||||
mod intents;
|
||||
mod i18n;
|
||||
mod i18n_table;
|
||||
mod icons;
|
||||
|
|
|
|||
Loading…
Reference in New Issue