AnyBot/src/main.rs

277 lines
11 KiB
Rust
Raw Normal View History

mod agent;
mod app;
mod appctl;
mod attach;
mod axread;
mod avatar;
mod botbrowser;
mod browser;
mod cloudbox;
mod channels;
mod computer;
mod desktop;
mod diagram;
mod platform;
mod mic;
mod pages;
#[cfg(target_os = "linux")]
mod x11;
#[cfg(target_os = "linux")]
mod wl;
#[cfg(target_os = "linux")]
mod portal;
#[cfg(target_os = "linux")]
mod desk;
mod egress;
mod httpd;
mod i18n;
mod i18n_table;
mod icons;
mod macapps;
mod mcp;
mod memory;
mod model;
mod notify;
mod phone;
mod policy;
mod preview;
mod providers;
mod remote;
mod routines;
mod setup;
mod teach;
mod share;
mod teams;
mod theme;
mod tools;
mod triggers;
mod vault;
mod vault_store;
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
mod vnc;
mod voice;
mod tunnel;
mod web;
mod wire;
use eframe::egui;
use std::path::PathBuf;
/// PingFang ships as a downloadable system asset on macOS; its folder name varies per Mac.
fn pingfang_path() -> Option<PathBuf> {
let root = std::path::Path::new("/System/Library/AssetsV2");
for set in std::fs::read_dir(root).ok()?.flatten() {
if !set.file_name().to_string_lossy().starts_with("com_apple_MobileAsset_Font") {
continue;
}
for asset in std::fs::read_dir(set.path()).into_iter().flatten().flatten() {
let p = asset.path().join("AssetData/PingFang.ttc");
if p.exists() {
return Some(p);
}
}
}
None
}
/// PingFang TC Regular inside PingFang.ttc.
const PINGFANG_TC_INDEX: u32 = 2;
/// The window's first size (screenshot mode can ask for another, e.g. ANYBOT_SHOT_SIZE=760x560).
fn shot_size() -> [f32; 2] {
std::env::var("ANYBOT_SHOT_SIZE").ok().and_then(|v| {
let (w, h) = v.split_once('x')?;
Some([w.parse().ok()?, h.parse().ok()?])
}).unwrap_or([1180.0, 780.0])
}
/// 粉圓體, bundled.
pub const HUNINN: &[u8] = include_bytes!("../fonts/jf-openhuninn-2.1.ttf");
/// "huninn" (粉圓體, the default) or "system".
pub fn font_choice() -> String {
model::State::path().parent().and_then(|d| std::fs::read_to_string(d.join("font.txt")).ok()).map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).unwrap_or_else(|| "huninn".into())
}
pub fn load_fonts(ctx: &egui::Context) {
let mut fonts = egui::FontDefinitions::default();
// One font for Chinese *and* English: mixing egui's built-in Latin font with a Chinese
// fallback put the two on different baselines and sizes ("Bot 設定" looked broken).
// So a CJK font that has good Latin letters goes first; egui's own fonts stay behind it
// only for symbols and emoji it lacks.
let mut candidates: Vec<(PathBuf, u32)> = pingfang_path().map(|p| vec![(p, PINGFANG_TC_INDEX)]).unwrap_or_default();
candidates.extend(
[
("/System/Library/Fonts/Hiragino Sans GB.ttc", 0),
("/System/Library/Fonts/STHeiti Medium.ttc", 0),
("/System/Library/Fonts/Supplemental/Arial Unicode.ttf", 0),
("C:\\Windows\\Fonts\\msjh.ttc", 0),
("C:\\Windows\\Fonts\\msyh.ttc", 0),
// Linux: Noto Sans CJK (face 3 = Traditional Chinese) on Debian/Ubuntu, Arch, Fedora; then WenQuanYi.
("/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", 3),
("/usr/share/fonts/noto-cjk/NotoSansCJK-Regular.ttc", 3),
("/usr/share/fonts/google-noto-cjk/NotoSansCJK-Regular.ttc", 3),
("/usr/share/fonts/truetype/wqy/wqy-microhei.ttc", 0),
]
.into_iter()
.map(|(p, i)| (PathBuf::from(p), i)),
);
// Code: the original's `--cursor-font-family-mono` list is ui-monospace, SFMono-Regular, Menlo…
for path in ["/System/Library/Fonts/SFNSMono.ttf", "/System/Library/Fonts/Menlo.ttc"] {
if let Ok(bytes) = std::fs::read(path) {
fonts.font_data.insert("mono".into(), egui::FontData::from_owned(bytes).into());
fonts.families.entry(egui::FontFamily::Monospace).or_default().insert(0, "mono".into());
break;
}
}
for (path, index) in candidates {
if let Ok(bytes) = std::fs::read(&path) {
let mut data = egui::FontData::from_owned(bytes);
data.index = index;
fonts.font_data.insert("cjk".into(), data.into());
fonts.families.entry(egui::FontFamily::Proportional).or_default().insert(0, "cjk".into());
// Code stays in the monospace font; Chinese inside code falls back to the same font.
fonts.families.entry(egui::FontFamily::Monospace).or_default().push("cjk".into());
break;
}
}
// 粉圓體 (jf open 粉圓, SIL OFL 1.1, fonts/OFL.txt) goes first everywhere it has the letter;
// the system font above fills in what it lacks (some simplified characters, symbols).
if font_choice() == "huninn" {
fonts.font_data.insert("huninn".into(), egui::FontData::from_static(HUNINN).into());
fonts.families.entry(egui::FontFamily::Proportional).or_default().insert(0, "huninn".into());
let mono = fonts.families.entry(egui::FontFamily::Monospace).or_default();
let at = mono.iter().position(|f| f == "cjk").unwrap_or(mono.len());
mono.insert(at, "huninn".into());
}
ctx.set_fonts(fonts);
}
/// The window icon (Linux taskbars and window switchers; macOS uses the app bundle's).
fn app_icon() -> egui::IconData {
svg_icon(include_str!("../icons/anybot.svg"), 256)
}
/// An SVG drawn to a square RGBA picture of `px` pixels.
fn svg_icon(svg: &str, px: u32) -> egui::IconData {
let tree = resvg::usvg::Tree::from_str(svg, &resvg::usvg::Options::default()).expect("icon svg");
let mut pm = resvg::tiny_skia::Pixmap::new(px, px).expect("icon pixmap");
let s = px as f32 / tree.size().width();
resvg::render(&tree, resvg::tiny_skia::Transform::from_scale(s, s), &mut pm.as_mut());
// tiny-skia is premultiplied; IconData wants straight alpha.
let mut rgba = pm.take();
for c in rgba.chunks_exact_mut(4) {
let a = c[3] as u32;
if a > 0 && a < 255 {
for v in &mut c[..3] {
*v = ((*v as u32 * 255 + a / 2) / a).min(255) as u8;
}
}
}
egui::IconData { rgba, width: px, height: px }
}
/// `anybot --write-icon out.png [size] [file.svg]`: the app icon (or that SVG) as a PNG, drawn at
/// that size (for the Mac bundle, Linux packages and the web icons).
fn write_icon(out: &str, px: u32, svg: Option<&str>) -> Result<(), String> {
let icon = match svg {
Some(path) => svg_icon(&std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}"))?, px),
None => svg_icon(include_str!("../icons/anybot.svg"), px),
};
let img = image::RgbaImage::from_raw(icon.width, icon.height, icon.rgba).ok_or("icon")?;
img.save(out).map_err(|e| e.to_string())
}
fn main() -> eframe::Result<()> {
let args: Vec<String> = std::env::args().collect();
let screenshot = args.iter().position(|a| a == "--screenshot").and_then(|i| args.get(i + 1)).map(PathBuf::from);
// For tests that start AnyBots of their own: a whole settings file (the sample bots) to edit.
if args.get(1).map(String::as_str) == Some("--sample-state") {
let out = args.get(2).cloned().unwrap_or_else(|| "state.json".into());
let mut st = model::State::samples();
st.remote = Default::default();
if let Err(e) = std::fs::write(&out, serde_json::to_vec_pretty(&st).unwrap_or_default()) {
eprintln!("{e}");
std::process::exit(1);
}
return Ok(());
}
if args.get(1).map(String::as_str) == Some("--write-icon") {
let out = args.get(2).cloned().unwrap_or_else(|| "anybot.png".into());
let px = args.get(3).and_then(|v| v.parse().ok()).unwrap_or(256);
if let Err(e) = write_icon(&out, px, args.get(4).map(String::as_str)) {
eprintln!("{e}");
std::process::exit(1);
}
return Ok(());
}
One way to act on a screen: a real click or keystroke, wherever the bot looked - Acting by an app's number (read_app's e7) is now a real click at that control's spot, the same path as screenshot coordinates. agent-desktop only reads; its own actions (a second set with its own key names and failures: no typing in terminals, "/" and "`" failing off the main thread) are gone, with computer_use_select, background clicks and press_key's app. - Every action comes back with the screen after it (or, with no picture allowed, the front app's numbered controls), so the bot no longer spends a step on a screenshot to check. - Coordinates map without remembering a screenshot: the picture's size depends only on the screen. The phone's taps go straight to the screen as the person's, in the pixels of the picture it was sent; they used to borrow the bot's last screenshot scale and landed 1.25x off on the bots' computer, or failed with "screenshot first". - Mac typing pastes (⌘V, clipboard put back): with 注音 on, text sent as key events vanished while return arrived, so commands typed into a terminal ran empty. Bare printing keys ("slash", "r") are typed the same way; keys like "slash" and "grave" have names. - The bots' Docker computer: one docker exec per step (action, settle, picture), never `mousemove --sync` (it waits forever when the pointer is already there: a second click on the same spot hung two minutes), open_app (chromium = the browser browser_* drive, xterm). Tests: box_screen_is_exact (real Docker: bot and phone coordinates exact, same spot twice in under 2 s, ASCII + Chinese + named key typed), Calculator by numbers and Terminal typing under 注音 through the signed app, remote_e2e 28/28, Linux build + 151 tests + x11_live. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 13:51:19 +00:00
// For testing on a new system: `anybot --tool computer_use_read_app '{"app":"Calculator"}'
// computer_use_click '{"ref":"e7"}' …` runs computer tools in order (one process, so a
// reading's numbers carry over), prints what the bot would hear, and saves the pictures that
// come back as anybot-tool-N.jpg in the temp folder.
if args.get(1).map(String::as_str) == Some("--tool") {
let mut all_ok = true;
for (i, pair) in args[2..].chunks(2).enumerate() {
let json = pair.get(1).cloned().unwrap_or_else(|| "{}".into());
let (text, ok, img) = computer::run(&pair[0], &json);
println!("{} {}: {text}", pair[0], if ok { "ok" } else { "failed" });
if let Some(b64) = img {
use base64::Engine;
let out = std::env::temp_dir().join(format!("anybot-tool-{i}.jpg"));
let _ = std::fs::write(&out, base64::engine::general_purpose::STANDARD.decode(b64).unwrap_or_default());
println!(" saved {}", out.display());
}
all_ok &= ok;
std::thread::sleep(std::time::Duration::from_millis(300));
}
std::process::exit(if all_ok { 0 } else { 1 });
}
// For trying agent-desktop on a new system: `anybot --desktop snapshot '{"app":"Finder"}'`.
if args.get(1).map(String::as_str) == Some("--desktop") {
let cmd = args.get(2).cloned().unwrap_or_else(|| "apps".into());
let v: serde_json::Value = serde_json::from_str(args.get(3).map(String::as_str).unwrap_or("{}")).unwrap_or_default();
match desktop::run(&cmd, &v) {
Ok(out) => println!("{}", serde_json::to_string_pretty(&out).unwrap_or_default()),
Err(e) => {
eprintln!("{e}");
std::process::exit(1);
}
}
std::process::exit(0);
}
// `anybot setup`: make this computer a brain, or hands for a brain (in a terminal).
if args.get(1).map(String::as_str) == Some("setup") {
std::process::exit(setup::run(&args[2..]));
}
if args.get(1).map(String::as_str) == Some("--ask") {
let text = args[2..].join(" ");
let ctx = egui::Context::default();
app::App::ask_cli(&ctx, &text);
}
// No window: keep the bots working and answer the phone (always-on Mac or server).
if args.iter().any(|a| a == "--serve") {
let ctx = egui::Context::default();
app::App::serve(&ctx);
}
let mut viewport = egui::ViewportBuilder::default()
.with_title("AnyBot")
.with_inner_size(shot_size())
.with_min_inner_size([app::SIDEBAR_MIN_WIDTH + app::CHAT_MIN_WIDTH, 480.0])
.with_app_id("anybot")
.with_icon(app_icon());
// On a Mac the window is one surface to the very top (the grey system title bar didn't match
// a dark window); the traffic lights sit over the sidebar's empty corner.
if cfg!(target_os = "macos") {
viewport = viewport.with_fullsize_content_view(true).with_titlebar_shown(false).with_title_shown(false);
}
let options = eframe::NativeOptions { viewport, ..Default::default() };
eframe::run_native(
"AnyBot",
options,
Box::new(move |cc| {
load_fonts(&cc.egui_ctx);
Ok(Box::new(app::App::new(&cc.egui_ctx, screenshot)))
}),
)
}