AnyBot/src/app/computers_ui.rs

1271 lines
64 KiB
Rust
Raw Normal View History

// The computers part of the window: 「Bot 的電腦」 (status, live screen, take control, rebuilds,
// snapshots, secrets, where it runs), 「你的電腦」 (macOS permissions, the per-app grants) and
// the floating bar shown while a bot drives this Mac.
//
// Wording follows the original's strings where it has them (ORIGINAL_FEATURES.md §9–§11).
use super::*;
use crate::cloudbox::{self, Rebuild, Run};
/// The floating bar stays this long after the bot's last mouse/keyboard action on this Mac.
const LOCAL_BAR_SECS: u64 = 8;
#[derive(Default)]
pub struct ComputerUi {
/// 其他電腦 is expanded.
pub others_open: bool,
/// The account's other AnyBots (id, name, online), fetched from the relay now and then.
pub peers: Arc<Mutex<Option<Vec<(String, String, bool)>>>>,
pub peers_at: Option<Instant>,
/// 你的電腦's name being edited.
pub renaming_machine: Option<String>,
/// 更新: the first click arms it for 3 s (「再點選一次以確認」).
pub arm_update: Option<Instant>,
/// 要移除「…」嗎? for another computer (its index).
pub confirm_remove: Option<usize>,
/// 要重設 AnyBot 的電腦嗎?
pub confirm_reset: bool,
/// 一律允許的應用程式 is expanded.
pub apps_open: bool,
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
/// 接管 during a hand-off: this app told the bots' computer to let the person have it.
pub control: bool,
pub confirm: Option<Rebuild>,
pub note: Arc<Mutex<String>>,
disk: Arc<Mutex<Option<String>>>,
new_secret: (String, String),
replace_secret: Option<String>,
secret_error: String,
docker_host: Option<String>,
show_log: bool,
teach_name: String,
teach_for: Option<u64>,
teach_error: String,
schedule_edit: String,
pub update_ready: bool,
update_checked: Option<Instant>,
update_flag: Arc<AtomicBool>,
/// The 電腦 card's small live picture of the box's screen.
thumb: Option<egui::TextureHandle>,
thumb_img: Arc<Mutex<Option<egui::ColorImage>>>,
thumb_busy: Arc<AtomicBool>,
thumb_at: Option<Instant>,
}
fn status_color(r: &Run, p: Palette) -> Color32 {
match r {
Run::Running => Color32::from_rgb(0x14, 0xc3, 0x70),
Run::Starting(_) | Run::Rebuilding(..) => Color32::from_rgb(0xf6, 0x94, 0x12),
Run::Failed(_) => p.danger,
_ => p.text3,
}
}
/// A rounded status pill: coloured dot + label.
fn status_pill(ui: &mut egui::Ui, r: &Run, p: Palette) {
let label = r.label();
let font = egui::FontId::proportional(FONT_XS);
let galley = ui.painter().layout_no_wrap(label, font, p.text2);
let size = vec2(galley.size().x + 26.0, 22.0);
let (rect, _) = ui.allocate_exact_size(size, egui::Sense::hover());
ui.painter().rect_filled(rect, 11, p.hover);
let c = rect.left_center() + vec2(11.0, 0.0);
let col = status_color(r, p);
if matches!(r, Run::Starting(_) | Run::Rebuilding(..)) {
let t = ui.input(|i| i.time) as f32;
ui.painter().circle_filled(c, 4.0 + (t * 4.0).sin().abs() * 2.0, col.gamma_multiply(0.35));
ui.ctx().request_repaint_after(std::time::Duration::from_millis(60));
}
ui.painter().circle_filled(c, 3.5, col);
ui.painter().galley(rect.left_center() + vec2(20.0, -galley.size().y / 2.0), galley, p.text2);
}
/// A little monitor drawn with lines (no emoji font needed).
impl App {
/// Called once at start.
pub(super) fn computers_start(&mut self) {
if !self.demo {
cloudbox::start_background();
}
}
// ---------- settings page ----------
pub(super) fn computers_page(&mut self, ui: &mut egui::Ui) {
let p = self.pal;
let run = cloudbox::run_state();
// Check now and then whether this AnyBot carries a newer computer.
if !self.demo && self.cu.update_checked.map_or(true, |t| t.elapsed().as_secs() > 30) {
self.cu.update_checked = Some(Instant::now());
let flag = self.cu.update_flag.clone();
std::thread::spawn(move || flag.store(cloudbox::update_available(), Ordering::SeqCst));
}
self.cu.update_ready = self.cu.update_flag.load(Ordering::SeqCst);
ui.spacing_mut().item_spacing.y = 0.0;
// ── 你的電腦 ── (title: this computer's name, with a pencil to rename it)
let label = if self.state.machine_label.trim().is_empty() { crate::remote::default_host_name() } else { self.state.machine_label.clone() };
let renaming = self.cu.renaming_machine.clone();
let mut rename_to: Option<Option<String>> = None;
let mut keep_buf: Option<String> = None;
let mut start_rename = false;
sec_title(ui, "你的電腦", p, |ui| match renaming {
Some(mut buf) => {
let r = ui.add(egui::TextEdit::singleline(&mut buf).desired_width(180.0).char_limit(100));
if !r.has_focus() && !r.lost_focus() {
r.request_focus();
}
if r.lost_focus() {
let cancel = ui.input(|i| i.key_pressed(Key::Escape));
let v = buf.trim().to_string();
rename_to = Some(if !cancel && !v.is_empty() { Some(v) } else { None });
} else {
keep_buf = Some(buf);
}
}
None => {
if crate::icons::button(ui, crate::icons::Icon::Pencil, 14.0, p.text3, p.hover).on_hover_text(format!("{},{label}", t("重新命名這台電腦"))).clicked() {
start_rename = true;
}
ui.label(RichText::new(&label).size(FONT_SM).color(p.text2));
}
});
if let Some(buf) = keep_buf {
self.cu.renaming_machine = Some(buf);
}
match rename_to {
Some(Some(v)) => {
self.state.machine_label = v;
self.cu.renaming_machine = None;
self.mark();
}
Some(None) => self.cu.renaming_machine = None,
None => {}
}
if start_rename {
self.cu.renaming_machine = Some(label.clone());
}
sec_card(ui, p, |ui| {
// 在這台電腦上執行: 一律允許 / 每次詢問 / 永不允許 (default 每次詢問).
let locked = !self.managed.local.allows();
let current = if self.state.local_never || locked { "never" } else if self.state.local_execution { "always" } else { "ask" };
let name = |v: &str| {
t(match v {
"always" => "一律允許",
"never" => "永不允許",
_ => "每次詢問",
})
};
let mut pick: Option<&str> = None;
sec_row(ui, p, "在這台電腦上執行", "讓 Bot 在你的電腦上開啟檔案並執行任務。自動審核仍會先檢查所有操作。", |ui| {
ui.add_enabled_ui(!locked, |ui| {
egui::ComboBox::from_id_salt("local-permission").selected_text(name(current)).width(110.0).show_ui(ui, |ui| {
for v in ["always", "ask", "never"] {
if ui.selectable_label(current == v, name(v)).clicked() {
pick = Some(v);
}
}
});
});
});
if let Some(v) = pick {
self.state.local_execution = v == "always";
self.state.local_never = v == "never";
self.mark();
}
if locked {
sec_note(ui, p, "由你的管理員設定,無法變更");
}
// 經這台電腦轉送流量 (the box's new connections leave through this computer).
let allowed = self.managed.egress.allows() && cloudbox::docker_host().is_empty();
let mut via = cloudbox::egress_via_mac() && allowed;
let help = "將 Bot 的電腦的網路流量改經這台電腦而非雲端轉送,例如要連公司內網或 VPN 才看得到的服務。僅對新連線生效。";
let desc = if !self.managed.egress.allows() {
"已由管理員停用".to_string()
} else if !cloudbox::docker_host().is_empty() {
"Bot 的電腦在別的主機上時不能用".to_string()
} else if via {
match (crate::egress::port(), &run) {
(_, Run::Asleep) => "電腦休眠中,喚醒後會重新連線".to_string(),
(Some(_), Run::Running) => format!("本次已轉送 {} 個連線", crate::egress::CONNECTIONS.load(Ordering::Relaxed)),
_ => "等待 Bot 的電腦連線".to_string(),
}
} else {
String::new()
};
let mut egress_err = None;
sec_row_s(ui, p, "經這台電腦轉送流量", &desc, help, |ui| {
if ui.add_enabled(allowed, toggle(&mut via)).changed() {
if let Err(e) = cloudbox::set_egress_via_mac(via) {
egress_err = Some(e);
}
}
});
if let Some(e) = egress_err {
*self.cu.note.lock().unwrap() = e;
}
// 其他電腦: every AnyBot signed in to this account (from the relay), then SSH computers.
let signed_in = !self.state.remote.relay_url.is_empty() && !self.state.remote.host_token.is_empty();
if signed_in && !self.demo && self.cu.peers_at.map_or(true, |t| t.elapsed().as_secs() > 20) {
self.cu.peers_at = Some(Instant::now());
let (slot, settings) = (self.cu.peers.clone(), self.state.remote.clone());
let ctx = ui.ctx().clone();
std::thread::spawn(move || {
let me = settings.host_id.clone();
let list: Vec<_> = crate::remote::devices_online(&settings).into_iter().filter(|(id, _, _)| *id != me).collect();
*slot.lock().unwrap() = Some(list);
ctx.request_repaint();
});
}
let peers: Vec<(String, String, bool)> = if self.demo { vec![("d1".into(), "辦公室的 Mac mini".into(), true), ("d2".into(), "家裡的筆電".into(), false)] } else { self.cu.peers.lock().unwrap().clone().unwrap_or_default() };
let ssh = self.state.remote_computers.clone();
let n = peers.len() + ssh.len();
let mut names: Vec<String> = peers.iter().map(|p| p.1.clone()).collect();
names.extend(ssh.iter().map(|c| c.name.clone()));
let others_help = if signed_in { "在其他電腦安裝 AnyBot 並登入同一個帳號,就會自動出現在這裡。也可以用 SSH 加入。" } else { "登入 AnyBot 帳號(設定 → 一般 → 手機與帳號)後,同一個帳號裡的電腦會自動出現在這裡。也可以用 SSH 加入。" };
let desc = if n == 0 {
String::new()
} else if n == 1 {
match (peers.first(), ssh.first()) {
(Some(p), _) => format!("{} · {}", p.1, t(if p.2 { "目前在線上" } else { "離線" })),
(_, Some(c)) => format!("{} · SSH", c.name),
_ => String::new(),
}
} else {
names.join("、")
};
let open = self.cu.others_open;
let mut flip = false;
sec_row_s(ui, p, "其他電腦", &desc, others_help, |ui| {
let b = egui::Button::new(RichText::new(format!("{n} {}", if open { "⌄" } else { "›" })).size(FONT_SM)).corner_radius(8);
if ui.add(b).clicked() {
flip = true;
}
});
if flip {
self.cu.others_open = !open;
}
if open {
for (_, name, online) in &peers {
let status = t(if *online { "目前在線上" } else { "離線" });
sec_row_s(ui, p, name, &format!("{} · {status}", t("同一個帳號")), "", |ui| {
let (r, _) = ui.allocate_exact_size(vec2(10.0, 10.0), egui::Sense::hover());
ui.painter().circle_filled(r.center(), 3.0, if *online { Color32::from_rgb(0x22, 0xa0, 0x55) } else { p.text3 });
});
}
for (i, c) in ssh.iter().enumerate() {
let mut remove = false;
sec_row_s(ui, p, &c.name, &format!("SSH · {}", c.ssh_target), "", |ui| {
if ui.add(egui::Button::new(RichText::new(t("移除")).color(p.danger)).corner_radius(8)).on_hover_text(format!("{} {}", t("移除"), c.name)).clicked() {
remove = true;
}
});
if remove {
self.cu.confirm_remove = Some(i);
}
}
let mut add = None;
let taken = self.state.computer_names();
Frame::new().inner_margin(Margin::symmetric(14, 8)).show(ui, |ui| {
ui.horizontal(|ui| {
let w = (ui.available_width() - 80.0) / 2.0;
ui.add(egui::TextEdit::singleline(&mut self.new_remote.0).hint_text(t("SSH 電腦名稱,例如 工作站")).desired_width(w - 8.0));
ui.add(egui::TextEdit::singleline(&mut self.new_remote.1).hint_text("me@studio.local").desired_width(w - 8.0));
let ok = !self.new_remote.0.trim().is_empty() && !self.new_remote.1.trim().is_empty() && !taken.contains(&self.new_remote.0.trim().to_string());
if ui.add_enabled(ok, secondary_button("新增")).clicked() {
add = Some(crate::model::RemoteComputer { name: self.new_remote.0.trim().into(), ssh_target: self.new_remote.1.trim().into(), dir: String::new() });
}
});
});
if let Some(c) = add {
self.state.remote_computers.push(c);
self.new_remote = Default::default();
self.mark();
}
}
// This computer for the account's other AnyBots (裝置連結).
if signed_in {
let mut link = self.state.remote.device_link;
sec_row(ui, p, "讓帳號裡的其他電腦使用這台", "同一個帳號的其他 AnyBot(例如放在伺服器上的)可以看這台的畫面、在這台上做事。每個動作還是會先問你。要讓 Bot 操作別台,就在那台打開這個。要移除一台電腦,在那台登出帳號即可。", |ui| {
ui.add(toggle(&mut link));
});
if link != self.state.remote.device_link {
self.state.remote.device_link = link;
self.mark();
}
}
});
// ── 訊息 (Mac) ──
if cfg!(target_os = "macos") {
sec_title(ui, "訊息", p, |_| {});
let mut send = self.state.messages_send;
let mut no_ask = self.state.messages_no_ask;
sec_card(ui, p, |ui| {
sec_row(ui, p, "傳送訊息", "允許 Bot 以你的身分從這台 Mac 上的「訊息」傳送訊息", |ui| {
ui.add(toggle(&mut send));
});
if send {
sec_row(ui, p, "無需詢問直接傳送", "每則訊息都跳過核准卡片", |ui| {
ui.add(toggle(&mut no_ask));
});
}
});
if send != self.state.messages_send || no_ask != self.state.messages_no_ask {
self.state.messages_send = send;
self.state.messages_no_ask = no_ask;
self.mark();
}
}
// ── AnyBot Computer Use: seeing and using this computer's screen ──
self.mac_permissions_card(ui);
// ── Bot 的電腦 ── (on a screen of a brain, it is the brain's)
if self.mir.on {
sec_title(ui, "Bot 的電腦", p, |_| {});
sec_card(ui, p, |ui| {
let name = self.state.remote.brain_name.clone();
let mut view = false;
sec_row(ui, p, "在大腦上", &format!("每個 Bot 自己的電腦(例如「Master 的電腦」)在大腦「{name}」上,要在那台管理;這裡可以看它的畫面。這台電腦是大腦的手腳:上面的權限決定 Bot 能不能看這台的畫面、操作這台。"), |ui| {
view = ui.add(primary_button("開啟電腦", p)).clicked();
});
if view {
self.open_viewer();
}
});
} else {
sec_title(ui, "Bot 的電腦", p, |ui| status_pill(ui, &run, p));
#[derive(PartialEq)]
enum Act {
None,
Recheck,
Download,
Docker,
Browser,
Wake,
View,
UpdateClick,
Reset,
Snapshot,
AskClean,
Disk,
}
let mut act = Act::None;
let armed = self.cu.arm_update.filter(|t| t.elapsed().as_secs_f32() < 3.0).is_some();
if armed {
ui.ctx().request_repaint_after(std::time::Duration::from_millis(300));
}
let busy_rebuild = matches!(run, Run::Rebuilding(..));
let update_ready = self.cu.update_ready;
let working: Vec<String> = self.streams.keys().filter_map(|id| self.state.bot(*id).map(|b| b.name.clone())).collect();
let mut on = cloudbox::auto_update_at().is_some();
let mut at = cloudbox::auto_update_at().unwrap_or_else(|| self.cu.schedule_edit.clone());
if at.is_empty() {
at = "03:00".into();
}
let mut schedule_changed = false;
let disk = self.cu.disk.lock().unwrap().clone();
let note = self.cu.note.lock().unwrap().clone();
sec_card(ui, p, |ui| {
match &run {
Run::NoDocker(why) => {
sec_row_s(ui, p, "Bot 的電腦", "還沒有 Docker", &format!("{why}\n在那之前,Bot 的電腦是一個封閉的資料夾(沒有桌面)。"), |ui| {
if ui.add(secondary_button("重新檢查")).clicked() {
act = Act::Recheck;
}
if cfg!(target_os = "macos") {
if ui.add(primary_button("下載 OrbStack", p)).clicked() {
act = Act::Download;
}
} else if ui.add(primary_button("安裝 Docker 的說明", p)).clicked() {
act = Act::Docker;
}
});
return;
}
Run::Rebuilding(phase, detail) => {
let steps: &[&str] = if matches!(phase.as_str(), "backing-up" | "moving") { &["backing-up", "creating", "moving"] } else { &["cleaning-up", "wiping", "creating", "restoring"] };
let at = steps.iter().position(|s| s == phase).unwrap_or(0);
Frame::new().inner_margin(Margin::symmetric(14, 10)).show(ui, |ui| {
ui.set_width(ui.available_width());
ui.horizontal(|ui| {
for (i, s) in steps.iter().enumerate() {
let done = i < at;
let on = i == at;
let col = if done || on { p.accent } else { p.text3 };
let icon = if done { crate::icons::Icon::CircleCheck } else if on { crate::icons::Icon::Loader } else { crate::icons::Icon::Minus };
crate::icons::label(ui, icon, cloudbox::phase_label(s), FONT_XS, col);
if i + 1 < steps.len() {
ui.label(RichText::new("—").size(FONT_XS).color(p.text3));
}
}
});
ui.add_space(6.0);
ui.add(egui::ProgressBar::new((at as f32 + 0.5) / steps.len() as f32).desired_height(6.0).fill(p.accent));
ui.add_space(4.0);
hint(ui, format!("{detail}…(開始後就不能取消)"), p);
});
}
_ => {
// The screen: AnyBot has no per-bot overview, so it opens from here.
sec_row(ui, p, "Bot 的電腦", "一台 Linux 電腦,你所有的 Bot 共用,每個 Bot 有自己的資料夾(例如「Master 的電腦」)。一直開著,不會休眠。", |ui| {
if ui.add(primary_button("開啟電腦", p)).on_hover_text(t("看 Bot 的電腦的螢幕,也可以取得控制權自己操作")).clicked() {
act = Act::View;
}
match run {
Run::Running => {}
_ => {
if ui.add(secondary_button("喚醒電腦")).clicked() {
act = Act::Wake;
}
}
}
if ui.add(secondary_button("在瀏覽器開啟")).on_hover_text(t("用 noVNC 全螢幕觀看")).clicked() {
act = Act::Browser;
}
});
}
}
// 更新 Bot 的電腦: two clicks (the first arms it for 3 s).
sec_row(ui, p, "更新 Bot 的電腦", "更新你的 Bot 共用的電腦。你的檔案和登入資訊會保留,但已安裝的應用程式和套件會被移除。所有 Bot 會一起更新。", |ui| {
let b = if armed {
egui::Button::new(RichText::new(t("再點選一次以確認")).color(p.danger)).fill(p.danger.gamma_multiply(0.14)).corner_radius(8).min_size(vec2(64.0, 30.0))
} else {
secondary_button("更新")
};
if ui.add_enabled(update_ready && !busy_rebuild, b).clicked() {
act = Act::UpdateClick;
}
});
// The status line under it.
let (icon, text, accent) = if !update_ready {
(crate::icons::Icon::CircleCheck, t("你的電腦已是最新版本"), true)
} else if !working.is_empty() {
let names = if working.len() > 3 { format!("{}、另外 {} 位", working[..3].join("、"), working.len() - 3) } else { working.join("、") };
(crate::icons::Icon::Clock, format!("更新會停止 {names}"), false)
} else {
(crate::icons::Icon::Update, t("有新版的 Bot 的電腦可以更新"), true)
};
Frame::new().inner_margin(Margin { left: 14, right: 14, top: 0, bottom: 8 }).show(ui, |ui| {
let fill = if accent { p.accent.gamma_multiply(0.09) } else { p.hover };
let stroke = if accent { Stroke::new(0.5, p.accent.gamma_multiply(0.6)) } else { Stroke::NONE };
Frame::new().fill(fill).stroke(stroke).corner_radius(8).inner_margin(Margin { left: 12, right: 8, top: 8, bottom: 8 }).show(ui, |ui| {
ui.set_width(ui.available_width());
crate::icons::label(ui, icon, &text, FONT_SM, if accent { p.accent } else { p.text2 });
});
});
// 自動更新 (at a local time each day, an hour to catch up).
let desc = if on { format!("每天 {}", at.trim()) } else { String::new() };
sec_row_s(ui, p, "自動更新", &desc, "有新映像檔可用時,電腦會在你選的時間自動更新;錯過的話一小時內還會補做。開始後不能取消;有 Bot 在用電腦時會等它做完。", |ui| {
let changed = ui.add(toggle(&mut on)).changed();
let r = ui.add(egui::TextEdit::singleline(&mut at).desired_width(56.0).char_limit(5).font(egui::TextStyle::Monospace));
if changed || r.lost_focus() {
schedule_changed = true;
}
});
// 重設 Bot 的電腦: from the last snapshot (red, asks first).
let snap = cloudbox::snapshot_time().map(|t| t.format("%m/%d %H:%M").to_string());
let desc = match &snap { Some(t) => format!("上次快照:{t}"), None => t("還沒有快照") };
sec_row_s(ui, p, "重設 Bot 的電腦", &desc, "電腦卡住時可重新開始。它會從你上次儲存的快照重建,最近的變更可能會遺失;沒有快照時會換一台全新的電腦。「儲存快照」把 Bot 的檔案備份到這台電腦,每天也會自動存一份。", |ui| {
if ui.add_enabled(!busy_rebuild, danger_button("重設", p)).clicked() {
act = Act::Reset;
}
if ui.add_enabled(run.is_running(), secondary_button("儲存快照")).on_hover_text(t("把 Bot 的檔案備份到這台電腦;每天也會自動存一份")).clicked() {
act = Act::Snapshot;
}
});
// 磁碟 (AnyBot).
sec_row(ui, p, "磁碟", "查看 Bot 的電腦用了多少空間,或請主要 Bot 提出清理方案(等你核准才刪)", |ui| {
if ui.add(secondary_button("請 Bot 提出清理方案")).clicked() {
act = Act::AskClean;
}
if ui.add(secondary_button("檢查用量")).clicked() {
act = Act::Disk;
}
});
if let Some(d) = &disk {
Frame::new().inner_margin(Margin::symmetric(14, 6)).show(ui, |ui| {
Frame::new().fill(p.hover).corner_radius(8).inner_margin(Margin::symmetric(10, 8)).show(ui, |ui| {
ui.set_width(ui.available_width());
ui.label(RichText::new(d).monospace().size(11.5).color(p.text2));
});
});
}
if !note.is_empty() {
sec_note(ui, p, &note);
}
});
if schedule_changed && chrono::NaiveTime::parse_from_str(at.trim(), "%H:%M").is_ok() {
cloudbox::set_auto_update_at(on.then(|| at.trim().to_string()));
}
self.cu.schedule_edit = at;
match act {
Act::None => {}
Act::Recheck => cloudbox::refresh(),
Act::Download => crate::preview::open_with_system("https://orbstack.dev/download"),
Act::Docker => crate::preview::open_with_system("https://docs.docker.com/engine/install/"),
Act::View => self.open_viewer(),
Act::Wake => self.box_task(|| cloudbox::ensure_running().map(|_| "Bot 的電腦已經開好了。".to_string())),
Act::Browser => self.box_task(|| {
let url = cloudbox::viewer_url(false)?;
crate::preview::open_with_system(&url);
Ok(String::new())
}),
Act::UpdateClick => {
if armed {
self.cu.arm_update = None;
let ids: Vec<u64> = self.streams.keys().copied().collect();
for id in ids {
self.stop(id);
}
*self.cu.note.lock().unwrap() = match cloudbox::rebuild(Rebuild::Update) {
Ok(()) => t("更新已開始。"),
Err(e) => e,
};
} else {
self.cu.arm_update = Some(Instant::now());
}
}
Act::Reset => self.cu.confirm_reset = true,
Act::Snapshot => self.box_task(|| cloudbox::save_snapshot().map(|n| format!("已儲存快照({})。", cloudbox::human(n)))),
Act::AskClean => self.ask_primary_bot("檢查「Bot 的電腦」的磁碟使用情況,並提出安全的清理方案供我核准。先列出用量最大的地方,不要直接刪除任何東西。"),
Act::Disk => {
let slot = self.cu.disk.clone();
*slot.lock().unwrap() = Some("檢查中…".into());
std::thread::spawn(move || {
let r = cloudbox::disk_report().unwrap_or_else(|e| e);
*slot.lock().unwrap() = Some(r);
});
}
}
// ── 終端機與環境密鑰 ──
if !matches!(run, Run::NoDocker(_)) {
sec_title(ui, "終端機與環境密鑰", p, |_| {});
sec_card(ui, p, |ui| {
Frame::new().inner_margin(Margin::symmetric(14, 8)).show(ui, |ui| {
ui.set_width(ui.available_width());
ui.spacing_mut().item_spacing.y = 4.0;
ui.horizontal(|ui| {
ui.spacing_mut().item_spacing.x = 4.0;
ui.label(RichText::new(t("給指令用的帳密")).size(FONT_SM).color(p.text2));
help_mark(ui, "例如 CRM_API_TOKEN。Bot 執行指令時會變成環境變數;值只能寫入,不能讀出。", p);
});
let names = cloudbox::secret_names();
let mut remove = None;
for n in &names {
ui.horizontal(|ui| {
ui.label(RichText::new(n).monospace().size(FONT_SM));
ui.label(RichText::new("••••••••").size(FONT_SM).color(p.text3));
ui.with_layout(Layout::right_to_left(Align::Center), |ui| {
if ui.add(egui::Button::new(RichText::new(t("刪除")).size(12.0).color(p.danger)).frame(false)).clicked() {
remove = Some(n.clone());
}
if ui.add(egui::Button::new(RichText::new(t("取代")).size(12.0).color(p.text2)).frame(false)).clicked() {
self.cu.new_secret = (n.clone(), String::new());
self.cu.replace_secret = Some(n.clone());
}
});
});
}
if let Some(n) = remove {
cloudbox::remove_secret(&n);
}
ui.horizontal(|ui| {
let w = (ui.available_width() - 80.0) / 2.0;
let replacing = self.cu.replace_secret.is_some();
ui.add_enabled(!replacing, egui::TextEdit::singleline(&mut self.cu.new_secret.0).hint_text(t("名稱,例如 CRM_API_TOKEN")).desired_width(w - 8.0).font(egui::TextStyle::Monospace));
ui.add(egui::TextEdit::singleline(&mut self.cu.new_secret.1).password(true).hint_text(t("值")).desired_width(w - 8.0));
let ok = !self.cu.new_secret.0.trim().is_empty() && !self.cu.new_secret.1.is_empty();
if ui.add_enabled(ok, primary_button(if replacing { "取代" } else { "新增" }, p)).clicked() {
match cloudbox::set_secret(&self.cu.new_secret.0, &self.cu.new_secret.1) {
Ok(()) => {
self.cu.new_secret = Default::default();
self.cu.replace_secret = None;
self.cu.secret_error.clear();
}
Err(e) => self.cu.secret_error = e,
}
}
});
let up = self.cu.new_secret.0.to_uppercase();
if up != self.cu.new_secret.0 {
self.cu.new_secret.0 = up;
}
if !self.cu.secret_error.is_empty() {
ui.label(RichText::new(&self.cu.secret_error).size(FONT_XS).color(p.danger));
}
});
});
// ── 在哪裡執行 ──
sec_title(ui, "在哪裡執行", p, |_| {});
let current = cloudbox::docker_host();
let mut edit = self.cu.docker_host.take().unwrap_or_else(|| current.clone());
let mut apply = false;
sec_card(ui, p, |ui| {
sec_row(ui, p, "Docker 主機", "預設在這台電腦的 Docker 裡。也可以放到雲端或公司伺服器,例如 ssh://me@server。換位置後會在新的地方重新建立,舊檔案可以用快照搬過去。", |ui| {
if ui.add_enabled(edit.trim() != current, primary_button("套用", p)).clicked() {
apply = true;
}
ui.add(egui::TextEdit::singleline(&mut edit).hint_text(t("這台電腦(預設)")).desired_width(180.0).font(egui::TextStyle::Monospace));
});
});
if apply {
cloudbox::set_docker_host(edit.trim());
}
self.cu.docker_host = Some(edit);
if ui.add(egui::Button::new(RichText::new(t(if self.cu.show_log { "隱藏紀錄" } else { "顯示紀錄" })).size(FONT_XS).color(p.text3)).frame(false)).clicked() {
self.cu.show_log = !self.cu.show_log;
}
if self.cu.show_log {
let lines = cloudbox::log_lines();
Frame::new().fill(p.hover).corner_radius(8).inner_margin(Margin::symmetric(10, 8)).show(ui, |ui| {
ui.set_width(ui.available_width());
if lines.is_empty() {
ui.label(RichText::new(t("(沒有紀錄)")).size(11.5).color(p.text3));
}
for l in lines.iter().rev().take(30) {
ui.label(RichText::new(l).monospace().size(11.0).color(p.text2));
}
});
}
}
}
// ── 示範教學、瀏覽器 ──
ui.add_space(SEC_GAP * 0.5);
ui.spacing_mut().item_spacing.y = 6.0;
self.teach_card(ui);
setting_card(ui, p, |ui| self.browser_profiles_section(ui, p));
self.computers_dialogs(ui.ctx());
}
/// 要移除「…」嗎? and 要重設 AnyBot 的電腦嗎?
fn computers_dialogs(&mut self, ctx: &egui::Context) {
let p = self.pal;
if let Some(i) = self.cu.confirm_remove {
let name = self.state.remote_computers.get(i).map(|c| c.name.clone()).unwrap_or_default();
let mut keep = true;
let mut remove = false;
let close = dialog(ctx, "remove-computer", &format!("要移除「{name}」嗎?"), 420.0, true, p, |ui| {
dialog_body(ui, "remove-computer-body", 200.0, |ui| {
ui.label(t("Bot 將無法再在這台電腦上執行任務。之後可以再用 SSH 加回來。"));
});
dialog_footer(ui, p, |ui| {
ui.with_layout(Layout::right_to_left(Align::Center), |ui| {
if ui.add(danger_button("移除", p)).clicked() {
remove = true;
keep = false;
}
if ui.add(secondary_button("取消")).clicked() {
keep = false;
}
});
});
});
if remove && i < self.state.remote_computers.len() {
self.state.remote_computers.remove(i);
self.mark();
}
if !keep || close {
self.cu.confirm_remove = None;
}
}
if self.cu.confirm_reset {
let working = self.streams.len();
let has_snap = cloudbox::snapshot_time().is_some();
let mut keep = true;
let mut go = false;
let close = dialog(ctx, "reset-computer", "要重設 AnyBot 的電腦嗎?", 460.0, true, p, |ui| {
dialog_body(ui, "reset-computer-body", 320.0, |ui| {
let w = if working == 0 { t("所有執行中的工作都將被中斷。") } else { format!("{working} 個 Bot 正在工作,將會被中斷。") };
ui.label(if has_snap { format!("從上次儲存的快照還原。這會關閉目前的電腦,並以最新版本啟動一台全新的電腦。{w}") } else { format!("還沒有快照:這會換一台全新的電腦,裡面的檔案和登入資訊都會清除。{w}") });
ui.add_space(8.0);
Frame::new().fill(p.danger.gamma_multiply(0.08)).corner_radius(8).inner_margin(Margin::same(10)).show(ui, |ui| {
ui.add(egui::Label::new(RichText::new(t("警告:尚未同步到快照的工作(包括最近的檔案)將會遺失。已安裝的應用程式和套件會被移除。")).size(FONT_SM).color(p.danger)).wrap());
});
});
dialog_footer(ui, p, |ui| {
ui.with_layout(Layout::right_to_left(Align::Center), |ui| {
if ui.add(danger_button("重設 AnyBot 的電腦", p)).clicked() {
go = true;
keep = false;
}
if ui.add(secondary_button("取消")).clicked() {
keep = false;
}
});
});
});
if go {
let ids: Vec<u64> = self.streams.keys().copied().collect();
for id in ids {
self.stop(id);
}
let kind = if has_snap { Rebuild::Restore } else { Rebuild::Reset };
*self.cu.note.lock().unwrap() = match cloudbox::rebuild(kind) {
Ok(()) => t("重設已開始。"),
Err(e) => e,
};
}
if !keep || close {
self.cu.confirm_reset = false;
}
}
}
/// 「AnyBot Computer Use」: what seeing and using this computer's screen needs.
pub(super) fn mac_permissions_card(&mut self, ui: &mut egui::Ui) {
let p = self.pal;
if cfg!(target_os = "linux") {
#[cfg(target_os = "linux")]
{
let kind = crate::desk::kind();
let (state, sub) = match kind {
crate::desk::Kind::X11 => ("已授予", "X11 桌面:可以截圖、點擊、打字,也能錄製示範"),
crate::desk::Kind::Wlr => ("已授予", "Wayland 桌面:可以截圖、點擊、打字(錄製示範需要 Xorg 工作階段)"),
crate::desk::Kind::Portal => ("第一次使用時詢問", "Wayland 桌面:第一次使用時系統會跳出視窗,打開「允許遠端操作」再按「分享」(錄製示範需要 Xorg 工作階段)"),
crate::desk::Kind::None if crate::desk::wayland() => ("無法使用", "找不到 xdg-desktop-portal 的遠端桌面功能,請安裝 xdg-desktop-portal-gnome 或 -kde"),
crate::desk::Kind::None => ("無法使用", "這台電腦沒有顯示器"),
};
sec_title(ui, "AnyBot Computer Use", p, |_| {});
sec_card(ui, p, |ui| {
sec_row(ui, p, "看畫面和操作", sub, |ui| {
ui.label(RichText::new(t(state)).size(FONT_SM).color(p.text2));
});
});
}
self.allowed_apps_section(ui);
return;
}
if !cfg!(target_os = "macos") {
return;
}
let see = crate::computer::screen_capture_ok();
let control = crate::axread::trusted(false);
sec_title(ui, "AnyBot Computer Use", p, |_| {});
let mut open_pane = None;
sec_card(ui, p, |ui| {
let state = |ok: bool| t(if ok { "已授予" } else { "未授予" });
sec_row(ui, p, "螢幕錄製", "讓 Bot 看到你的螢幕", |ui| {
ui.label(RichText::new(state(see)).size(FONT_SM).color(if see { p.text2 } else { p.danger }));
});
sec_row(ui, p, "輔助使用", "讓 Bot 點擊和打字,也用來錄製示範", |ui| {
ui.label(RichText::new(state(control)).size(FONT_SM).color(if control { p.text2 } else { p.danger }));
});
if !see || !control {
sec_row(ui, p, "授予權限", "打開系統設定的面板。把 AnyBot(或「終端機」)加到清單裡並打開。", |ui| {
if ui.add(primary_button("開啟設定", p)).clicked() {
open_pane = Some(if !see { "Privacy_ScreenCapture" } else { "Privacy_Accessibility" });
}
});
}
});
if let Some(pane) = open_pane {
let _ = std::process::Command::new("open").arg(format!("x-apple.systempreferences:com.apple.preference.security?{pane}")).spawn();
}
self.allowed_apps_section(ui);
}
/// 一律允許的應用程式: apps bots may use without asking (per bot), with a remove button.
fn allowed_apps_section(&mut self, ui: &mut egui::Ui) {
let p = self.pal;
let apps: Vec<(u64, String, String)> = self
.state
.bots
.iter()
.flat_map(|b| b.allow_rules.iter().filter_map(|g| g.strip_prefix("app:")).map(|a| (b.id, b.name.clone(), a.to_string())).collect::<Vec<_>>())
.collect();
if apps.is_empty() {
return;
}
let open = self.cu.apps_open;
let mut remove = None;
let mut flip = false;
sec_card(ui, p, |ui| {
sec_row(ui, p, "一律允許的應用程式", "Bot 無需事先詢問即可使用這些應用程式", |ui| {
let b = egui::Button::new(RichText::new(format!("{} 個應用程式 {}", apps.len(), if open { "⌄" } else { "›" })).size(FONT_SM)).corner_radius(8);
if ui.add(b).clicked() {
flip = true;
}
});
if open {
for (id, bot, app) in &apps {
sec_row_s(ui, p, app, &format!("{bot} 可以使用"), "", |ui| {
if crate::icons::button(ui, crate::icons::Icon::Trash, 14.0, p.text3, p.hover).on_hover_text(format!("{} {app}", t("移除"))).clicked() {
remove = Some((*id, format!("app:{app}")));
}
});
}
}
});
if flip {
self.cu.apps_open = !open;
}
if let Some((id, g)) = remove {
if let Some(b) = self.state.bot_mut(id) {
b.allow_rules.retain(|x| *x != g);
}
self.mark();
}
}
fn box_task(&mut self, f: impl FnOnce() -> Result<String, String> + Send + 'static) {
let note = self.cu.note.clone();
*note.lock().unwrap() = "處理中…".into();
std::thread::spawn(move || {
let r = f();
*note.lock().unwrap() = match r {
Ok(s) => s,
Err(e) => e,
};
cloudbox::refresh();
});
}
fn ask_primary_bot(&mut self, text: &str) {
let id = self.state.primary_bot;
self.drafts.insert(id, text.to_string());
self.state.selected = id;
self.show_settings = false;
}
/// The confirm dialog for a rebuild (the original's wording).
pub(super) fn rebuild_dialog(&mut self, ctx: &egui::Context) {
let Some(kind) = self.cu.confirm else { return };
let p = self.pal;
let mut go = false;
let mut close = dialog(ctx, "box-rebuild", &format!("{}?", kind.title()), 420.0, true, p, |ui| {
dialog_body(ui, "box-rebuild-body", 300.0, |ui| {
ui.add(egui::Label::new(RichText::new(kind.warning()).size(FONT_BASE)).wrap());
if kind == Rebuild::Update && cloudbox::is_busy() {
ui.add_space(6.0);
ui.label(RichText::new(t("有 Bot 正在用這台電腦,等它做完再重建。")).size(FONT_XS).color(p.danger));
}
});
dialog_footer(ui, p, |ui| {
ui.with_layout(Layout::right_to_left(Align::Center), |ui| {
let btn = if kind == Rebuild::Reset { danger_button(kind.title(), p) } else { primary_button(kind.title(), p) };
if ui.add(btn).clicked() {
go = true;
}
if ui.add(secondary_button("取消")).clicked() {
go = false;
self.cu.confirm = None;
}
});
});
});
if go {
if kind == Rebuild::Update {
// 更新會先停掉正在工作的 Bot,聊天會保留.
let ids: Vec<u64> = self.streams.keys().copied().collect();
for id in ids {
self.stop(id);
}
}
*self.cu.note.lock().unwrap() = match cloudbox::rebuild(kind) {
Ok(()) => format!("{}已開始。", kind.title()),
Err(e) => e,
};
close = true;
}
if close {
self.cu.confirm = None;
}
}
// ---------- the live screen ----------
/// The box's screen as a small picture, refreshed every few seconds while it is shown (taking
/// it never wakes the box; asleep it shows the last picture).
pub(super) fn box_thumbnail(&mut self, ctx: &egui::Context) -> Option<(egui::TextureId, egui::Vec2)> {
if let Some(img) = self.cu.thumb_img.lock().unwrap().take() {
match &mut self.cu.thumb {
Some(t) => t.set(img, egui::TextureOptions::LINEAR),
None => self.cu.thumb = Some(ctx.load_texture("box-thumb", img, egui::TextureOptions::LINEAR)),
}
}
// On a screen of a brain the bots' computer is the brain's: its picture comes from there.
let brain = self.mir.on.then(|| self.state.remote.clone());
let running = if brain.is_some() { self.mir.box_state["running"].as_bool().unwrap_or(false) } else { cloudbox::run_state().is_running() };
let every = if running { 3 } else { 30 };
if !self.cu.thumb_at.map(|t| t.elapsed().as_secs() < every).unwrap_or(false) && !self.cu.thumb_busy.swap(true, Ordering::SeqCst) {
self.cu.thumb_at = Some(Instant::now());
let (slot, busy, c) = (self.cu.thumb_img.clone(), self.cu.thumb_busy.clone(), ctx.clone());
std::thread::spawn(move || {
let img = match &brain {
Some(s) => crate::remote::sync_screen(s, true).ok().and_then(|b| image::load_from_memory(&b).ok()).map(|i| i.to_rgba8()),
None => cloudbox::thumbnail(320),
};
if let Some(img) = img {
let size = [img.width() as usize, img.height() as usize];
*slot.lock().unwrap() = Some(egui::ColorImage::from_rgba_unmultiplied(size, img.as_raw()));
c.request_repaint();
}
busy.store(false, Ordering::SeqCst);
});
}
ctx.request_repaint_after(std::time::Duration::from_secs(every));
self.cu.thumb.as_ref().map(|t| (t.id(), t.size_vec2()))
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
/// 看畫面: the bots' computer in the browser (noVNC), live, where the person can also take
/// over. On a screen of a brain it is the brain's computer, through the account server (a
/// link made for this once); here it is this computer's own box.
pub(super) fn open_viewer(&mut self) {
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
self.open_screen(false);
}
fn open_screen(&mut self, control: bool) {
self.show_settings = false;
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
let name = self.state.bots.iter().find(|b| b.id == self.state.selected).map(|b| format!("{} 的電腦", b.name)).unwrap_or_else(|| "Bot 的電腦".into());
if self.mir.on {
let s = self.state.remote.clone();
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
self.box_task(move || {
let ticket = crate::remote::screen_ticket(&s)?;
let url = format!("{}/screen#host={}&ticket={ticket}&name={}{}", s.relay_url.trim_end_matches('/'), s.brain_id, crate::httpd::url_encode(&name), if control { "&control=1" } else { "" });
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
crate::preview::open_with_system(&url);
Ok(String::new())
});
return;
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
if self.demo {
return;
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
self.box_task(move || {
let url = cloudbox::viewer_url(control)?;
crate::preview::open_with_system(&url);
Ok(String::new())
});
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
/// 接管: open the computer with control (the bot waits for the person meanwhile).
pub(super) fn take_over_computer(&mut self) {
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
self.open_screen(true);
if self.cu.control {
return;
}
self.cu.control = true;
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
self.hold_computer(true);
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
/// 我完成了 / 跳過: the bot gets its computer back.
pub(super) fn give_back_computer(&mut self) {
if !self.cu.control {
return;
}
self.cu.control = false;
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
self.hold_computer(false);
}
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
fn hold_computer(&self, on: bool) {
if self.mir.on {
let s = self.state.remote.clone();
std::thread::spawn(move || {
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
let _ = crate::remote::sync_post(&s, "screen", &serde_json::json!({"kind": "hold", "on": on}));
});
} else {
People watch and help on the bots' computer through VNC (noVNC), carried by the account server The core: the bot works the computer; a person watches it like a colleague, and takes over when it needs them. VNC is made for that (only what changed, every mouse move and key, touch on phones), so the pictures sent one by one go, and with them their own input path. - vnc.rs: a VNC proxy to the box's x11vnc through one docker exec (a Python bridge that ends with either side). It signs in to the box with the box's own password (its VNC_PASSWORD) and offers the viewer no password, so the password never leaves the brain. The person's clicks and keys make the bot's steps wait until 8 s after they stop; text typed in the viewer's box (any language) is pasted in the box. - Account server: /screen (the viewer page) and /novnc.js (noVNC 1.6.0, MPL-2.0, bundled); /relay/vnc/<host> takes the browser's WebSocket (account session, or a 10-minute ticket the app asks for with POST /relay/screen-ticket; owners and admins) and carries it over that AnyBot's lasting connection (vopen / vdata / vtext / vclose). Both ends of that connection wait 2 ms instead of 40 ms to read while things move: a request and its answer took up to 160 ms more than needed. - Phone page: 看 Bot 的電腦 and 接管 open the viewer page (also on the local network: /api/vnc on the AnyBot's own phone server). This Mac's own screen moved to the ⋯ menu. - Mac app: 看畫面 opens the viewer in the browser (a screen of a brain: the brain's computer by ticket; else this computer's box, noVNC straight from it). The in-app picture viewer, the live JPEG stream, the person's click/move/scroll/key endpoints and the input shell are gone. - Opening a Mac app uses LaunchServices (`open -a`): agent-desktop's launch waited on the window through accessibility and failed without 輔助使用 now and then. Measured on this Mac (made-up account, local relay + headless brain + the real box): viewer signed in in 0.4–0.7 s, a request and its answer through relay + brain + box 25–58 ms median; headless Chrome shows the desktop live in /screen. remote_e2e 28/28 ×2, 149 unit tests, relay 11. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 15:05:44 +00:00
cloudbox::HELD_BY_PERSON.store(on, Ordering::SeqCst);
}
}
pub(super) fn local_control_bar(&mut self, ctx: &egui::Context) {
if self.demo {
return;
}
let held = crate::computer::HELD_BY_PERSON.load(Ordering::SeqCst);
let recent = crate::computer::LAST_ACTION.lock().unwrap().clone().filter(|(t, _)| t.elapsed().as_secs() < LOCAL_BAR_SECS);
if !held && recent.is_none() {
return;
}
let who = recent.map(|r| r.1).filter(|w| !w.is_empty()).unwrap_or_else(|| "Bot".into());
let p = self.pal;
let mut stop_bot: Option<String> = None;
let mut toggle = false;
let monitor_w = ctx.input(|i| i.viewport().monitor_size).map(|s| s.x).unwrap_or(1440.0);
ctx.show_viewport_immediate(
egui::ViewportId::from_hash_of("anybot-local-control"),
egui::ViewportBuilder::default()
.with_title("AnyBot 正在操作你的電腦")
.with_inner_size([440.0, 46.0])
.with_position([(monitor_w - 440.0) / 2.0, 34.0])
.with_decorations(false)
.with_always_on_top()
.with_resizable(false)
.with_transparent(true),
|ui, _| {
Frame::new().fill(p.elevated).stroke(Stroke::new(1.0, p.border)).corner_radius(23).inner_margin(Margin::symmetric(16, 8)).show(ui, |ui| {
ui.set_width(ui.available_width());
ui.horizontal(|ui| {
let (r, _) = ui.allocate_exact_size(vec2(10.0, 10.0), egui::Sense::hover());
ui.painter().circle_filled(r.center(), 4.5, if held { p.text3 } else { Color32::from_rgb(0xfc, 0x2c, 0x3e) });
let text = if held { "你正在操作,Bot 已暫停".to_string() } else { format!("{who} 正在操作你的電腦") };
ui.label(RichText::new(text).size(FONT_SM).strong());
ui.with_layout(Layout::right_to_left(Align::Center), |ui| {
if !held && ui.add(egui::Button::new(RichText::new(t("停止")).size(FONT_XS)).corner_radius(14)).clicked() {
stop_bot = Some(who.clone());
}
let label = if held { "交還給 Bot" } else { "取得控制權" };
if ui.add(egui::Button::new(RichText::new(label).size(FONT_XS).strong().color(p.on_primary)).fill(p.primary).corner_radius(14)).clicked() {
toggle = true;
}
});
});
});
},
);
if toggle {
crate::computer::HELD_BY_PERSON.store(!held, Ordering::SeqCst);
if held {
*crate::computer::LAST_ACTION.lock().unwrap() = None;
}
}
if let Some(name) = stop_bot {
let ids: Vec<u64> = self.streams.keys().copied().filter(|id| self.state.bot(*id).map(|b| b.name == name).unwrap_or(false)).collect();
for id in ids {
self.stop(id);
}
*crate::computer::LAST_ACTION.lock().unwrap() = None;
}
ctx.request_repaint_after(std::time::Duration::from_millis(500));
}
}
// ---------- 示範教學 ----------
impl App {
/// 教它一項任務 from the + menu: start recording right away under a dated name.
pub(super) fn teach_quick(&mut self, bot_id: u64) {
let Some(bot) = self.state.bot(bot_id).cloned() else { return };
if crate::teach::recording().is_some() {
return;
}
self.cu.teach_name = format!("{} {}", t("示範"), chrono::Local::now().format("%m/%d %H:%M"));
match crate::teach::start(&bot.name) {
Ok(()) => {
self.cu.teach_for = Some(bot_id);
self.cu.teach_error.clear();
}
Err(e) => self.att.error = Some(e),
}
}
pub(super) fn teach_card(&mut self, ui: &mut egui::Ui) {
if !crate::computer::demo_supported() {
return;
}
let p = self.pal;
let bot_id = self.state.selected;
let Some(bot) = self.state.bot(bot_id).cloned() else { return };
setting_card(ui, p, |ui| {
setting_title(ui, "示範教學", &format!("錄下你在這台 Mac 上執行任務的過程。{} 會學習這些步驟,之後可以自行再次執行。", bot.name), p);
ui.horizontal(|ui| {
ui.add(egui::TextEdit::singleline(&mut self.cu.teach_name).hint_text(t("這個任務叫什麼,例如 匯出本月報表")).desired_width(ui.available_width() - 110.0));
let ok = !self.cu.teach_name.trim().is_empty() && crate::teach::recording().is_none();
if ui.add_enabled(ok, primary_button("開始錄製", p)).clicked() {
match crate::teach::start(&bot.name) {
Ok(()) => {
self.cu.teach_for = Some(bot_id);
self.cu.teach_error.clear();
self.show_settings = false;
}
Err(e) => self.cu.teach_error = e,
}
}
});
if !self.cu.teach_error.is_empty() {
ui.add(egui::Label::new(RichText::new(&self.cu.teach_error).size(FONT_XS).color(p.danger)).wrap());
}
hint(ui, "按下開始後照平常的方式操作;做完按畫面上方的「完成」。會記錄點擊、打字和捲動,並且每 0.5 秒拍一張螢幕畫面(需要「螢幕與系統錄音」權限),完成後合成錄影;每一步會留一張當時的畫面給 Bot 看。不會擋住或改變你的操作。", p);
let mut remove = None;
for (i, d) in bot.demos.iter().enumerate() {
ui.add_space(6.0);
Frame::new().fill(p.elevated).corner_radius(10).inner_margin(Margin::symmetric(12, 8)).show(ui, |ui| {
ui.set_width(ui.available_width());
ui.horizontal(|ui| {
ui.label(RichText::new(&d.name).strong().size(FONT_SM));
ui.label(RichText::new(format!("{} · {} 個步驟", d.created, d.steps.len())).size(11.5).color(p.text3));
ui.with_layout(Layout::right_to_left(Align::Center), |ui| {
if ui.add(egui::Button::new(RichText::new(t("刪除")).size(12.0).color(p.text3)).frame(false)).clicked() {
remove = Some(i);
}
if !d.video.is_empty() && ui.add(egui::Button::new(RichText::new(t("播放錄影")).size(12.0).color(p.accent)).frame(false)).clicked() {
crate::preview::open_with_system(&d.video.to_string());
}
});
});
egui::CollapsingHeader::new(RichText::new(t("步驟")).size(FONT_XS).color(p.text2)).id_salt(("demo", bot_id, i)).show(ui, |ui| {
for (n, s) in d.steps.iter().enumerate() {
ui.horizontal(|ui| {
ui.label(RichText::new(format!("{}. {s}", n + 1)).size(FONT_XS).color(p.text2));
if let Some(shot) = d.shots.get(n).filter(|x| !x.is_empty()) {
if ui.add(egui::Button::new(RichText::new(t("畫面")).size(11.0).color(p.accent)).frame(false)).on_hover_text(t("打開這一步當時的畫面")).clicked() {
crate::preview::open_with_system(&shot.to_string());
}
}
});
}
});
});
}
if let Some(i) = remove {
if let Some(b) = self.state.bot_mut(bot_id) {
b.demos.remove(i);
}
self.mark();
}
});
}
/// 「{Bot} 正在觀看並學習」 with 完成 / 取消, on top of every window while recording.
pub(super) fn teach_bar(&mut self, ctx: &egui::Context) {
let Some((who, since)) = crate::teach::recording() else { return };
let p = self.pal;
let mut finish = false;
let mut cancel = false;
let monitor_w = ctx.input(|i| i.viewport().monitor_size).map(|s| s.x).unwrap_or(1440.0);
ctx.show_viewport_immediate(
egui::ViewportId::from_hash_of("anybot-teach"),
egui::ViewportBuilder::default()
.with_title("AnyBot 示範教學")
.with_inner_size([420.0, 46.0])
.with_position([(monitor_w - 420.0) / 2.0, 34.0])
.with_decorations(false)
.with_always_on_top()
.with_resizable(false)
.with_transparent(true),
|ui, _| {
Frame::new().fill(p.elevated).stroke(Stroke::new(1.0, p.border)).corner_radius(23).inner_margin(Margin::symmetric(16, 8)).show(ui, |ui| {
ui.set_width(ui.available_width());
ui.horizontal(|ui| {
let (r, _) = ui.allocate_exact_size(vec2(10.0, 10.0), egui::Sense::hover());
let t = ui.input(|i| i.time) as f32;
ui.painter().circle_filled(r.center(), 4.5, Color32::from_rgb(0xfc, 0x2c, 0x3e).gamma_multiply(0.6 + 0.4 * (t * 3.0).sin().abs()));
let secs = since.elapsed().as_secs();
ui.label(RichText::new(format!("{who} 正在觀看並學習 {:02}:{:02}", secs / 60, secs % 60)).size(FONT_SM).strong());
ui.with_layout(Layout::right_to_left(Align::Center), |ui| {
if ui.add(egui::Button::new(RichText::new(crate::i18n::t("取消")).size(FONT_XS)).corner_radius(14)).clicked() {
cancel = true;
}
if ui.add(egui::Button::new(RichText::new(crate::i18n::t("完成")).size(FONT_XS).strong().color(p.on_primary)).fill(p.primary).corner_radius(14)).clicked() {
finish = true;
}
});
});
});
},
);
ctx.request_repaint_after(std::time::Duration::from_millis(250));
if cancel {
crate::teach::cancel();
}
if finish {
let demo = crate::teach::finish(&self.cu.teach_name);
let n = demo.steps.len();
if let Some(id) = self.cu.teach_for.take() {
if let Some(b) = self.state.bot_mut(id) {
b.demos.retain(|d| d.name != demo.name);
b.demos.push(demo);
}
self.notice = Some(format!("已記下「{}」({n} 個步驟)。之後跟 Bot 說「照示範做 {}」就可以。", self.cu.teach_name.trim(), self.cu.teach_name.trim()));
self.state.selected = id;
}
self.cu.teach_name.clear();
self.mark();
}
}
}
/// An on/off switch (the original's toggle, size md).
fn toggle(on: &mut bool) -> impl egui::Widget + '_ {
move |ui: &mut egui::Ui| -> egui::Response {
let size = vec2(36.0, 20.0);
let (rect, mut r) = ui.allocate_exact_size(size, egui::Sense::click());
if r.clicked() {
*on = !*on;
r.mark_changed();
}
let t = ui.ctx().animate_bool_responsive(r.id, *on);
let off_col = ui.visuals().widgets.inactive.bg_fill;
let on_col = ui.visuals().selection.bg_fill;
let fill = if ui.is_enabled() { egui::lerp(egui::Rgba::from(off_col)..=egui::Rgba::from(on_col), t).into() } else { off_col.gamma_multiply(0.6) };
ui.painter().rect_filled(rect, 10, fill);
let x = egui::lerp((rect.left() + 10.0)..=(rect.right() - 10.0), t);
ui.painter().circle_filled(egui::pos2(x, rect.center().y), 8.0, Color32::WHITE);
r.on_hover_cursor(egui::CursorIcon::PointingHand)
}
}
/// The original's settings layout: sections 28 px apart, a small grey title (and something on
/// its right), then a card of rows (12 px corners, a hairline, 8 px top and bottom).
const SEC_GAP: f32 = 28.0;
fn sec_title(ui: &mut egui::Ui, title: &str, p: Palette, accessory: impl FnOnce(&mut egui::Ui)) {
ui.add_space(SEC_GAP * 0.5);
ui.horizontal(|ui| {
ui.add_space(14.0);
ui.label(RichText::new(t(title)).size(FONT_SM).color(p.text2));
ui.with_layout(Layout::right_to_left(Align::Center), |ui| {
ui.add_space(4.0);
accessory(ui);
});
});
ui.add_space(8.0);
}
fn sec_card(ui: &mut egui::Ui, p: Palette, body: impl FnOnce(&mut egui::Ui)) {
Frame::new()
.fill(p.hover.gamma_multiply(0.45))
.stroke(Stroke::new(0.5, p.border))
.corner_radius(12)
.inner_margin(Margin::symmetric(0, 8))
.show(ui, |ui| {
ui.set_width(ui.available_width());
ui.spacing_mut().item_spacing.y = 0.0;
body(ui)
});
ui.add_space(SEC_GAP * 0.5);
}
/// A row: the setting's name (「?」 explains on hover) on the left, its control on the right.
fn sec_row(ui: &mut egui::Ui, p: Palette, label: &str, help: &str, control: impl FnOnce(&mut egui::Ui)) {
sec_row_s(ui, p, label, "", help, control)
}
/// A row with a short status line under the name (目前在線上、上次快照…).
fn sec_row_s(ui: &mut egui::Ui, p: Palette, label: &str, status: &str, help: &str, control: impl FnOnce(&mut egui::Ui)) {
Frame::new().inner_margin(Margin::symmetric(14, 10)).show(ui, |ui| {
ui.set_width(ui.available_width());
ui.horizontal(|ui| {
let w = (ui.available_width() * 0.58).max(200.0);
ui.vertical(|ui| {
ui.set_width(w);
ui.spacing_mut().item_spacing.y = 2.0;
ui.horizontal(|ui| {
ui.spacing_mut().item_spacing.x = 4.0;
ui.label(RichText::new(t(label)).size(FONT_BASE).color(p.text));
help_mark(ui, help, p);
});
if !status.is_empty() {
ui.add(egui::Label::new(RichText::new(t(status)).size(FONT_SM).color(p.text2)).truncate());
}
});
ui.with_layout(Layout::right_to_left(Align::Center), |ui| {
ui.spacing_mut().item_spacing.x = 6.0;
control(ui)
});
});
});
}
/// A small note inside a card.
fn sec_note(ui: &mut egui::Ui, p: Palette, text: &str) {
Frame::new().inner_margin(Margin { left: 14, right: 14, top: 0, bottom: 6 }).show(ui, |ui| {
ui.add(egui::Label::new(RichText::new(t(text)).size(FONT_XS).color(p.text3)).wrap());
});
}