AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
// Read the person's own browser window through macOS accessibility, the way Grok Bot's Computer
|
|
|
|
|
|
// Use helper (CUGrokBotService) sees their screen: no browser setting ("允許來自 Apple 事件的
|
|
|
|
|
|
// JavaScript") and no remote control of Chrome, so it works in whichever Chrome profile the person
|
|
|
|
|
|
// has open. macOS asks once to let AnyBot use 輔助使用 (Accessibility), as it does for that helper.
|
|
|
|
|
|
//
|
|
|
|
|
|
// Decompiled rules (CUGrokBotService, CUCore):
|
|
|
|
|
|
// - `CURendererApp.isRenderer(bundleIdentifier:executablePath:)` @0x1000a1b6c: an app is a
|
|
|
|
|
|
// Chromium/Electron "renderer" if its bundle id is `com.github.Electron` or `com.google.Chrome`,
|
|
|
|
|
|
// starts with `com.google.Chrome.`, its executable path contains
|
|
|
|
|
|
// `/Electron.app/Contents/MacOS/Electron`, or `isRendererBundle(atPath:)` says so.
|
|
|
|
|
|
// - `CURendererApp.isRendererBundle(atPath:)` @0x1000a19ec: `<bundle>/Contents/Frameworks/` holds
|
|
|
|
|
|
// one of 3 frameworks: Electron Framework, Chromium Embedded Framework, Google Chrome Framework.
|
|
|
|
|
|
// - `CURendererAccessibility.writeFlags(on:)` @0x1000a1554: on such apps sets
|
|
|
|
|
|
// `AXManualAccessibility` then `AXEnhancedUserInterface` on the app's accessibility element, so
|
|
|
|
|
|
// their web content shows up in the accessibility tree.
|
|
|
|
|
|
|
|
|
|
|
|
/// Bundle ids that are renderers outright (isRenderer @0x1000a1b6c).
|
|
|
|
|
|
pub const RENDERER_BUNDLE_IDS: [&str; 2] = ["com.github.Electron", "com.google.Chrome"];
|
|
|
|
|
|
/// Bundle id prefix for Chrome's channels and helpers (isRenderer @0x1000a1b6c).
|
|
|
|
|
|
pub const RENDERER_BUNDLE_PREFIX: &str = "com.google.Chrome.";
|
|
|
|
|
|
/// Executable path piece of a bare Electron app (isRenderer @0x1000a1b6c).
|
|
|
|
|
|
pub const ELECTRON_EXE_PIECE: &str = "/Electron.app/Contents/MacOS/Electron";
|
|
|
|
|
|
/// Frameworks that mark a Chromium-based bundle (isRendererBundle @0x1000a19ec).
|
|
|
|
|
|
pub const RENDERER_FRAMEWORKS: [&str; 3] = ["Electron Framework.framework", "Chromium Embedded Framework.framework", "Google Chrome Framework.framework"];
|
|
|
|
|
|
/// The two flags written on renderer apps, in order (writeFlags @0x1000a1554).
|
|
|
|
|
|
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
|
|
|
|
|
pub const RENDERER_AX_FLAGS: [&str; 2] = ["AXManualAccessibility", "AXEnhancedUserInterface"];
|
|
|
|
|
|
/// Most accessibility elements walked for one read (estimated).
|
|
|
|
|
|
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
|
|
|
|
|
pub const MAX_AX_NODES: usize = 20_000;
|
|
|
|
|
|
/// Deepest accessibility nesting followed (estimated).
|
|
|
|
|
|
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
|
|
|
|
|
const MAX_AX_DEPTH: usize = 80;
|
|
|
|
|
|
/// Tries while Chrome builds its accessibility tree after the flags are set (estimated).
|
|
|
|
|
|
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
|
|
|
|
|
const READ_TRIES: usize = 5;
|
|
|
|
|
|
/// Pause between those tries (estimated).
|
|
|
|
|
|
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
|
|
|
|
|
const RETRY_PAUSE_MS: u64 = 700;
|
|
|
|
|
|
/// Longest wait for one accessibility answer (estimated).
|
|
|
|
|
|
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
|
|
|
|
|
const AX_TIMEOUT_SECS: f32 = 2.0;
|
|
|
|
|
|
|
|
|
|
|
|
/// isRenderer(bundleIdentifier:executablePath:) + isRendererBundle(atPath:), with the framework
|
|
|
|
|
|
/// check passed in so it can be tested.
|
|
|
|
|
|
pub fn is_renderer(bundle_id: Option<&str>, exe_path: Option<&str>, has_framework: impl Fn(&str) -> bool) -> bool {
|
|
|
|
|
|
if let Some(id) = bundle_id {
|
|
|
|
|
|
if RENDERER_BUNDLE_IDS.contains(&id) || id.starts_with(RENDERER_BUNDLE_PREFIX) {
|
|
|
|
|
|
return true;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Some(p) = exe_path {
|
|
|
|
|
|
if p.contains(ELECTRON_EXE_PIECE) {
|
|
|
|
|
|
return true;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
RENDERER_FRAMEWORKS.iter().any(|f| has_framework(f))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// The `.app` folder an executable lives in.
|
|
|
|
|
|
pub fn bundle_of(exe: &str) -> Option<&str> {
|
|
|
|
|
|
exe.find(".app/").map(|i| &exe[..i + 4])
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// The person's own browser process (not AnyBot's own window, which runs from AnyBot's folder).
|
|
|
|
|
|
pub fn pick_person_pid(ps: &str, exe: &str) -> Option<i32> {
|
|
|
|
|
|
ps.lines().find_map(|l| {
|
|
|
|
|
|
let l = l.trim_start();
|
|
|
|
|
|
let (pid, args) = l.split_once(' ')?;
|
|
|
|
|
|
let args = args.trim_start();
|
|
|
|
|
|
let own = args.strip_prefix(exe)?;
|
|
|
|
|
|
(own.is_empty() || own.starts_with(' ')).then_some(())?;
|
|
|
|
|
|
(!args.contains("/AnyBot/browser") && !args.contains(" --type=")).then_some(())?;
|
|
|
|
|
|
pid.parse().ok()
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub struct Page {
|
|
|
|
|
|
pub title: String,
|
|
|
|
|
|
pub url: String,
|
|
|
|
|
|
pub text: String,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[cfg(target_os = "macos")]
|
|
|
|
|
|
mod mac {
|
|
|
|
|
|
use super::*;
|
|
|
|
|
|
use std::ffi::{c_char, c_void, CString};
|
|
|
|
|
|
|
|
|
|
|
|
type CFTypeRef = *const c_void;
|
|
|
|
|
|
|
|
|
|
|
|
#[link(name = "ApplicationServices", kind = "framework")]
|
|
|
|
|
|
extern "C" {
|
|
|
|
|
|
fn AXIsProcessTrustedWithOptions(options: CFTypeRef) -> bool;
|
|
|
|
|
|
fn AXUIElementCreateApplication(pid: i32) -> CFTypeRef;
|
|
|
|
|
|
fn AXUIElementSetAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: CFTypeRef) -> i32;
|
|
|
|
|
|
fn AXUIElementCopyAttributeValue(el: CFTypeRef, attr: CFTypeRef, value: *mut CFTypeRef) -> i32;
|
|
|
|
|
|
fn AXUIElementSetMessagingTimeout(el: CFTypeRef, secs: f32) -> i32;
|
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model
Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before;
16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s.
Knowing the machine
- facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager,
tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the
instructions at the start of a turn.
- The box's sudo is the bot's: only what follows sudo is checked against the always-denied
list (it was refused outright, while the instructions said it could sudo), and commands with
sudo run as the box's root (setuid does nothing under gVisor).
- Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE /
ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one.
- Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run
outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do
shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the
sandbox broke says so, instead of looking like a broken program.
Reaching what it sees
- The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside
for the system's own), computer_use_zoom looks closer at full sharpness.
- Numbers stay with their controls from look to look (they were reassigned every look, so a
step's second click landed elsewhere); the box's front window is listed first; a control can
be named instead of numbered (name "4").
- Web page numbering walks open shadow DOM (Chromium settings, web components).
- A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry
without it came up with no network).
Working like a person
- Instructions: pick the route, know what done looks like, one route, check once, say what's
stuck instead of faking it with a substitute, ask before cutting the connection.
- StepPlan: the bot's model takes the first step, the step after trouble (failed, refused,
repeated, nothing changed on screen, the quick model switching routes), one in four, and the
final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does
the planned steps in between, falling back to the bot's model if it can't be used.
- Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does
them again with no model in between, finding each control again, handing back on a mismatch.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
|
|
|
|
fn AXValueGetValue(value: CFTypeRef, kind: u32, out: *mut c_void) -> bool;
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
static kAXTrustedCheckOptionPrompt: CFTypeRef;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[link(name = "CoreFoundation", kind = "framework")]
|
|
|
|
|
|
extern "C" {
|
|
|
|
|
|
fn CFStringCreateWithCString(alloc: CFTypeRef, s: *const c_char, enc: u32) -> CFTypeRef;
|
|
|
|
|
|
fn CFRelease(x: CFTypeRef);
|
|
|
|
|
|
fn CFRetain(x: CFTypeRef) -> CFTypeRef;
|
|
|
|
|
|
fn CFGetTypeID(x: CFTypeRef) -> usize;
|
|
|
|
|
|
fn CFStringGetTypeID() -> usize;
|
|
|
|
|
|
fn CFArrayGetTypeID() -> usize;
|
|
|
|
|
|
fn CFURLGetTypeID() -> usize;
|
|
|
|
|
|
fn CFArrayGetCount(a: CFTypeRef) -> isize;
|
|
|
|
|
|
fn CFArrayGetValueAtIndex(a: CFTypeRef, i: isize) -> CFTypeRef;
|
|
|
|
|
|
fn CFStringGetLength(s: CFTypeRef) -> isize;
|
|
|
|
|
|
fn CFStringGetMaximumSizeForEncoding(len: isize, enc: u32) -> isize;
|
|
|
|
|
|
fn CFStringGetCString(s: CFTypeRef, buf: *mut c_char, size: isize, enc: u32) -> bool;
|
|
|
|
|
|
fn CFURLGetString(u: CFTypeRef) -> CFTypeRef;
|
2026-11-09 06:58:40 +00:00
|
|
|
|
fn CFNumberGetTypeID() -> usize;
|
|
|
|
|
|
fn CFNumberGetValue(n: CFTypeRef, kind: isize, out: *mut c_void) -> bool;
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
fn CFDictionaryCreate(alloc: CFTypeRef, keys: *const CFTypeRef, values: *const CFTypeRef, n: isize, kcb: *const c_void, vcb: *const c_void) -> CFTypeRef;
|
|
|
|
|
|
static kCFBooleanTrue: CFTypeRef;
|
|
|
|
|
|
static kCFTypeDictionaryKeyCallBacks: [usize; 6];
|
|
|
|
|
|
static kCFTypeDictionaryValueCallBacks: [usize; 5];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
const UTF8: u32 = 0x0800_0100;
|
|
|
|
|
|
|
|
|
|
|
|
/// An owned Core Foundation value, released when dropped.
|
|
|
|
|
|
struct Cf(CFTypeRef);
|
|
|
|
|
|
impl Drop for Cf {
|
|
|
|
|
|
fn drop(&mut self) {
|
|
|
|
|
|
if !self.0.is_null() {
|
|
|
|
|
|
unsafe { CFRelease(self.0) }
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn cfstr(s: &str) -> Cf {
|
|
|
|
|
|
let c = CString::new(s).unwrap_or_default();
|
|
|
|
|
|
Cf(unsafe { CFStringCreateWithCString(std::ptr::null(), c.as_ptr(), UTF8) })
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn string_of(v: CFTypeRef) -> Option<String> {
|
|
|
|
|
|
unsafe {
|
|
|
|
|
|
if v.is_null() || CFGetTypeID(v) != CFStringGetTypeID() {
|
|
|
|
|
|
return None;
|
|
|
|
|
|
}
|
|
|
|
|
|
let size = CFStringGetMaximumSizeForEncoding(CFStringGetLength(v), UTF8) + 1;
|
|
|
|
|
|
let mut buf = vec![0u8; size.max(1) as usize];
|
|
|
|
|
|
if !CFStringGetCString(v, buf.as_mut_ptr() as *mut c_char, size, UTF8) {
|
|
|
|
|
|
return None;
|
|
|
|
|
|
}
|
|
|
|
|
|
let end = buf.iter().position(|b| *b == 0).unwrap_or(buf.len());
|
|
|
|
|
|
Some(String::from_utf8_lossy(&buf[..end]).into_owned())
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn attr(el: CFTypeRef, name: &str) -> Option<Cf> {
|
|
|
|
|
|
let key = cfstr(name);
|
|
|
|
|
|
let mut out: CFTypeRef = std::ptr::null();
|
|
|
|
|
|
let err = unsafe { AXUIElementCopyAttributeValue(el, key.0, &mut out) };
|
|
|
|
|
|
(err == 0 && !out.is_null()).then(|| Cf(out))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn attr_string(el: CFTypeRef, name: &str) -> Option<String> {
|
|
|
|
|
|
attr(el, name).and_then(|v| string_of(v.0))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn children(el: CFTypeRef) -> (Option<Cf>, Vec<CFTypeRef>) {
|
|
|
|
|
|
let Some(arr) = attr(el, "AXChildren") else { return (None, vec![]) };
|
|
|
|
|
|
unsafe {
|
|
|
|
|
|
if CFGetTypeID(arr.0) != CFArrayGetTypeID() {
|
|
|
|
|
|
return (Some(arr), vec![]);
|
|
|
|
|
|
}
|
|
|
|
|
|
let n = CFArrayGetCount(arr.0);
|
|
|
|
|
|
let items = (0..n).map(|i| CFArrayGetValueAtIndex(arr.0, i)).collect();
|
|
|
|
|
|
(Some(arr), items)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Is AnyBot allowed to use accessibility? `prompt` makes macOS show its own question.
|
|
|
|
|
|
pub fn trusted(prompt: bool) -> bool {
|
|
|
|
|
|
unsafe {
|
|
|
|
|
|
if !prompt {
|
|
|
|
|
|
return AXIsProcessTrustedWithOptions(std::ptr::null());
|
|
|
|
|
|
}
|
|
|
|
|
|
let keys = [kAXTrustedCheckOptionPrompt];
|
|
|
|
|
|
let values = [kCFBooleanTrue];
|
|
|
|
|
|
let dict = Cf(CFDictionaryCreate(std::ptr::null(), keys.as_ptr(), values.as_ptr(), 1, kCFTypeDictionaryKeyCallBacks.as_ptr() as *const c_void, kCFTypeDictionaryValueCallBacks.as_ptr() as *const c_void));
|
|
|
|
|
|
AXIsProcessTrustedWithOptions(dict.0)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// writeFlags(on:) @0x1000a1554: both flags on the app element, in the same order.
|
|
|
|
|
|
fn write_flags(app: CFTypeRef) -> bool {
|
|
|
|
|
|
RENDERER_AX_FLAGS.iter().fold(true, |ok, f| {
|
|
|
|
|
|
let key = cfstr(f);
|
|
|
|
|
|
let err = unsafe { AXUIElementSetAttributeValue(app, key.0, kCFBooleanTrue) };
|
|
|
|
|
|
ok && err == 0
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn find_role(el: CFTypeRef, role: &str, seen: &mut usize, depth: usize) -> Option<Cf> {
|
|
|
|
|
|
if *seen >= MAX_AX_NODES || depth > MAX_AX_DEPTH {
|
|
|
|
|
|
return None;
|
|
|
|
|
|
}
|
|
|
|
|
|
*seen += 1;
|
|
|
|
|
|
let (_keep, kids) = children(el);
|
|
|
|
|
|
for k in kids {
|
|
|
|
|
|
if attr_string(k, "AXRole").as_deref() == Some(role) {
|
|
|
|
|
|
unsafe { CFRetain(k) };
|
|
|
|
|
|
return Some(Cf(k));
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Some(found) = find_role(k, role, seen, depth + 1) {
|
|
|
|
|
|
return Some(found);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
None
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Visible text under the page: runs of static text become one line.
|
|
|
|
|
|
fn collect(el: CFTypeRef, out: &mut Vec<String>, line: &mut String, seen: &mut usize, depth: usize, max_chars: usize, total: &mut usize) {
|
|
|
|
|
|
if *seen >= MAX_AX_NODES || depth > MAX_AX_DEPTH || *total > max_chars {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
*seen += 1;
|
|
|
|
|
|
let (_keep, kids) = children(el);
|
|
|
|
|
|
for k in kids {
|
|
|
|
|
|
let role = attr_string(k, "AXRole").unwrap_or_default();
|
|
|
|
|
|
if role == "AXStaticText" {
|
|
|
|
|
|
if let Some(t) = attr_string(k, "AXValue") {
|
|
|
|
|
|
*total += t.chars().count();
|
|
|
|
|
|
line.push_str(&t);
|
|
|
|
|
|
}
|
|
|
|
|
|
} else {
|
|
|
|
|
|
flush(out, line);
|
|
|
|
|
|
collect(k, out, line, seen, depth + 1, max_chars, total);
|
|
|
|
|
|
flush(out, line);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn flush(out: &mut Vec<String>, line: &mut String) {
|
|
|
|
|
|
let t = line.trim();
|
|
|
|
|
|
if !t.is_empty() {
|
|
|
|
|
|
out.push(t.to_string());
|
|
|
|
|
|
}
|
|
|
|
|
|
line.clear();
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn url_of(web: CFTypeRef) -> String {
|
|
|
|
|
|
let Some(u) = attr(web, "AXURL") else { return String::new() };
|
|
|
|
|
|
unsafe {
|
|
|
|
|
|
if CFGetTypeID(u.0) == CFURLGetTypeID() {
|
|
|
|
|
|
string_of(CFURLGetString(u.0)).unwrap_or_default()
|
|
|
|
|
|
} else {
|
|
|
|
|
|
string_of(u.0).unwrap_or_default()
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub fn read(pid: i32, renderer: bool, max_chars: usize) -> Result<Page, String> {
|
|
|
|
|
|
let app = Cf(unsafe { AXUIElementCreateApplication(pid) });
|
|
|
|
|
|
if app.0.is_null() {
|
|
|
|
|
|
return Err("連不上你的瀏覽器".into());
|
|
|
|
|
|
}
|
|
|
|
|
|
unsafe { AXUIElementSetMessagingTimeout(app.0, AX_TIMEOUT_SECS) };
|
|
|
|
|
|
if renderer {
|
|
|
|
|
|
write_flags(app.0);
|
|
|
|
|
|
}
|
|
|
|
|
|
let mut last_title = String::new();
|
|
|
|
|
|
for i in 0..READ_TRIES {
|
|
|
|
|
|
if i > 0 {
|
|
|
|
|
|
std::thread::sleep(std::time::Duration::from_millis(RETRY_PAUSE_MS));
|
|
|
|
|
|
}
|
|
|
|
|
|
let Some(win) = attr(app.0, "AXFocusedWindow").or_else(|| attr(app.0, "AXMainWindow")) else { continue };
|
|
|
|
|
|
last_title = attr_string(win.0, "AXTitle").unwrap_or_default();
|
|
|
|
|
|
let mut seen = 0;
|
|
|
|
|
|
let Some(web) = find_role(win.0, "AXWebArea", &mut seen, 0) else { continue };
|
|
|
|
|
|
let title = attr_string(web.0, "AXTitle").filter(|t| !t.is_empty()).unwrap_or_else(|| last_title.clone());
|
|
|
|
|
|
let (mut lines, mut line, mut seen, mut total) = (vec![], String::new(), 0, 0);
|
|
|
|
|
|
collect(web.0, &mut lines, &mut line, &mut seen, 0, max_chars, &mut total);
|
|
|
|
|
|
flush(&mut lines, &mut line);
|
|
|
|
|
|
if lines.is_empty() && i + 1 < READ_TRIES {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
return Ok(Page { title, url: url_of(web.0), text: lines.join("\n") });
|
|
|
|
|
|
}
|
|
|
|
|
|
if last_title.is_empty() {
|
|
|
|
|
|
Err("你的瀏覽器現在沒有打開的視窗".into())
|
|
|
|
|
|
} else {
|
|
|
|
|
|
Err(format!("讀不到「{last_title}」的頁面內容"))
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model
Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before;
16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s.
Knowing the machine
- facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager,
tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the
instructions at the start of a turn.
- The box's sudo is the bot's: only what follows sudo is checked against the always-denied
list (it was refused outright, while the instructions said it could sudo), and commands with
sudo run as the box's root (setuid does nothing under gVisor).
- Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE /
ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one.
- Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run
outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do
shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the
sandbox broke says so, instead of looking like a broken program.
Reaching what it sees
- The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside
for the system's own), computer_use_zoom looks closer at full sharpness.
- Numbers stay with their controls from look to look (they were reassigned every look, so a
step's second click landed elsewhere); the box's front window is listed first; a control can
be named instead of numbered (name "4").
- Web page numbering walks open shadow DOM (Chromium settings, web components).
- A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry
without it came up with no network).
Working like a person
- Instructions: pick the route, know what done looks like, one route, check once, say what's
stuck instead of faking it with a substitute, ask before cutting the connection.
- StepPlan: the bot's model takes the first step, the step after trouble (failed, refused,
repeated, nothing changed on screen, the quick model switching routes), one in four, and the
final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does
the planned steps in between, falling back to the bot's model if it can't be used.
- Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does
them again with no model in between, finding each control again, handing back on a mismatch.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
|
|
|
|
|
|
|
|
|
|
/// One status item in the menu bar (an app's icon at the top right): its app, its words,
|
|
|
|
|
|
/// and where it is in screen points.
|
|
|
|
|
|
pub struct StatusItem {
|
|
|
|
|
|
pub app: String,
|
|
|
|
|
|
pub label: String,
|
|
|
|
|
|
pub rect: [f64; 4],
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// The status items of these apps (pid, name), read through each app's AXExtrasMenuBar. An
|
|
|
|
|
|
/// app that doesn't answer within a quarter second is skipped.
|
|
|
|
|
|
pub fn status_items(apps: &[(i32, String)]) -> Vec<StatusItem> {
|
|
|
|
|
|
// Several apps at once: one that is slow to answer holds up only its share.
|
|
|
|
|
|
let chunk = apps.len().div_ceil(8).max(1);
|
|
|
|
|
|
std::thread::scope(|sc| {
|
|
|
|
|
|
let parts: Vec<_> = apps.chunks(chunk).map(|part| sc.spawn(move || status_items_of(part))).collect();
|
|
|
|
|
|
parts.into_iter().flat_map(|h| h.join().unwrap_or_default()).collect()
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// An icon's words: its own title or description, else the first found among its children
|
|
|
|
|
|
/// (the system's own icons are a group hosting the real button), else its identifier.
|
|
|
|
|
|
fn words_within(el: CFTypeRef, depth: usize) -> String {
|
|
|
|
|
|
let own = ["AXTitle", "AXDescription", "AXValue", "AXHelp"].iter().filter_map(|k| attr_string(el, k)).map(|t| t.trim().to_string()).find(|t| !t.is_empty());
|
|
|
|
|
|
if let Some(w) = own {
|
|
|
|
|
|
return w;
|
|
|
|
|
|
}
|
|
|
|
|
|
if depth > 0 {
|
|
|
|
|
|
let (_keep, kids) = children(el);
|
|
|
|
|
|
for k in kids {
|
|
|
|
|
|
let w = words_within(k, depth - 1);
|
|
|
|
|
|
if !w.is_empty() {
|
|
|
|
|
|
return w;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
attr_string(el, "AXIdentifier").map(|id| super::menu_extra_name(&id)).unwrap_or_default()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-09 06:58:40 +00:00
|
|
|
|
/// One entry of an app's menus: a title in its menu bar (「檔案」), or an item of a menu that is
|
|
|
|
|
|
/// open now, with its keyboard shortcut when it has one.
|
|
|
|
|
|
pub struct MenuEntry {
|
|
|
|
|
|
pub label: String,
|
|
|
|
|
|
pub rect: [f64; 4],
|
|
|
|
|
|
/// 0: in the menu bar; 1: in the open menu; 2 and more: in a submenu opened from it.
|
|
|
|
|
|
pub depth: u8,
|
|
|
|
|
|
pub enabled: bool,
|
|
|
|
|
|
pub shortcut: String,
|
|
|
|
|
|
/// The item macOS shows in this place while option is held (「複製」 hides 「儲存為⋯」).
|
|
|
|
|
|
pub alternate: Option<(String, String)>,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn rect_of(el: CFTypeRef) -> Option<[f64; 4]> {
|
|
|
|
|
|
#[repr(C)]
|
|
|
|
|
|
#[derive(Default, Clone, Copy)]
|
|
|
|
|
|
struct Xy {
|
|
|
|
|
|
a: f64,
|
|
|
|
|
|
b: f64,
|
|
|
|
|
|
}
|
|
|
|
|
|
let get = |k: &str, kind: u32| -> Option<Xy> {
|
|
|
|
|
|
let v = attr(el, k)?;
|
|
|
|
|
|
let mut xy = Xy::default();
|
|
|
|
|
|
unsafe { AXValueGetValue(v.0, kind, &mut xy as *mut Xy as *mut c_void) }.then_some(xy)
|
|
|
|
|
|
};
|
|
|
|
|
|
// kAXValueCGPointType = 1, kAXValueCGSizeType = 2
|
|
|
|
|
|
let (p, z) = (get("AXPosition", 1)?, get("AXSize", 2)?);
|
|
|
|
|
|
(z.a >= 1.0 && z.b >= 1.0).then_some([p.a, p.b, z.a, z.b])
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn attr_bool(el: CFTypeRef, name: &str) -> Option<bool> {
|
|
|
|
|
|
attr(el, name).map(|v| v.0 == unsafe { kCFBooleanTrue })
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn attr_int(el: CFTypeRef, name: &str) -> Option<i64> {
|
|
|
|
|
|
let v = attr(el, name)?;
|
|
|
|
|
|
let mut n: i64 = 0;
|
|
|
|
|
|
// kCFNumberSInt64Type = 4
|
|
|
|
|
|
unsafe { (CFGetTypeID(v.0) == CFNumberGetTypeID() && CFNumberGetValue(v.0, 4, &mut n as *mut i64 as *mut c_void)).then_some(n) }
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// The menus of the app `pid`: its menu bar's titles, and the items of whichever menu is open
|
|
|
|
|
|
/// (one pulled down from the bar, a submenu opened from it, or a right-click menu). Nothing
|
|
|
|
|
|
/// when the app doesn't answer within a quarter second.
|
|
|
|
|
|
pub fn app_menus(pid: i32) -> Vec<MenuEntry> {
|
|
|
|
|
|
let app = Cf(unsafe { AXUIElementCreateApplication(pid) });
|
|
|
|
|
|
if app.0.is_null() {
|
|
|
|
|
|
return vec![];
|
|
|
|
|
|
}
|
|
|
|
|
|
unsafe { AXUIElementSetMessagingTimeout(app.0, 0.25) };
|
|
|
|
|
|
let mut out = vec![];
|
|
|
|
|
|
if let Some(bar) = attr(app.0, "AXMenuBar") {
|
|
|
|
|
|
let (_keep, items) = children(bar.0);
|
|
|
|
|
|
for it in items {
|
|
|
|
|
|
let Some(rect) = rect_of(it) else { continue };
|
|
|
|
|
|
let label = attr_string(it, "AXTitle").unwrap_or_default().trim().to_string();
|
|
|
|
|
|
if label.is_empty() || rect[1] > 60.0 {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
out.push(MenuEntry { label, rect, depth: 0, enabled: attr_bool(it, "AXEnabled").unwrap_or(true), shortcut: String::new(), alternate: None });
|
|
|
|
|
|
if attr_bool(it, "AXSelected") == Some(true) {
|
|
|
|
|
|
let (_k, menus) = children(it);
|
|
|
|
|
|
for m in menus {
|
|
|
|
|
|
menu_items(m, 1, &mut out);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
// A right-click menu hangs off the app itself.
|
|
|
|
|
|
let (_keep, kids) = children(app.0);
|
|
|
|
|
|
for k in kids {
|
|
|
|
|
|
if attr_string(k, "AXRole").as_deref() == Some("AXMenu") {
|
|
|
|
|
|
menu_items(k, 1, &mut out);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
out
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn menu_items(menu: CFTypeRef, depth: u8, out: &mut Vec<MenuEntry>) {
|
|
|
|
|
|
if depth > 3 || out.len() > 300 {
|
|
|
|
|
|
return;
|
|
|
|
|
|
}
|
|
|
|
|
|
let (_keep, items) = children(menu);
|
|
|
|
|
|
for it in items {
|
|
|
|
|
|
let label = attr_string(it, "AXTitle").unwrap_or_default().trim().to_string();
|
|
|
|
|
|
// Separators have no title; a closed menu's items have no place on the screen.
|
|
|
|
|
|
let Some(rect) = rect_of(it) else { continue };
|
|
|
|
|
|
if label.is_empty() {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
let shortcut = attr_string(it, "AXMenuItemCmdChar").filter(|c| !c.is_empty()).map(|c| shortcut_words(&c, attr_int(it, "AXMenuItemCmdModifiers").unwrap_or(0))).unwrap_or_default();
|
|
|
|
|
|
let entry = MenuEntry { label, rect, depth, enabled: attr_bool(it, "AXEnabled").unwrap_or(true), shortcut, alternate: None };
|
|
|
|
|
|
// Two items in one place: one of them is the option-key alternate of the other, and
|
|
|
|
|
|
// only one shows. AX says nothing about which (AXMenuItemPrimaryUIElement is empty),
|
|
|
|
|
|
// so the one that needs option is taken for the hidden one.
|
|
|
|
|
|
if let Some(last) = out.last_mut().filter(|l| l.depth == depth && l.rect == entry.rect && l.alternate.is_none()) {
|
|
|
|
|
|
let needs_option = |e: &MenuEntry| e.shortcut.contains("option");
|
|
|
|
|
|
if needs_option(last) && !needs_option(&entry) {
|
|
|
|
|
|
let hidden = std::mem::replace(last, entry);
|
|
|
|
|
|
last.alternate = Some((hidden.label, hidden.shortcut));
|
|
|
|
|
|
} else {
|
|
|
|
|
|
last.alternate = Some((entry.label, entry.shortcut));
|
|
|
|
|
|
}
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
let entry_selected = attr_bool(it, "AXSelected") == Some(true);
|
|
|
|
|
|
out.push(entry);
|
|
|
|
|
|
if entry_selected {
|
|
|
|
|
|
let (_k, subs) = children(it);
|
|
|
|
|
|
for m in subs {
|
|
|
|
|
|
menu_items(m, depth + 1, out);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model
Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before;
16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s.
Knowing the machine
- facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager,
tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the
instructions at the start of a turn.
- The box's sudo is the bot's: only what follows sudo is checked against the always-denied
list (it was refused outright, while the instructions said it could sudo), and commands with
sudo run as the box's root (setuid does nothing under gVisor).
- Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE /
ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one.
- Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run
outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do
shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the
sandbox broke says so, instead of looking like a broken program.
Reaching what it sees
- The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside
for the system's own), computer_use_zoom looks closer at full sharpness.
- Numbers stay with their controls from look to look (they were reassigned every look, so a
step's second click landed elsewhere); the box's front window is listed first; a control can
be named instead of numbered (name "4").
- Web page numbering walks open shadow DOM (Chromium settings, web components).
- A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry
without it came up with no network).
Working like a person
- Instructions: pick the route, know what done looks like, one route, check once, say what's
stuck instead of faking it with a substitute, ask before cutting the connection.
- StepPlan: the bot's model takes the first step, the step after trouble (failed, refused,
repeated, nothing changed on screen, the quick model switching routes), one in four, and the
final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does
the planned steps in between, falling back to the bot's model if it can't be used.
- Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does
them again with no model in between, finding each control again, handing back on a mismatch.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
|
|
|
|
fn status_items_of(apps: &[(i32, String)]) -> Vec<StatusItem> {
|
|
|
|
|
|
#[repr(C)]
|
|
|
|
|
|
#[derive(Default, Clone, Copy)]
|
|
|
|
|
|
struct Xy {
|
|
|
|
|
|
a: f64,
|
|
|
|
|
|
b: f64,
|
|
|
|
|
|
}
|
|
|
|
|
|
let mut out = vec![];
|
|
|
|
|
|
for (pid, name) in apps {
|
|
|
|
|
|
let app = Cf(unsafe { AXUIElementCreateApplication(*pid) });
|
|
|
|
|
|
if app.0.is_null() {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
unsafe { AXUIElementSetMessagingTimeout(app.0, 0.25) };
|
|
|
|
|
|
let Some(bar) = attr(app.0, "AXExtrasMenuBar") else { continue };
|
|
|
|
|
|
let (_keep, items) = children(bar.0);
|
|
|
|
|
|
for it in items {
|
|
|
|
|
|
let get = |k: &str, kind: u32| -> Option<Xy> {
|
|
|
|
|
|
let v = attr(it, k)?;
|
|
|
|
|
|
let mut xy = Xy::default();
|
|
|
|
|
|
unsafe { AXValueGetValue(v.0, kind, &mut xy as *mut Xy as *mut c_void) }.then_some(xy)
|
|
|
|
|
|
};
|
|
|
|
|
|
// kAXValueCGPointType = 1, kAXValueCGSizeType = 2
|
|
|
|
|
|
let (Some(p), Some(z)) = (get("AXPosition", 1), get("AXSize", 2)) else { continue };
|
|
|
|
|
|
if z.a < 1.0 || z.b < 1.0 || p.b > 60.0 {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
let label = words_within(it, 3);
|
|
|
|
|
|
out.push(StatusItem { app: name.clone(), label, rect: [p.a, p.b, z.a, z.b] });
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
out
|
|
|
|
|
|
}
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-09 06:58:40 +00:00
|
|
|
|
#[cfg(target_os = "macos")]
|
|
|
|
|
|
pub use mac::app_menus;
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
#[cfg(target_os = "macos")]
|
|
|
|
|
|
pub use mac::trusted;
|
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model
Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before;
16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s.
Knowing the machine
- facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager,
tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the
instructions at the start of a turn.
- The box's sudo is the bot's: only what follows sudo is checked against the always-denied
list (it was refused outright, while the instructions said it could sudo), and commands with
sudo run as the box's root (setuid does nothing under gVisor).
- Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE /
ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one.
- Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run
outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do
shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the
sandbox broke says so, instead of looking like a broken program.
Reaching what it sees
- The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside
for the system's own), computer_use_zoom looks closer at full sharpness.
- Numbers stay with their controls from look to look (they were reassigned every look, so a
step's second click landed elsewhere); the box's front window is listed first; a control can
be named instead of numbered (name "4").
- Web page numbering walks open shadow DOM (Chromium settings, web components).
- A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry
without it came up with no network).
Working like a person
- Instructions: pick the route, know what done looks like, one route, check once, say what's
stuck instead of faking it with a substitute, ask before cutting the connection.
- StepPlan: the bot's model takes the first step, the step after trouble (failed, refused,
repeated, nothing changed on screen, the quick model switching routes), one in four, and the
final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does
the planned steps in between, falling back to the bot's model if it can't be used.
- Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does
them again with no model in between, finding each control again, handing back on a mismatch.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
|
|
|
|
#[cfg(target_os = "macos")]
|
|
|
|
|
|
pub use mac::status_items;
|
|
|
|
|
|
|
2026-11-09 06:58:40 +00:00
|
|
|
|
/// A menu item's shortcut as the bot presses it (computer_use_press_key): AXMenuItemCmdModifiers
|
|
|
|
|
|
/// is 0 for ⌘ alone, plus 1 shift, 2 option, 4 control, 8 without ⌘.
|
|
|
|
|
|
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
|
|
|
|
|
pub fn shortcut_words(key: &str, mods: i64) -> String {
|
|
|
|
|
|
let mut parts = vec![];
|
|
|
|
|
|
if mods & 4 != 0 {
|
|
|
|
|
|
parts.push("ctrl");
|
|
|
|
|
|
}
|
|
|
|
|
|
if mods & 2 != 0 {
|
|
|
|
|
|
parts.push("option");
|
|
|
|
|
|
}
|
|
|
|
|
|
if mods & 1 != 0 {
|
|
|
|
|
|
parts.push("shift");
|
|
|
|
|
|
}
|
|
|
|
|
|
if mods & 8 == 0 {
|
|
|
|
|
|
parts.push("cmd");
|
|
|
|
|
|
}
|
|
|
|
|
|
// macOS writes arrows and function keys as private-use characters (NSUpArrowFunctionKey…).
|
|
|
|
|
|
let key = match key {
|
|
|
|
|
|
"\u{8}" | "\u{7f}" => "delete".to_string(),
|
|
|
|
|
|
"\r" | "\u{3}" => "return".to_string(),
|
|
|
|
|
|
"\u{1b}" => "escape".to_string(),
|
|
|
|
|
|
"\t" => "tab".to_string(),
|
|
|
|
|
|
" " => "space".to_string(),
|
|
|
|
|
|
"\u{f700}" => "up".to_string(),
|
|
|
|
|
|
"\u{f701}" => "down".to_string(),
|
|
|
|
|
|
"\u{f702}" => "left".to_string(),
|
|
|
|
|
|
"\u{f703}" => "right".to_string(),
|
|
|
|
|
|
k if k.chars().count() == 1 && ('\u{f704}'..='\u{f726}').contains(&k.chars().next().unwrap_or(' ')) => format!("f{}", k.chars().next().unwrap_or(' ') as u32 - 0xf703),
|
|
|
|
|
|
k => k.trim().to_lowercase(),
|
|
|
|
|
|
};
|
|
|
|
|
|
parts.push(&key);
|
|
|
|
|
|
parts.join("+")
|
|
|
|
|
|
}
|
|
|
|
|
|
|
Bots work like someone who knows the computer: they know each machine, reach what they see, plan, and hand the planned steps to a quick model
Measured on the bot's own box (8 tasks x 2, grok-4.7): 14/16 done, 120 steps, 1114 s before;
16/16, 102 steps, 574 s after (model time 1015 -> 471 s). A replayed skill: 49 s -> 15 s.
Knowing the machine
- facts.rs: 電腦現況 per computer (system, desktop, systemd, sudo, network, package manager,
tools, apps, a fixed TZ), read once by a small read-only probe, kept 30 min, in the
instructions at the start of a turn.
- The box's sudo is the bot's: only what follows sudo is checked against the always-denied
list (it was refused outright, while the instructions said it could sudo), and commands with
sudo run as the box's root (setuid does nothing under gVisor).
- Box image v8 drops ENV TZ=UTC (it overrode /etc/localtime). ANYBOT_BOX_IMAGE /
ANYBOT_BOX_VOLUME give a throwaway box its own image and volume, never adopting the real one.
- Mac shell sandbox: App Store app CLIs (they carry their own app sandbox) and ps/top run
outside it; plain `osascript -e` too, asked the first time even in direct mode, refusing do
shell script, terminals, keystrokes, file reads, ObjC and the secret folders. A command the
sandbox broke says so, instead of looking like a broken program.
Reaching what it sees
- The Mac menu bar's status icons are numbered (each app's AXExtrasMenuBar, named from inside
for the system's own), computer_use_zoom looks closer at full sharpness.
- Numbers stay with their controls from look to look (they were reassigned every look, so a
step's second click landed elsewhere); the box's front window is listed first; a control can
be named instead of numbered (name "4").
- Web page numbering walks open shadow DOM (Chromium settings, web components).
- A box on a Docker host elsewhere retries the next port when the usual one is taken (a retry
without it came up with no network).
Working like a person
- Instructions: pick the route, know what done looks like, one route, check once, say what's
stuck instead of faking it with a substitute, ask before cutting the connection.
- StepPlan: the bot's model takes the first step, the step after trouble (failed, refused,
repeated, nothing changed on screen, the quick model switching routes), one in four, and the
final check; a quick model (Bot.step_model, auto: grok-4.20-0309-non-reasoning on xAI) does
the planned steps in between, falling back to the bot's model if it can't be used.
- Skills keep the steps that worked (with what each acted on, not numbers); replay_skill does
them again with no model in between, finding each control again, handing back on a mismatch.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha5F5gJt93t3yEx3XcS
2026-11-09 01:01:03 +00:00
|
|
|
|
/// A system menu-bar icon's words from its identifier (com.apple.menuextra.wifi → Wi-Fi): the
|
|
|
|
|
|
/// system's own icons carry no title.
|
|
|
|
|
|
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
|
|
|
|
|
|
pub fn menu_extra_name(id: &str) -> String {
|
|
|
|
|
|
let tail = id.rsplit(['.', '/']).next().unwrap_or(id).to_lowercase();
|
|
|
|
|
|
let known = [
|
|
|
|
|
|
("wifi", "Wi-Fi"), ("airport", "Wi-Fi"), ("battery", "電池"), ("bluetooth", "藍牙"), ("clock", "時鐘"),
|
|
|
|
|
|
("controlcenter", "控制中心"), ("sound", "聲音"), ("volume", "聲音"), ("focusmodes", "專注模式"), ("focus", "專注模式"),
|
|
|
|
|
|
("display", "顯示器"), ("nowplaying", "正在播放"), ("screenmirroring", "螢幕鏡像輸出"), ("vpn", "VPN"),
|
|
|
|
|
|
("textinput", "輸入法"), ("spotlight", "Spotlight"), ("siri", "Siri"), ("user", "使用者"), ("timemachine", "時光機"),
|
|
|
|
|
|
("keyboardbrightness", "鍵盤亮度"), ("accessibility", "輔助使用"), ("stagemanager", "幕前調度"), ("airdrop", "AirDrop"),
|
|
|
|
|
|
];
|
|
|
|
|
|
known.iter().find(|(k, _)| tail.contains(k)).map(|(_, v)| v.to_string()).unwrap_or_else(|| id.to_string())
|
|
|
|
|
|
}
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
|
|
|
|
|
|
#[cfg(not(target_os = "macos"))]
|
|
|
|
|
|
pub fn trusted(_prompt: bool) -> bool {
|
|
|
|
|
|
false
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Read the front window of the person's browser at `exe` (e.g. Chrome's executable).
|
|
|
|
|
|
pub fn read_person_browser(exe: &str, max_chars: usize) -> Result<Page, String> {
|
|
|
|
|
|
if !cfg!(target_os = "macos") {
|
|
|
|
|
|
return Err("只支援 macOS".into());
|
|
|
|
|
|
}
|
2026-11-05 08:57:44 +00:00
|
|
|
|
if !trusted(false) {
|
|
|
|
|
|
crate::computer::ask_permission("Privacy_Accessibility");
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
return Err("macOS 還沒讓 AnyBot 使用「輔助使用」,所以看不到你的瀏覽器。我已經打開「系統設定 → 隱私權與安全性 → 輔助使用」:把 AnyBot(或「終端機」)打開,然後再叫我試一次。只要開一次。".into());
|
|
|
|
|
|
}
|
|
|
|
|
|
let ps = std::process::Command::new("ps").args(["-axo", "pid=,args="]).output().map(|o| String::from_utf8_lossy(&o.stdout).into_owned()).unwrap_or_default();
|
|
|
|
|
|
let Some(pid) = pick_person_pid(&ps, exe) else { return Err("你的瀏覽器沒有開著。請先打開它(或叫我用 browser_open 打開網址)。".into()) };
|
|
|
|
|
|
let bundle = bundle_of(exe).unwrap_or("");
|
|
|
|
|
|
let bundle_id = std::process::Command::new("plutil")
|
|
|
|
|
|
.args(["-extract", "CFBundleIdentifier", "raw", "-o", "-", &format!("{bundle}/Contents/Info.plist")])
|
|
|
|
|
|
.output()
|
|
|
|
|
|
.ok()
|
|
|
|
|
|
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
|
|
|
|
|
|
.filter(|s| !s.is_empty());
|
|
|
|
|
|
let renderer = is_renderer(bundle_id.as_deref(), Some(exe), |f| std::path::Path::new(bundle).join("Contents/Frameworks").join(f).exists());
|
|
|
|
|
|
#[cfg(target_os = "macos")]
|
|
|
|
|
|
{
|
|
|
|
|
|
mac::read(pid, renderer, max_chars)
|
|
|
|
|
|
}
|
|
|
|
|
|
#[cfg(not(target_os = "macos"))]
|
|
|
|
|
|
{
|
|
|
|
|
|
let _ = (pid, renderer, max_chars);
|
|
|
|
|
|
Err("只支援 macOS".into())
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
|
|
mod tests {
|
|
|
|
|
|
use super::*;
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn renderer_apps_follow_the_original_rule() {
|
|
|
|
|
|
let none = |_: &str| false;
|
|
|
|
|
|
assert!(is_renderer(Some("com.google.Chrome"), None, none));
|
|
|
|
|
|
assert!(is_renderer(Some("com.google.Chrome.canary"), None, none));
|
|
|
|
|
|
assert!(is_renderer(Some("com.github.Electron"), None, none));
|
|
|
|
|
|
assert!(is_renderer(None, Some("/tmp/x/Electron.app/Contents/MacOS/Electron"), none));
|
|
|
|
|
|
assert!(!is_renderer(Some("com.google.Chromeish"), None, none));
|
|
|
|
|
|
assert!(!is_renderer(Some("com.apple.Safari"), Some("/Applications/Safari.app/Contents/MacOS/Safari"), none));
|
|
|
|
|
|
// Brave / Edge carry their own framework names, so only the framework check could catch them.
|
|
|
|
|
|
assert!(is_renderer(Some("com.example.app"), None, |f| f == "Electron Framework.framework"));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn the_persons_browser_is_not_anybots_window() {
|
|
|
|
|
|
let exe = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
|
|
|
|
|
|
let ps = format!(
|
|
|
|
|
|
" 101 {exe} --user-data-dir=/Users/me/Library/Application Support/AnyBot/browser --remote-debugging-port=9333\n 102 {exe} --type=utility\n 205 {exe}\n 300 {exe} --type=renderer"
|
|
|
|
|
|
);
|
|
|
|
|
|
assert_eq!(pick_person_pid(&ps, exe), Some(205));
|
|
|
|
|
|
assert_eq!(pick_person_pid(" 1 /Applications/Safari.app/Contents/MacOS/Safari", exe), None);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-09 06:58:40 +00:00
|
|
|
|
#[test]
|
|
|
|
|
|
fn a_menu_items_shortcut_reads_as_keys_to_press() {
|
|
|
|
|
|
assert_eq!(shortcut_words("S", 0), "cmd+s");
|
|
|
|
|
|
assert_eq!(shortcut_words("S", 3), "option+shift+cmd+s");
|
|
|
|
|
|
assert_eq!(shortcut_words("F", 4), "ctrl+cmd+f");
|
|
|
|
|
|
assert_eq!(shortcut_words("\u{8}", 0), "cmd+delete");
|
|
|
|
|
|
assert_eq!(shortcut_words("\u{f701}", 2), "option+cmd+down");
|
|
|
|
|
|
assert_eq!(shortcut_words("\u{f704}", 8), "f1");
|
|
|
|
|
|
}
|
|
|
|
|
|
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
#[test]
|
|
|
|
|
|
fn bundle_is_found_from_the_executable() {
|
|
|
|
|
|
assert_eq!(bundle_of("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"), Some("/Applications/Google Chrome.app"));
|
|
|
|
|
|
assert_eq!(bundle_of("/usr/bin/true"), None);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
|
|
mod live {
|
2026-11-09 06:58:40 +00:00
|
|
|
|
/// By hand: `ANYBOT_MENU_PID=<pid> cargo test live_app_menus -- --ignored --nocapture`.
|
|
|
|
|
|
#[cfg(target_os = "macos")]
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
#[ignore]
|
|
|
|
|
|
fn live_app_menus() {
|
|
|
|
|
|
let pid: i32 = std::env::var("ANYBOT_MENU_PID").ok().and_then(|p| p.parse().ok()).unwrap_or(0);
|
|
|
|
|
|
let t = std::time::Instant::now();
|
|
|
|
|
|
let m = super::app_menus(pid);
|
|
|
|
|
|
println!("{} entries in {} ms", m.len(), t.elapsed().as_millis());
|
|
|
|
|
|
for e in m.iter().take(40) {
|
|
|
|
|
|
println!("{} {} [{}] {:?} {} {:?}", e.depth, e.label, e.shortcut, e.rect, if e.enabled { "" } else { "停用" }, e.alternate);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
AnyBot: first commit of the working desktop app
Rust/egui rebuild of Grok Bot as of 2026-11-04: chat with streaming
providers (xAI, OpenAI, Claude, Gemini, OpenRouter, OpenCode Go, Ollama,
custom), tools and approvals, the bot's Docker computer, computer use on
macOS / Linux X11 / Wayland, AnyBot's browser with a full DevTools-based
toolset, the encrypted vault, pinned grid and sidebar sections, Markdown
with Mermaid diagrams, phone access through the relay, and the Linux
build and test scripts.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uirha3oEtWeHh58zA7ErHa
2026-11-04 08:51:51 +00:00
|
|
|
|
/// By hand: `cargo test live_person_chrome -- --ignored --nocapture`. Prints counts only.
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
#[ignore]
|
|
|
|
|
|
fn live_person_chrome() {
|
|
|
|
|
|
println!("trusted={}", super::trusted(false));
|
|
|
|
|
|
if super::trusted(false) {
|
|
|
|
|
|
match super::read_person_browser("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", 5000) {
|
|
|
|
|
|
Ok(p) => println!("lines={} title_chars={} url_ok={}", p.text.lines().count(), p.title.chars().count(), p.url.starts_with("http")),
|
|
|
|
|
|
Err(e) => println!("err={e}"),
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|