AnyBot/src/bidi.rs

379 lines
18 KiB
Rust
Raw Normal View History

// Firefox, driven through WebDriver BiDi: the W3C standard Firefox speaks (it dropped Chrome's
// DevTools protocol). browser.rs keeps speaking DevTools; `cdp` turns each DevTools command it
// uses into the BiDi one, so every browser tool works the same on Firefox: real mouse and keys,
// the numbered elements, screenshots, PDF, page dialogs, uploads and downloads.
//
// Firefox allows one BiDi session per window (profile), so one connection per port is kept and
// shared, one command at a time.
use serde_json::{json, Value};
use std::collections::HashMap;
use std::io::ErrorKind;
use std::net::{SocketAddr, TcpStream};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, Instant};
const CALL_TIMEOUT_SECS: u64 = 30;
/// A tab's title for the tab list: a page held up by a dialog mustn't hold the list up (estimated).
const TITLE_TIMEOUT_SECS: u64 = 3;
const SLOW: &str = "瀏覽器太久沒有回應(頁面可能跳出了對話框:可以用 browser_dialog 處理)";
struct Conn {
ws: tungstenite::WebSocket<TcpStream>,
next: u64,
}
enum Fail {
/// The connection is gone (Firefox closed): connect again.
Link(String),
/// No answer in time; a late answer is skipped later (by its id).
Slow,
/// Firefox answered with an error.
Said(String),
}
impl Conn {
fn open(port: u16) -> Result<Conn, String> {
let addr: SocketAddr = format!("127.0.0.1:{port}").parse().map_err(|_| "Firefox 的連接埠不對".to_string())?;
let stream = TcpStream::connect_timeout(&addr, Duration::from_secs(5)).map_err(|e| format!("連不上 Firefox:{e}"))?;
let (ws, _) = tungstenite::client::client(format!("ws://127.0.0.1:{port}/session"), stream).map_err(|e| format!("連不上 Firefox:{e}"))?;
let mut c = Conn { ws, next: 0 };
// Page dialogs stay open until browser_dialog answers them (as in Chrome).
let caps = json!({"capabilities": {"alwaysMatch": {"unhandledPromptBehavior": {"default": "ignore"}}}});
if c.call("session.new", caps, CALL_TIMEOUT_SECS).is_err() {
c.call("session.new", json!({"capabilities": {}}), CALL_TIMEOUT_SECS).map_err(|f| match f {
Fail::Said(e) => format!("Firefox 不讓 AnyBot 控制({e})。可能有別的自動化工具正連著它。"),
Fail::Slow => "Firefox 沒有回應".to_string(),
Fail::Link(e) => format!("連不上 Firefox:{e}"),
})?;
}
Ok(c)
}
fn call(&mut self, method: &str, params: Value, secs: u64) -> Result<Value, Fail> {
self.next += 1;
let id = self.next;
let _ = self.ws.get_mut().set_read_timeout(Some(Duration::from_secs(secs)));
self.ws.send(tungstenite::Message::Text(json!({"id": id, "method": method, "params": params}).to_string().into())).map_err(|e| Fail::Link(e.to_string()))?;
let start = Instant::now();
loop {
if start.elapsed() > Duration::from_secs(secs) {
return Err(Fail::Slow);
}
match self.ws.read() {
Ok(tungstenite::Message::Text(t)) => {
let v: Value = serde_json::from_str(t.as_str()).unwrap_or(Value::Null);
if v["id"] != id {
continue; // an event, or the late answer to an earlier command
}
if v["type"] == "error" {
return Err(Fail::Said(format!("{}:{}", v["error"].as_str().unwrap_or("error"), v["message"].as_str().unwrap_or(""))));
}
return Ok(v["result"].clone());
}
Ok(tungstenite::Message::Close(_)) => return Err(Fail::Link("Firefox 關掉了連線".into())),
Ok(_) => {}
Err(tungstenite::Error::Io(e)) if matches!(e.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => return Err(Fail::Slow),
Err(e) => return Err(Fail::Link(e.to_string())),
}
}
}
}
fn conns() -> &'static Mutex<HashMap<u16, Arc<Mutex<Conn>>>> {
static C: OnceLock<Mutex<HashMap<u16, Arc<Mutex<Conn>>>>> = OnceLock::new();
C.get_or_init(Mutex::default)
}
/// One BiDi command to the Firefox on `port`.
pub fn command(port: u16, method: &str, params: Value) -> Result<Value, String> {
command_within(port, method, params, CALL_TIMEOUT_SECS)
}
fn command_within(port: u16, method: &str, params: Value, secs: u64) -> Result<Value, String> {
// A connection left from a Firefox that has since closed fails once: connect again and retry.
for attempt in 0..2 {
let conn = {
let mut m = conns().lock().unwrap_or_else(|e| e.into_inner());
match m.get(&port) {
Some(c) => c.clone(),
None => {
let c = Arc::new(Mutex::new(Conn::open(port)?));
m.insert(port, c.clone());
c
}
}
};
let r = conn.lock().unwrap_or_else(|e| e.into_inner()).call(method, params.clone(), secs);
match r {
Ok(v) => return Ok(v),
Err(Fail::Said(e)) => return Err(e),
Err(Fail::Slow) => return Err(SLOW.into()),
Err(Fail::Link(e)) => {
conns().lock().unwrap_or_else(|e| e.into_inner()).remove(&port);
if attempt == 1 {
return Err(format!("和 Firefox 的連線斷了:{e}"));
}
}
}
}
Err("和 Firefox 的連線斷了".into())
}
/// A value Firefox sends back (a BiDi RemoteValue) as plain JSON, like DevTools' returnByValue.
pub fn plain(v: &Value) -> Value {
let list = |v: &Value| Value::Array(v["value"].as_array().map(|a| a.iter().map(plain).collect()).unwrap_or_default());
match v["type"].as_str().unwrap_or("") {
"string" | "boolean" | "date" => v["value"].clone(),
"number" => match &v["value"] {
Value::String(s) if s == "-0" => json!(0),
Value::String(_) => Value::Null, // NaN, Infinity: what JSON.stringify gives
n => n.clone(),
},
"bigint" => v["value"].as_str().and_then(|s| s.parse::<i64>().ok()).map(Value::from).unwrap_or(Value::Null),
"array" | "set" | "nodelist" | "htmlcollection" => list(v),
"object" | "map" => {
let mut o = serde_json::Map::new();
for pair in v["value"].as_array().cloned().unwrap_or_default() {
let key = match &pair[0] {
Value::String(s) => s.clone(),
k => plain(k).as_str().map(String::from).unwrap_or_else(|| plain(k).to_string()),
};
o.insert(key, plain(&pair[1]));
}
Value::Object(o)
}
"regexp" => v["value"]["pattern"].clone(),
"null" | "undefined" => Value::Null,
// A page element, a window…: DevTools gives an empty object for those too.
_ => json!({}),
}
}
// ---------- tabs ----------
/// The tabs AnyBot used last on each Firefox, latest first.
fn recent() -> &'static Mutex<HashMap<u16, Vec<String>>> {
static R: OnceLock<Mutex<HashMap<u16, Vec<String>>>> = OnceLock::new();
R.get_or_init(Mutex::default)
}
/// This tab is the one in use now.
pub fn touch(port: u16, id: &str) {
let mut r = recent().lock().unwrap_or_else(|e| e.into_inner());
let l = r.entry(port).or_default();
l.retain(|t| t != id);
l.insert(0, id.to_string());
l.truncate(50);
}
/// The open tabs as DevTools lists them ({id, type: "page", url, title}), in the order DevTools
/// gives too: the one in use first (the tab showing in the window, then the ones AnyBot used
/// last, then the newest).
pub fn pages(port: u16) -> Result<Vec<Value>, String> {
let tree = command(port, "browsingContext.getTree", json!({"maxDepth": 0}))?;
let used = recent().lock().unwrap_or_else(|e| e.into_inner()).get(&port).cloned().unwrap_or_default();
let mut tabs = vec![];
for (i, c) in tree["contexts"].as_array().cloned().unwrap_or_default().iter().enumerate() {
let id = c["context"].as_str().unwrap_or("").to_string();
let info = command_within(port, "script.evaluate", json!({"expression": "JSON.stringify([document.title, document.visibilityState])", "target": {"context": id}, "awaitPromise": false, "resultOwnership": "none"}), TITLE_TIMEOUT_SECS)
.ok()
.and_then(|r| plain(&r["result"]).as_str().and_then(|s| serde_json::from_str::<Value>(s).ok()))
.unwrap_or(Value::Null);
let showing = info[1] == "visible";
let rank = used.iter().position(|u| *u == id).unwrap_or(usize::MAX);
tabs.push(((!showing, rank, std::cmp::Reverse(i)), json!({"id": id, "type": "page", "url": c["url"], "title": info[0].as_str().unwrap_or("")})));
}
tabs.sort_by_key(|t| t.0);
Ok(tabs.into_iter().map(|(_, t)| t).collect())
}
/// Open a new tab at `url` (in front). Returns its id.
pub fn new_tab(port: u16, url: &str) -> Result<String, String> {
let r = command(port, "browsingContext.create", json!({"type": "tab"}))?;
let id = r["context"].as_str().ok_or("Firefox 沒有開新分頁")?.to_string();
if url != "about:blank" {
command(port, "browsingContext.navigate", json!({"context": id, "url": url, "wait": "none"}))?;
}
touch(port, &id);
Ok(id)
}
pub fn activate(port: u16, id: &str) -> Result<(), String> {
command(port, "browsingContext.activate", json!({"context": id}))?;
touch(port, id);
Ok(())
}
pub fn close(port: u16, id: &str) -> Result<(), String> {
command(port, "browsingContext.close", json!({"context": id})).map(|_| ())
}
// ---------- DevTools commands, done the BiDi way ----------
/// WebDriver's code for a key DevTools names (Enter, ArrowDown…); a letter is itself.
fn key_value(key: &str) -> String {
match key {
"Enter" => "\u{E007}",
"Tab" => "\u{E004}",
"Escape" => "\u{E00C}",
"Backspace" => "\u{E003}",
"Delete" => "\u{E017}",
"ArrowDown" => "\u{E015}",
"ArrowUp" => "\u{E013}",
"ArrowLeft" => "\u{E012}",
"ArrowRight" => "\u{E014}",
"PageDown" => "\u{E00F}",
"PageUp" => "\u{E00E}",
"Home" => "\u{E011}",
"End" => "\u{E010}",
other => return other.to_string(),
}
.to_string()
}
/// DevTools' modifier bits (alt 1, ctrl 2, meta 4, shift 8) as WebDriver keys.
fn modifier_keys(bits: i64) -> Vec<&'static str> {
[(1, "\u{E00A}"), (2, "\u{E009}"), (4, "\u{E03D}"), (8, "\u{E008}")].iter().filter(|(b, _)| bits & b != 0).map(|(_, k)| *k).collect()
}
fn keys(port: u16, ctx: &str, actions: Vec<Value>) -> Result<Value, String> {
command(port, "input.performActions", json!({"context": ctx, "actions": [{"type": "key", "id": "keyboard", "actions": actions}]}))?;
Ok(json!({}))
}
/// Typing text: each character a real key press (Firefox puts in any character, 中文 too, the way
/// a keyboard would, so editors that only listen to typing get it). A line break is Enter.
fn type_actions(text: &str) -> Vec<Value> {
text.chars()
.filter(|c| *c != '\r')
.flat_map(|c| {
let k = if c == '\n' { "\u{E007}".to_string() } else { c.to_string() };
[json!({"type": "keyDown", "value": k}), json!({"type": "keyUp", "value": k})]
})
.collect()
}
/// Run one DevTools command (the ones browser.rs uses) on tab `ctx` of the Firefox on `port`;
/// the answer is shaped like DevTools' own.
pub fn cdp(port: u16, ctx: &str, method: &str, p: &Value) -> Result<Value, String> {
match method {
"Runtime.evaluate" => {
let by_value = p["returnByValue"].as_bool().unwrap_or(false);
let r = command(port, "script.evaluate", json!({
"expression": p["expression"], "target": {"context": ctx},
"awaitPromise": p["awaitPromise"].as_bool().unwrap_or(false),
"resultOwnership": if by_value { "none" } else { "root" },
"userActivation": p["userGesture"].as_bool().unwrap_or(false),
}))?;
if r["type"] == "exception" {
let d = &r["exceptionDetails"];
let text = d["text"].as_str().unwrap_or("").to_string();
return Ok(json!({"exceptionDetails": {"text": text, "exception": {"description": text}}}));
}
let v = &r["result"];
Ok(if by_value {
json!({"result": {"value": plain(v)}})
} else {
// A page element stays reachable by its shared id (for DOM.setFileInputFiles).
json!({"result": {"objectId": v["sharedId"].as_str().or(v["handle"].as_str())}})
})
}
"Input.dispatchMouseEvent" => {
// Firefox refuses a pointer outside the page; DevTools let it pass.
let at = |k: &str| p[k].as_f64().unwrap_or(0.0).max(0.0).round() as i64;
let mv = json!({"type": "pointerMove", "x": at("x"), "y": at("y"), "duration": 0});
let button = match p["button"].as_str() {
Some("right") => 2,
Some("middle") => 1,
_ => 0,
};
let actions = match p["type"].as_str() {
Some("mousePressed") => vec![mv, json!({"type": "pointerDown", "button": button})],
Some("mouseReleased") => vec![mv, json!({"type": "pointerUp", "button": button})],
_ => vec![mv],
};
command(port, "input.performActions", json!({"context": ctx, "actions": [{"type": "pointer", "id": "mouse", "parameters": {"pointerType": "mouse"}, "actions": actions}]}))?;
Ok(json!({}))
}
"Input.dispatchKeyEvent" => {
let key = key_value(p["key"].as_str().unwrap_or(""));
let mods = modifier_keys(p["modifiers"].as_i64().unwrap_or(0));
let mut actions = vec![];
if p["type"] == "keyUp" {
actions.push(json!({"type": "keyUp", "value": key}));
actions.extend(mods.iter().rev().map(|m| json!({"type": "keyUp", "value": m})));
} else {
actions.extend(mods.iter().map(|m| json!({"type": "keyDown", "value": m})));
actions.push(json!({"type": "keyDown", "value": key}));
}
keys(port, ctx, actions)
}
"Input.insertText" => keys(port, ctx, type_actions(p["text"].as_str().unwrap_or(""))),
"Page.captureScreenshot" => {
let mut q = json!({"context": ctx, "origin": if p["captureBeyondViewport"] == true { "document" } else { "viewport" }});
if p["format"] == "jpeg" {
q["format"] = json!({"type": "image/jpeg", "quality": p["quality"].as_f64().unwrap_or(75.0) / 100.0});
}
if let Some(c) = p.get("clip") {
q["clip"] = json!({"type": "box", "x": c["x"], "y": c["y"], "width": c["width"], "height": c["height"]});
}
let r = command(port, "browsingContext.captureScreenshot", q)?;
Ok(json!({"data": r["data"]}))
}
"Page.getLayoutMetrics" => {
let r = command(port, "script.evaluate", json!({"expression": "JSON.stringify([document.documentElement.scrollWidth, document.documentElement.scrollHeight])", "target": {"context": ctx}, "awaitPromise": false, "resultOwnership": "none"}))?;
let wh: Vec<f64> = plain(&r["result"]).as_str().and_then(|s| serde_json::from_str(s).ok()).unwrap_or_default();
Ok(json!({"cssContentSize": {"width": wh.first().copied().unwrap_or(1280.0), "height": wh.get(1).copied().unwrap_or(2000.0)}}))
}
"Page.reload" => command(port, "browsingContext.reload", json!({"context": ctx, "wait": "none"})).map(|_| json!({})),
"Page.printToPDF" => {
let r = command(port, "browsingContext.print", json!({"context": ctx, "background": p["printBackground"].as_bool().unwrap_or(true)}))?;
Ok(json!({"data": r["data"]}))
}
"Page.handleJavaScriptDialog" => {
let mut q = json!({"context": ctx, "accept": p["accept"].as_bool().unwrap_or(true)});
if let Some(t) = p["promptText"].as_str() {
q["userText"] = json!(t);
}
command(port, "browsingContext.handleUserPrompt", q).map(|_| json!({}))
}
"DOM.setFileInputFiles" => {
let el = p["objectId"].as_str().ok_or("找不到上傳檔案的欄位")?;
command(port, "input.setFiles", json!({"context": ctx, "element": {"sharedId": el}, "files": p["files"]})).map(|_| json!({}))
}
"Browser.setDownloadBehavior" | "Page.setDownloadBehavior" => {
command(port, "browser.setDownloadBehavior", json!({"downloadBehavior": {"type": "allowed", "destinationFolder": p["downloadPath"]}})).map(|_| json!({}))
}
other => Err(format!("Firefox 不支援 {other}")),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn values_come_back_as_plain_json() {
assert_eq!(plain(&json!({"type": "string", "value": "hi"})), json!("hi"));
assert_eq!(plain(&json!({"type": "number", "value": 3})), json!(3));
assert_eq!(plain(&json!({"type": "number", "value": "NaN"})), Value::Null);
assert_eq!(plain(&json!({"type": "undefined"})), Value::Null);
assert_eq!(plain(&json!({"type": "array", "value": [{"type": "boolean", "value": true}, {"type": "null"}]})), json!([true, null]));
let obj = json!({"type": "object", "value": [["a", {"type": "number", "value": 1}], ["b", {"type": "array", "value": [{"type": "string", "value": "x"}]}]]});
assert_eq!(plain(&obj), json!({"a": 1, "b": ["x"]}));
assert_eq!(plain(&json!({"type": "node", "sharedId": "abc"})), json!({}));
}
#[test]
fn keys_and_text_become_webdriver_actions() {
assert_eq!(key_value("Enter"), "\u{E007}");
assert_eq!(key_value("a"), "a");
assert_eq!(modifier_keys(2 | 8), vec!["\u{E009}", "\u{E008}"]);
let t = type_actions("中a\n");
assert_eq!(t.len(), 6);
assert_eq!(t[0], json!({"type": "keyDown", "value": "中"}));
assert_eq!(t[4], json!({"type": "keyDown", "value": "\u{E007}"}));
}
}