697 lines
27 KiB
Rust
697 lines
27 KiB
Rust
|
//! Public web is a service surface, independent of both browser profiles.
|
||
|
|
//! `web_search` goes through a search service (AiService gateway or native xAI
|
||
|
|
//! web search). `web_fetch` is a direct anonymous HTTP GET from this host with
|
||
|
|
//! HTML reduced to text; the xAI model-rendered path is only a fallback for
|
||
|
|
//! pages that refuse or cannot serve plain HTTP.
|
||
|
|
use anyhow::{anyhow, Context, Result};
|
||
|
|
use serde_json::{json, Value};
|
||
|
|
use std::collections::HashMap;
|
||
|
|
use std::sync::{Mutex, OnceLock};
|
||
|
|
use std::time::{Duration, Instant};
|
||
|
|
|
||
|
|
const FETCH_BODY_LIMIT: usize = 2 * 1024 * 1024;
|
||
|
|
const FETCH_CACHE_TTL: Duration = Duration::from_secs(10 * 60);
|
||
|
|
const FETCH_CACHE_CAP: usize = 64;
|
||
|
|
const BROWSER_UA: &str = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36";
|
||
|
|
|
||
|
|
fn official_xai(config: &crate::Config) -> bool {
|
||
|
|
let official = reqwest::Url::parse(&config.base_url)
|
||
|
|
.is_ok_and(|url| url.scheme() == "https" && url.host_str() == Some("api.x.ai"));
|
||
|
|
official || std::env::var("GROKBOY_WEB_PROVIDER").as_deref() == Ok("xai")
|
||
|
|
}
|
||
|
|
|
||
|
|
/// One pooled client for gateway/xAI calls: no per-call TLS handshake.
|
||
|
|
fn service_client() -> &'static reqwest::Client {
|
||
|
|
static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
|
||
|
|
CLIENT.get_or_init(|| {
|
||
|
|
reqwest::Client::builder()
|
||
|
|
.timeout(Duration::from_secs(120))
|
||
|
|
.redirect(reqwest::redirect::Policy::none())
|
||
|
|
.build()
|
||
|
|
.expect("reqwest client")
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn call(method: &str, body: Value) -> Result<Value> {
|
||
|
|
if std::env::var("GROKBOY_WEB_BACKEND_URL").is_err() {
|
||
|
|
if let Ok(config) = crate::Config::from_env() {
|
||
|
|
if official_xai(&config) {
|
||
|
|
return xai_call(&config, method, body).await;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
let base = std::env::var("GROKBOY_WEB_BACKEND_URL").map_err(|_| anyhow!(
|
||
|
|
"Remote web service is not configured. Set GROKBOY_WEB_BACKEND_URL to a compatible AiService gateway (and GROKBOY_WEB_TOKEN if required). Model API credentials and browser login profiles are separate; no local/browser fallback was run."
|
||
|
|
))?;
|
||
|
|
let url = format!(
|
||
|
|
"{}/aiserver.v1.AiService/{method}",
|
||
|
|
base.trim_end_matches('/')
|
||
|
|
);
|
||
|
|
let mut request = service_client()
|
||
|
|
.post(url)
|
||
|
|
.header("Connect-Protocol-Version", "1")
|
||
|
|
.json(&body);
|
||
|
|
if let Ok(token) = std::env::var("GROKBOY_WEB_TOKEN") {
|
||
|
|
request = request.bearer_auth(token);
|
||
|
|
}
|
||
|
|
let response = request.send().await.context("remote web service request")?;
|
||
|
|
let status = response.status();
|
||
|
|
if !status.is_success() {
|
||
|
|
return Err(anyhow!("Remote web service returned HTTP {status}"));
|
||
|
|
}
|
||
|
|
let result: Value = response
|
||
|
|
.json()
|
||
|
|
.await
|
||
|
|
.context("invalid remote web response")?;
|
||
|
|
if let Some(error) = result.get("error") {
|
||
|
|
return Err(anyhow!("Remote web service: {error}"));
|
||
|
|
}
|
||
|
|
Ok(result)
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn xai_call(config: &crate::Config, method: &str, args: Value) -> Result<Value> {
|
||
|
|
let prompt = if method == "RunWebSearch" {
|
||
|
|
format!("Search the public web for the following query and report findings with source URLs. Use web search; do not answer solely from memory. Query: {}", args["searchTerm"].as_str().unwrap_or(""))
|
||
|
|
} else {
|
||
|
|
format!("Open this exact public URL with the web tool and return its readable content with source citations. If inaccessible or requiring login, report that; never invent page contents. URL: {}", args["url"].as_str().unwrap_or(""))
|
||
|
|
};
|
||
|
|
let body = json!({"model":config.model,"input":[{"role":"user","content":prompt}],
|
||
|
|
"tools":[{"type":"web_search"}],"stream":false});
|
||
|
|
let response = service_client()
|
||
|
|
.post(format!(
|
||
|
|
"{}/responses",
|
||
|
|
config.base_url.trim_end_matches('/')
|
||
|
|
))
|
||
|
|
.bearer_auth(&config.api_key)
|
||
|
|
.json(&body)
|
||
|
|
.send()
|
||
|
|
.await?;
|
||
|
|
if !response.status().is_success() {
|
||
|
|
return Err(anyhow!(
|
||
|
|
"xAI web service returned HTTP {}",
|
||
|
|
response.status()
|
||
|
|
));
|
||
|
|
}
|
||
|
|
let value: Value = response.json().await?;
|
||
|
|
xai_result(method, &value)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn xai_result(method: &str, value: &Value) -> Result<Value> {
|
||
|
|
let output = value["output"]
|
||
|
|
.as_array()
|
||
|
|
.ok_or_else(|| anyhow!("xAI web response has no output"))?;
|
||
|
|
if !output
|
||
|
|
.iter()
|
||
|
|
.any(|item| item["type"] == "web_search_call" && item["status"] == "completed")
|
||
|
|
{
|
||
|
|
return Err(anyhow!("xAI returned no completed web search; unverified model text was not treated as fetched content"));
|
||
|
|
}
|
||
|
|
let mut text = Vec::new();
|
||
|
|
let mut documents = Vec::new();
|
||
|
|
for item in output {
|
||
|
|
if let Some(content) = item["content"].as_array() {
|
||
|
|
for part in content {
|
||
|
|
if part["type"] == "output_text" {
|
||
|
|
if let Some(value) = part["text"].as_str() {
|
||
|
|
text.push(value);
|
||
|
|
}
|
||
|
|
if let Some(annotations) = part["annotations"].as_array() {
|
||
|
|
for citation in annotations {
|
||
|
|
if citation["type"] == "url_citation" && citation["url"].is_string() {
|
||
|
|
documents
|
||
|
|
.push(json!({"url":citation["url"],"title":citation["title"]}));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if text.is_empty() {
|
||
|
|
return Err(anyhow!("xAI web response has no readable output"));
|
||
|
|
}
|
||
|
|
let content = text.join("\n");
|
||
|
|
Ok(if method == "RunWebSearch" {
|
||
|
|
json!({"answer":content,"documents":documents,"provider":"xai"})
|
||
|
|
} else {
|
||
|
|
json!({"content":content,"documents":documents,"provider":"xai","content_kind":"model_rendered_web_content"})
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
pub async fn search(args: &Value) -> Result<Value> {
|
||
|
|
let term = args["searchTerm"]
|
||
|
|
.as_str()
|
||
|
|
.filter(|s| !s.trim().is_empty())
|
||
|
|
.ok_or_else(|| anyhow!("searchTerm is required"))?;
|
||
|
|
let mut body = json!({"searchTerm":term,"modelId":std::env::var("GROKBOY_MODEL").unwrap_or_else(|_| "grok-4.6".into())});
|
||
|
|
if let Some(explanation) = args["explanation"].as_str() {
|
||
|
|
body["explanation"] = json!(explanation);
|
||
|
|
}
|
||
|
|
let result = call("RunWebSearch", body).await?;
|
||
|
|
if !result["documents"].is_array() && !result["answer"].is_string() {
|
||
|
|
return Err(anyhow!(
|
||
|
|
"Remote search returned neither answer nor documents"
|
||
|
|
));
|
||
|
|
}
|
||
|
|
Ok(
|
||
|
|
json!({"surface":"remote_web","browser_profile":null,"answer":result["answer"],"documents":result["documents"],"provider":result["provider"]}),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn is_private_host(host: &str) -> bool {
|
||
|
|
let host = host.trim_matches(['[', ']']);
|
||
|
|
host == "localhost"
|
||
|
|
|| host.ends_with(".localhost")
|
||
|
|
|| host.parse::<std::net::IpAddr>().is_ok_and(|ip| match ip {
|
||
|
|
std::net::IpAddr::V4(ip) => {
|
||
|
|
ip.is_private() || ip.is_loopback() || ip.is_link_local() || ip.is_unspecified()
|
||
|
|
}
|
||
|
|
std::net::IpAddr::V6(ip) => {
|
||
|
|
ip.is_loopback()
|
||
|
|
|| ip.is_unspecified()
|
||
|
|
|| (ip.segments()[0] & 0xfe00 == 0xfc00)
|
||
|
|
|| (ip.segments()[0] & 0xffc0 == 0xfe80)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
fn check_public(url: &reqwest::Url) -> Result<()> {
|
||
|
|
if !matches!(url.scheme(), "http" | "https") {
|
||
|
|
return Err(anyhow!("only http/https URLs are allowed"));
|
||
|
|
}
|
||
|
|
if is_private_host(url.host_str().unwrap_or("")) {
|
||
|
|
return Err(anyhow!("web_fetch only reaches public hosts; local/private hosts require the appropriate computer tool"));
|
||
|
|
}
|
||
|
|
if !url.username().is_empty() || url.password().is_some() {
|
||
|
|
return Err(anyhow!("web_fetch does not accept credentialed URLs"));
|
||
|
|
}
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
fn public_url(args: &Value) -> Result<reqwest::Url> {
|
||
|
|
let url = reqwest::Url::parse(
|
||
|
|
args["url"]
|
||
|
|
.as_str()
|
||
|
|
.ok_or_else(|| anyhow!("url is required"))?,
|
||
|
|
)?;
|
||
|
|
check_public(&url)?;
|
||
|
|
Ok(url)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Anonymous pooled client: browser-like headers, no cookie store, redirects
|
||
|
|
/// re-checked against the public-host rule so a hop cannot land on a private address.
|
||
|
|
fn fetch_client() -> &'static reqwest::Client {
|
||
|
|
static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
|
||
|
|
CLIENT.get_or_init(|| {
|
||
|
|
let mut headers = reqwest::header::HeaderMap::new();
|
||
|
|
headers.insert(
|
||
|
|
reqwest::header::ACCEPT,
|
||
|
|
"text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5"
|
||
|
|
.parse()
|
||
|
|
.unwrap(),
|
||
|
|
);
|
||
|
|
headers.insert(
|
||
|
|
reqwest::header::ACCEPT_LANGUAGE,
|
||
|
|
"zh-TW,zh;q=0.9,en;q=0.8".parse().unwrap(),
|
||
|
|
);
|
||
|
|
reqwest::Client::builder()
|
||
|
|
.user_agent(BROWSER_UA)
|
||
|
|
.default_headers(headers)
|
||
|
|
.connect_timeout(Duration::from_secs(10))
|
||
|
|
.timeout(Duration::from_secs(30))
|
||
|
|
.redirect(reqwest::redirect::Policy::custom(|attempt| {
|
||
|
|
if attempt.previous().len() >= 5 {
|
||
|
|
return attempt.error("too many redirects");
|
||
|
|
}
|
||
|
|
match check_public(attempt.url()) {
|
||
|
|
Ok(()) => attempt.follow(),
|
||
|
|
Err(error) => attempt.error(error.to_string()),
|
||
|
|
}
|
||
|
|
}))
|
||
|
|
.build()
|
||
|
|
.expect("reqwest client")
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
fn fetch_cache() -> &'static Mutex<HashMap<String, (Instant, Value)>> {
|
||
|
|
static CACHE: OnceLock<Mutex<HashMap<String, (Instant, Value)>>> = OnceLock::new();
|
||
|
|
CACHE.get_or_init(Default::default)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn cache_get(url: &str) -> Option<Value> {
|
||
|
|
let mut cache = fetch_cache().lock().ok()?;
|
||
|
|
cache.retain(|_, (at, _)| at.elapsed() < FETCH_CACHE_TTL);
|
||
|
|
cache.get(url).map(|(_, value)| value.clone())
|
||
|
|
}
|
||
|
|
|
||
|
|
fn cache_put(url: &str, value: &Value) {
|
||
|
|
if let Ok(mut cache) = fetch_cache().lock() {
|
||
|
|
if cache.len() >= FETCH_CACHE_CAP {
|
||
|
|
let oldest = cache
|
||
|
|
.iter()
|
||
|
|
.min_by_key(|(_, (at, _))| *at)
|
||
|
|
.map(|(key, _)| key.clone());
|
||
|
|
if let Some(key) = oldest {
|
||
|
|
cache.remove(&key);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
cache.insert(url.to_string(), (Instant::now(), value.clone()));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fn charset_label(content_type: &str, body: &[u8]) -> Option<String> {
|
||
|
|
let from_header = content_type
|
||
|
|
.split(';')
|
||
|
|
.map(str::trim)
|
||
|
|
.find_map(|part| part.strip_prefix("charset="))
|
||
|
|
.map(|label| label.trim_matches('"').to_string());
|
||
|
|
from_header.or_else(|| {
|
||
|
|
let head = String::from_utf8_lossy(&body[..body.len().min(4096)]).to_ascii_lowercase();
|
||
|
|
let index = head.find("charset=")?;
|
||
|
|
let rest = &head[index + "charset=".len()..];
|
||
|
|
let label: String = rest
|
||
|
|
.trim_start_matches(['"', '\''])
|
||
|
|
.chars()
|
||
|
|
.take_while(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | ':'))
|
||
|
|
.collect();
|
||
|
|
(!label.is_empty()).then_some(label)
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
fn decode_body(content_type: &str, body: &[u8]) -> String {
|
||
|
|
let encoding = charset_label(content_type, body)
|
||
|
|
.and_then(|label| encoding_rs::Encoding::for_label(label.as_bytes()))
|
||
|
|
.unwrap_or(encoding_rs::UTF_8);
|
||
|
|
encoding.decode(body).0.into_owned()
|
||
|
|
}
|
||
|
|
|
||
|
|
fn html_title(html: &str) -> Option<String> {
|
||
|
|
let lower = html.to_ascii_lowercase();
|
||
|
|
let start = lower.find("<title")?;
|
||
|
|
let open_end = lower[start..].find('>')? + start + 1;
|
||
|
|
let close = lower[open_end..].find("</title>")? + open_end;
|
||
|
|
let title = html2text::from_read(html[open_end..close].as_bytes(), usize::MAX)
|
||
|
|
.ok()?
|
||
|
|
.split_whitespace()
|
||
|
|
.collect::<Vec<_>>()
|
||
|
|
.join(" ");
|
||
|
|
(!title.is_empty()).then_some(title)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn html_to_text(html: &str) -> Result<String> {
|
||
|
|
let text = html2text::config::plain()
|
||
|
|
.string_from_read(html.as_bytes(), 120)
|
||
|
|
.map_err(|error| anyhow!("HTML to text failed: {error}"))?;
|
||
|
|
let mut out = String::with_capacity(text.len());
|
||
|
|
let mut blank = 0;
|
||
|
|
for line in text.lines() {
|
||
|
|
let line = line.trim_end();
|
||
|
|
if line.trim().is_empty() {
|
||
|
|
blank += 1;
|
||
|
|
if blank > 1 {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
blank = 0;
|
||
|
|
}
|
||
|
|
out.push_str(line);
|
||
|
|
out.push('\n');
|
||
|
|
}
|
||
|
|
Ok(out.trim().to_string())
|
||
|
|
}
|
||
|
|
|
||
|
|
fn is_textual(content_type: &str) -> bool {
|
||
|
|
let mime = content_type
|
||
|
|
.split(';')
|
||
|
|
.next()
|
||
|
|
.unwrap_or("")
|
||
|
|
.trim()
|
||
|
|
.to_ascii_lowercase();
|
||
|
|
mime.starts_with("text/")
|
||
|
|
|| mime.ends_with("+xml")
|
||
|
|
|| mime.ends_with("+json")
|
||
|
|
|| matches!(
|
||
|
|
mime.as_str(),
|
||
|
|
"application/json"
|
||
|
|
| "application/xml"
|
||
|
|
| "application/javascript"
|
||
|
|
| "application/x-yaml"
|
||
|
|
| "application/yaml"
|
||
|
|
| "application/x-ndjson"
|
||
|
|
| ""
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn is_html(content_type: &str, body: &[u8]) -> bool {
|
||
|
|
let mime = content_type.split(';').next().unwrap_or("").trim().to_ascii_lowercase();
|
||
|
|
if mime == "text/html" || mime == "application/xhtml+xml" {
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
if !mime.is_empty() && mime != "text/plain" {
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
let head = String::from_utf8_lossy(&body[..body.len().min(512)]).to_ascii_lowercase();
|
||
|
|
head.contains("<html") || head.contains("<!doctype html")
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug)]
|
||
|
|
enum Direct {
|
||
|
|
Text(Value),
|
||
|
|
/// Server answered but plain HTTP cannot yield usable text (bot wall, JS-only shell).
|
||
|
|
Unusable(String),
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn direct_fetch(url: &reqwest::Url) -> Result<Direct> {
|
||
|
|
let response = fetch_client()
|
||
|
|
.get(url.clone())
|
||
|
|
.send()
|
||
|
|
.await
|
||
|
|
.with_context(|| format!("GET {url}"))?;
|
||
|
|
let status = response.status();
|
||
|
|
let final_url = response.url().to_string();
|
||
|
|
let content_type = response
|
||
|
|
.headers()
|
||
|
|
.get(reqwest::header::CONTENT_TYPE)
|
||
|
|
.and_then(|value| value.to_str().ok())
|
||
|
|
.unwrap_or("")
|
||
|
|
.to_string();
|
||
|
|
let mut response = response;
|
||
|
|
let mut body = Vec::new();
|
||
|
|
let mut body_truncated = false;
|
||
|
|
while let Some(chunk) = response.chunk().await.context("reading response body")? {
|
||
|
|
if body.len() + chunk.len() > FETCH_BODY_LIMIT {
|
||
|
|
body.extend_from_slice(&chunk[..FETCH_BODY_LIMIT - body.len()]);
|
||
|
|
body_truncated = true;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
body.extend_from_slice(&chunk);
|
||
|
|
}
|
||
|
|
if matches!(status.as_u16(), 401 | 403 | 405 | 406 | 429 | 503) {
|
||
|
|
return Ok(Direct::Unusable(format!("HTTP {status}")));
|
||
|
|
}
|
||
|
|
if !status.is_success() {
|
||
|
|
return Err(anyhow!("{url} returned HTTP {status}"));
|
||
|
|
}
|
||
|
|
if !is_textual(&content_type) {
|
||
|
|
return Err(anyhow!(
|
||
|
|
"{url} is {content_type} ({} bytes); web_fetch only extracts text. Use shell (curl -o) on the box to download binary files.",
|
||
|
|
body.len()
|
||
|
|
));
|
||
|
|
}
|
||
|
|
let decoded = decode_body(&content_type, &body);
|
||
|
|
let html = is_html(&content_type, &body);
|
||
|
|
let (title, content) = if html {
|
||
|
|
(html_title(&decoded), html_to_text(&decoded)?)
|
||
|
|
} else {
|
||
|
|
(None, decoded.trim().to_string())
|
||
|
|
};
|
||
|
|
if html && content.chars().filter(|c| !c.is_whitespace()).count() < 80 {
|
||
|
|
return Ok(Direct::Unusable(
|
||
|
|
"page body has no readable text (likely JavaScript-rendered or a bot wall)".into(),
|
||
|
|
));
|
||
|
|
}
|
||
|
|
Ok(Direct::Text(json!({
|
||
|
|
"surface":"direct_http",
|
||
|
|
"browser_profile":null,
|
||
|
|
"url":final_url,
|
||
|
|
"status":status.as_u16(),
|
||
|
|
"content_type":content_type,
|
||
|
|
"title":title,
|
||
|
|
"content":content,
|
||
|
|
"content_kind":if html {"extracted_text"} else {"raw_text"},
|
||
|
|
"body_truncated":body_truncated,
|
||
|
|
"provider":"direct",
|
||
|
|
})))
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn rendered_fetch(url: &reqwest::Url, reason: &str) -> Result<Value> {
|
||
|
|
let config = crate::Config::from_env().map_err(|error| anyhow!(error))?;
|
||
|
|
if !official_xai(&config) {
|
||
|
|
return Err(anyhow!(
|
||
|
|
"{url}: {reason}; no model-rendered fallback is configured for this provider"
|
||
|
|
));
|
||
|
|
}
|
||
|
|
let result = xai_call(&config, "RunWebFetch", json!({"url":url.as_str()})).await?;
|
||
|
|
Ok(json!({
|
||
|
|
"surface":"remote_web",
|
||
|
|
"browser_profile":null,
|
||
|
|
"url":url.as_str(),
|
||
|
|
"content":result["content"],
|
||
|
|
"content_kind":result["content_kind"],
|
||
|
|
"provider":result["provider"],
|
||
|
|
"citations":result["documents"],
|
||
|
|
"fallback_reason":reason,
|
||
|
|
}))
|
||
|
|
}
|
||
|
|
|
||
|
|
fn truncate_content(mut value: Value, max: usize) -> Value {
|
||
|
|
let content = value["content"].as_str().unwrap_or("").to_string();
|
||
|
|
let mut end = content.len().min(max);
|
||
|
|
while !content.is_char_boundary(end) {
|
||
|
|
end -= 1;
|
||
|
|
}
|
||
|
|
value["truncated"] = json!(end < content.len());
|
||
|
|
value["content"] = json!(&content[..end]);
|
||
|
|
value
|
||
|
|
}
|
||
|
|
|
||
|
|
pub async fn fetch(args: &Value) -> Result<Value> {
|
||
|
|
let url = public_url(args)?;
|
||
|
|
let max = args["max_bytes"]
|
||
|
|
.as_u64()
|
||
|
|
.unwrap_or(100_000)
|
||
|
|
.clamp(1, 1_048_576) as usize;
|
||
|
|
let key = url.to_string();
|
||
|
|
if let Some(mut hit) = cache_get(&key) {
|
||
|
|
hit["cached"] = json!(true);
|
||
|
|
return Ok(truncate_content(hit, max));
|
||
|
|
}
|
||
|
|
let mut value = match direct_fetch(&url).await? {
|
||
|
|
Direct::Text(value) => value,
|
||
|
|
Direct::Unusable(reason) => rendered_fetch(&url, &reason).await?,
|
||
|
|
};
|
||
|
|
value["cached"] = json!(false);
|
||
|
|
cache_put(&key, &value);
|
||
|
|
Ok(truncate_content(value, max))
|
||
|
|
}
|
||
|
|
|
||
|
|
#[cfg(test)]
|
||
|
|
mod tests {
|
||
|
|
use super::*;
|
||
|
|
#[test]
|
||
|
|
fn xai_results_require_completed_search_and_preserve_citations() {
|
||
|
|
let mut response = json!({"output":[{"type":"message","content":[{"type":"output_text","text":"fixture","annotations":[{"type":"url_citation","url":"https://example.com","title":"Example"}]}]}]});
|
||
|
|
assert!(xai_result("RunWebSearch", &response).is_err());
|
||
|
|
response["output"]
|
||
|
|
.as_array_mut()
|
||
|
|
.unwrap()
|
||
|
|
.push(json!({"type":"web_search_call","status":"completed"}));
|
||
|
|
let result = xai_result("RunWebSearch", &response).unwrap();
|
||
|
|
assert_eq!(result["documents"][0]["url"], "https://example.com");
|
||
|
|
assert_eq!(result["answer"], "fixture");
|
||
|
|
assert_eq!(
|
||
|
|
xai_result("RunWebFetch", &response).unwrap()["content_kind"],
|
||
|
|
"model_rendered_web_content"
|
||
|
|
);
|
||
|
|
}
|
||
|
|
#[test]
|
||
|
|
fn web_is_public_and_without_credentials() {
|
||
|
|
for url in [
|
||
|
|
"http://localhost/a",
|
||
|
|
"http://127.0.0.1",
|
||
|
|
"http://10.1.2.3",
|
||
|
|
"http://[::1]",
|
||
|
|
"http://[fc00::1]",
|
||
|
|
"https://user:pass@example.com",
|
||
|
|
"file:///tmp/x",
|
||
|
|
] {
|
||
|
|
assert!(public_url(&json!({"url":url})).is_err(), "{url}");
|
||
|
|
}
|
||
|
|
assert!(public_url(&json!({"url":"https://example.com"})).is_ok());
|
||
|
|
}
|
||
|
|
#[test]
|
||
|
|
fn html_is_reduced_to_readable_text() {
|
||
|
|
let html = "<html><head><title> Hello & World </title><style>p{}</style><script>var x=1;</script></head><body><h1>標題</h1><p>first <b>para</b></p><nav><a href=\"/x\">link</a></nav><p>second</p></body></html>";
|
||
|
|
assert_eq!(html_title(html).as_deref(), Some("Hello & World"));
|
||
|
|
let text = html_to_text(html).unwrap();
|
||
|
|
assert!(text.contains("標題"), "{text}");
|
||
|
|
assert!(text.contains("first **para**"), "{text}");
|
||
|
|
assert!(text.contains("[1]: /x"), "{text}");
|
||
|
|
assert!(text.contains("second"), "{text}");
|
||
|
|
assert!(!text.contains("var x"), "{text}");
|
||
|
|
assert!(!text.contains("p{}"), "{text}");
|
||
|
|
}
|
||
|
|
#[test]
|
||
|
|
fn charset_is_taken_from_header_then_meta() {
|
||
|
|
assert_eq!(
|
||
|
|
charset_label("text/html; charset=Big5", b"").as_deref(),
|
||
|
|
Some("Big5")
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
charset_label("text/html", b"<meta charset=\"gbk\">").as_deref(),
|
||
|
|
Some("gbk")
|
||
|
|
);
|
||
|
|
let big5 = encoding_rs::BIG5.encode("中文").0;
|
||
|
|
assert_eq!(decode_body("text/html; charset=big5", &big5), "中文");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#[cfg(test)]
|
||
|
|
mod service_tests {
|
||
|
|
use super::*;
|
||
|
|
use std::io::{Read, Write};
|
||
|
|
#[tokio::test]
|
||
|
|
async fn remote_contract_has_no_browser_cookie_and_no_fallback() {
|
||
|
|
let _guard = crate::test_env::lock_async().await;
|
||
|
|
let old_url = std::env::var("GROKBOY_WEB_BACKEND_URL").ok();
|
||
|
|
let old_token = std::env::var("GROKBOY_WEB_TOKEN").ok();
|
||
|
|
let server = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||
|
|
std::env::set_var(
|
||
|
|
"GROKBOY_WEB_BACKEND_URL",
|
||
|
|
format!("http://{}", server.local_addr().unwrap()),
|
||
|
|
);
|
||
|
|
std::env::set_var("GROKBOY_WEB_TOKEN", "fixture-token");
|
||
|
|
let thread = std::thread::spawn(move || {
|
||
|
|
for (method, response) in [(
|
||
|
|
"RunWebSearch",
|
||
|
|
r#"{"answer":"found","documents":[{"url":"https://example.com","title":"Example","text":"fixture"}]}"#,
|
||
|
|
)] {
|
||
|
|
let (mut socket, _) = server.accept().unwrap();
|
||
|
|
socket
|
||
|
|
.set_read_timeout(Some(std::time::Duration::from_secs(5)))
|
||
|
|
.unwrap();
|
||
|
|
let mut bytes = Vec::new();
|
||
|
|
loop {
|
||
|
|
let mut chunk = [0; 4096];
|
||
|
|
let n = socket.read(&mut chunk).unwrap();
|
||
|
|
assert!(n > 0);
|
||
|
|
bytes.extend_from_slice(&chunk[..n]);
|
||
|
|
let text = String::from_utf8_lossy(&bytes);
|
||
|
|
if let Some((headers, body)) = text.split_once("\r\n\r\n") {
|
||
|
|
let len = headers
|
||
|
|
.lines()
|
||
|
|
.find_map(|line| {
|
||
|
|
line.to_lowercase()
|
||
|
|
.strip_prefix("content-length: ")
|
||
|
|
.and_then(|s| s.parse::<usize>().ok())
|
||
|
|
})
|
||
|
|
.unwrap();
|
||
|
|
if body.len() >= len {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
let request = String::from_utf8(bytes).unwrap();
|
||
|
|
assert!(request.starts_with(&format!("POST /aiserver.v1.AiService/{method} ")));
|
||
|
|
assert!(request
|
||
|
|
.to_lowercase()
|
||
|
|
.contains("authorization: bearer fixture-token"));
|
||
|
|
assert!(!request.to_lowercase().contains("\r\ncookie:"));
|
||
|
|
write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{response}", response.len()).unwrap();
|
||
|
|
}
|
||
|
|
});
|
||
|
|
let search_result = search(&json!({"searchTerm":"fixture"})).await;
|
||
|
|
std::env::remove_var("GROKBOY_WEB_BACKEND_URL");
|
||
|
|
let old_model_url = std::env::var("GROKBOY_BASE_URL").ok();
|
||
|
|
std::env::set_var("GROKBOY_BASE_URL", "http://unconfigured.invalid");
|
||
|
|
let unavailable = search(&json!({"searchTerm":"fixture"})).await;
|
||
|
|
if let Some(value) = old_model_url {
|
||
|
|
std::env::set_var("GROKBOY_BASE_URL", value);
|
||
|
|
} else {
|
||
|
|
std::env::remove_var("GROKBOY_BASE_URL");
|
||
|
|
}
|
||
|
|
if let Some(value) = old_url {
|
||
|
|
std::env::set_var("GROKBOY_WEB_BACKEND_URL", value);
|
||
|
|
}
|
||
|
|
if let Some(value) = old_token {
|
||
|
|
std::env::set_var("GROKBOY_WEB_TOKEN", value);
|
||
|
|
} else {
|
||
|
|
std::env::remove_var("GROKBOY_WEB_TOKEN");
|
||
|
|
}
|
||
|
|
thread.join().unwrap();
|
||
|
|
assert_eq!(search_result.unwrap()["answer"], "found");
|
||
|
|
assert!(unavailable
|
||
|
|
.unwrap_err()
|
||
|
|
.to_string()
|
||
|
|
.contains("not configured"));
|
||
|
|
}
|
||
|
|
|
||
|
|
fn serve_once(server: std::net::TcpListener, response: String) -> std::thread::JoinHandle<String> {
|
||
|
|
std::thread::spawn(move || {
|
||
|
|
let (mut socket, _) = server.accept().unwrap();
|
||
|
|
socket
|
||
|
|
.set_read_timeout(Some(std::time::Duration::from_secs(5)))
|
||
|
|
.unwrap();
|
||
|
|
let mut bytes = Vec::new();
|
||
|
|
loop {
|
||
|
|
let mut chunk = [0; 4096];
|
||
|
|
let n = socket.read(&mut chunk).unwrap();
|
||
|
|
assert!(n > 0);
|
||
|
|
bytes.extend_from_slice(&chunk[..n]);
|
||
|
|
if bytes.windows(4).any(|w| w == b"\r\n\r\n") {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
socket.write_all(response.as_bytes()).unwrap();
|
||
|
|
String::from_utf8(bytes).unwrap()
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test]
|
||
|
|
async fn direct_fetch_is_anonymous_plain_http_reduced_to_text() {
|
||
|
|
let server = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||
|
|
let url = reqwest::Url::parse(&format!("http://{}/page", server.local_addr().unwrap())).unwrap();
|
||
|
|
let body = format!(
|
||
|
|
"<html><head><title>Fixture</title><script>x()</script></head><body><h1>你好</h1>{}</body></html>",
|
||
|
|
"<p>readable paragraph text</p>".repeat(6)
|
||
|
|
);
|
||
|
|
let thread = serve_once(
|
||
|
|
server,
|
||
|
|
format!("HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()),
|
||
|
|
);
|
||
|
|
let result = match direct_fetch(&url).await.unwrap() {
|
||
|
|
Direct::Text(value) => value,
|
||
|
|
Direct::Unusable(reason) => panic!("{reason}"),
|
||
|
|
};
|
||
|
|
let request = thread.join().unwrap().to_lowercase();
|
||
|
|
assert!(request.starts_with("get /page "));
|
||
|
|
assert!(!request.contains("\r\ncookie:"));
|
||
|
|
assert!(!request.contains("authorization:"));
|
||
|
|
assert!(request.contains("user-agent: mozilla/5.0"));
|
||
|
|
assert_eq!(result["title"], "Fixture");
|
||
|
|
assert_eq!(result["content_kind"], "extracted_text");
|
||
|
|
assert_eq!(result["surface"], "direct_http");
|
||
|
|
let content = result["content"].as_str().unwrap();
|
||
|
|
assert!(content.contains("你好"), "{content}");
|
||
|
|
assert!(content.contains("readable paragraph text"), "{content}");
|
||
|
|
assert!(!content.contains("x()"), "{content}");
|
||
|
|
let cut = truncate_content(result, 4);
|
||
|
|
assert_eq!(cut["content"], "你");
|
||
|
|
assert_eq!(cut["truncated"], true);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test]
|
||
|
|
async fn direct_fetch_refuses_redirect_to_private_host_and_flags_bot_walls() {
|
||
|
|
let server = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||
|
|
let addr = server.local_addr().unwrap();
|
||
|
|
let url = reqwest::Url::parse(&format!("http://{addr}/start")).unwrap();
|
||
|
|
let thread = serve_once(
|
||
|
|
server,
|
||
|
|
format!("HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:{}/internal\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", addr.port()),
|
||
|
|
);
|
||
|
|
let error = format!("{:#}", direct_fetch(&url).await.unwrap_err());
|
||
|
|
thread.join().unwrap();
|
||
|
|
assert!(error.contains("public hosts") || error.contains("redirect"), "{error}");
|
||
|
|
|
||
|
|
let server = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||
|
|
let url = reqwest::Url::parse(&format!("http://{}/blocked", server.local_addr().unwrap())).unwrap();
|
||
|
|
let thread = serve_once(
|
||
|
|
server,
|
||
|
|
"HTTP/1.1 403 Forbidden\r\nContent-Type: text/html\r\nContent-Length: 9\r\nConnection: close\r\n\r\n<p>no</p>".to_string(),
|
||
|
|
);
|
||
|
|
let outcome = direct_fetch(&url).await.unwrap();
|
||
|
|
thread.join().unwrap();
|
||
|
|
assert!(matches!(outcome, Direct::Unusable(reason) if reason.contains("403")));
|
||
|
|
}
|
||
|
|
}
|