LazyBoy2/crates/lazyboy-core/src/web.rs

//! Public web is a service surface, independent of both browser profiles.
//! `web_search` goes through a search service (AiService gateway or native xAI
//! web search). `web_fetch` is a direct anonymous HTTP GET from this host with
//! HTML reduced to text; the xAI model-rendered path is only a fallback for
//! pages that refuse or cannot serve plain HTTP.
use anyhow::{anyhow, Context, Result};
use serde_json::{json, Value};
use std::collections::HashMap;
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant};

const FETCH_BODY_LIMIT: usize = 2 * 1024 * 1024;
const FETCH_HTML_CONVERT_LIMIT: usize = 256 * 1024;
const FETCH_CACHE_TTL: Duration = Duration::from_secs(10 * 60);
const FETCH_CACHE_CAP: usize = 64;
const BROWSER_UA: &str = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36";

fn official_xai(config: &crate::Config) -> bool {
    let official = reqwest::Url::parse(&config.base_url)
        .is_ok_and(|url| url.scheme() == "https" && url.host_str() == Some("api.x.ai"));
    official || std::env::var("LAZYBOY_WEB_PROVIDER").as_deref() == Ok("xai")
}

/// One pooled client for gateway/xAI calls: no per-call TLS handshake.
fn service_client() -> &'static reqwest::Client {
    static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
    CLIENT.get_or_init(|| {
        reqwest::Client::builder()
            .timeout(Duration::from_secs(120))
            .redirect(reqwest::redirect::Policy::none())
            .build()
            .expect("reqwest client")
    })
}

async fn call(method: &str, body: Value) -> Result<Value> {
    if std::env::var("LAZYBOY_WEB_BACKEND_URL").is_err() {
        if let Ok(config) = crate::Config::from_env() {
            if official_xai(&config) {
                return xai_call(&config, method, body).await;
            }
        }
    }
    let base = std::env::var("LAZYBOY_WEB_BACKEND_URL").map_err(|_| anyhow!(
        "Remote web service is not configured. Set LAZYBOY_WEB_BACKEND_URL to a compatible AiService gateway (and LAZYBOY_WEB_TOKEN if required). Model API credentials and browser login profiles are separate; no local/browser fallback was run."
    ))?;
    let url = format!(
        "{}/aiserver.v1.AiService/{method}",
        base.trim_end_matches('/')
    );
    let mut request = service_client()
        .post(url)
        .header("Connect-Protocol-Version", "1")
        .json(&body);
    if let Ok(token) = std::env::var("LAZYBOY_WEB_TOKEN") {
        request = request.bearer_auth(token);
    }
    let response = request.send().await.context("remote web service request")?;
    let status = response.status();
    if !status.is_success() {
        return Err(anyhow!("Remote web service returned HTTP {status}"));
    }
    let result: Value = response
        .json()
        .await
        .context("invalid remote web response")?;
    if let Some(error) = result.get("error") {
        return Err(anyhow!("Remote web service: {error}"));
    }
    Ok(result)
}

async fn xai_call(config: &crate::Config, method: &str, args: Value) -> Result<Value> {
    let prompt = if method == "RunWebSearch" {
        format!("Search the public web for the following query and report findings with source URLs. Use web search; do not answer solely from memory. Query: {}", args["searchTerm"].as_str().unwrap_or(""))
    } else {
        format!("Open this exact public URL with the web tool and return its readable content with source citations. If inaccessible or requiring login, report that; never invent page contents. URL: {}", args["url"].as_str().unwrap_or(""))
    };
    let body = json!({"model":config.model,"input":[{"role":"user","content":prompt}],
        "tools":[{"type":"web_search"}],"stream":false});
    let response = service_client()
        .post(format!(
            "{}/responses",
            config.base_url.trim_end_matches('/')
        ))
        .bearer_auth(&config.api_key)
        .json(&body)
        .send()
        .await?;
    if !response.status().is_success() {
        return Err(anyhow!(
            "xAI web service returned HTTP {}",
            response.status()
        ));
    }
    let value: Value = response.json().await?;
    xai_result(method, &value)
}

fn xai_result(method: &str, value: &Value) -> Result<Value> {
    let output = value["output"]
        .as_array()
        .ok_or_else(|| anyhow!("xAI web response has no output"))?;
    if !output
        .iter()
        .any(|item| item["type"] == "web_search_call" && item["status"] == "completed")
    {
        return Err(anyhow!("xAI returned no completed web search; unverified model text was not treated as fetched content"));
    }
    let mut text = Vec::new();
    let mut documents = Vec::new();
    for item in output {
        if let Some(content) = item["content"].as_array() {
            for part in content {
                if part["type"] == "output_text" {
                    if let Some(value) = part["text"].as_str() {
                        text.push(value);
                    }
                    if let Some(annotations) = part["annotations"].as_array() {
                        for citation in annotations {
                            if citation["type"] == "url_citation" && citation["url"].is_string() {
                                documents
                                    .push(json!({"url":citation["url"],"title":citation["title"]}));
                            }
                        }
                    }
                }
            }
        }
    }
    if text.is_empty() {
        return Err(anyhow!("xAI web response has no readable output"));
    }
    let content = text.join("\n");
    Ok(if method == "RunWebSearch" {
        json!({"answer":content,"documents":documents,"provider":"xai"})
    } else {
        json!({"content":content,"documents":documents,"provider":"xai","content_kind":"model_rendered_web_content"})
    })
}

pub async fn search(args: &Value) -> Result<Value> {
    let _timing = crate::timing::Timing::new("web_search");
    let term = args["searchTerm"]
        .as_str()
        .filter(|s| !s.trim().is_empty())
        .ok_or_else(|| anyhow!("searchTerm is required"))?;
    let mut body = json!({"searchTerm":term,"modelId":std::env::var("LAZYBOY_MODEL").unwrap_or_else(|_| "grok-4.6".into())});
    if let Some(explanation) = args["explanation"].as_str() {
        body["explanation"] = json!(explanation);
    }
    let result = call("RunWebSearch", body).await?;
    if !result["documents"].is_array() && !result["answer"].is_string() {
        return Err(anyhow!(
            "Remote search returned neither answer nor documents"
        ));
    }
    Ok(
        json!({"surface":"remote_web","browser_profile":null,"answer":result["answer"],"documents":result["documents"],"provider":result["provider"]}),
    )
}

fn is_private_host(host: &str) -> bool {
    let host = host.trim_matches(['[', ']']);
    host == "localhost"
        || host.ends_with(".localhost")
        || host.parse::<std::net::IpAddr>().is_ok_and(|ip| match ip {
            std::net::IpAddr::V4(ip) => {
                ip.is_private() || ip.is_loopback() || ip.is_link_local() || ip.is_unspecified()
            }
            std::net::IpAddr::V6(ip) => {
                ip.is_loopback()
                    || ip.is_unspecified()
                    || (ip.segments()[0] & 0xfe00 == 0xfc00)
                    || (ip.segments()[0] & 0xffc0 == 0xfe80)
            }
        })
}

fn check_public(url: &reqwest::Url) -> Result<()> {
    if !matches!(url.scheme(), "http" | "https") {
        return Err(anyhow!("only http/https URLs are allowed"));
    }
    if is_private_host(url.host_str().unwrap_or("")) {
        return Err(anyhow!("web_fetch only reaches public hosts; local/private hosts require the appropriate computer tool"));
    }
    if !url.username().is_empty() || url.password().is_some() {
        return Err(anyhow!("web_fetch does not accept credentialed URLs"));
    }
    Ok(())
}

fn public_url(args: &Value) -> Result<reqwest::Url> {
    let url = reqwest::Url::parse(
        args["url"]
            .as_str()
            .ok_or_else(|| anyhow!("url is required"))?,
    )?;
    check_public(&url)?;
    Ok(url)
}

/// Anonymous pooled client: browser-like headers, no cookie store, redirects
/// re-checked against the public-host rule so a hop cannot land on a private address.
fn fetch_client() -> &'static reqwest::Client {
    static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
    CLIENT.get_or_init(|| {
        let mut headers = reqwest::header::HeaderMap::new();
        headers.insert(
            reqwest::header::ACCEPT,
            "text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5"
                .parse()
                .unwrap(),
        );
        headers.insert(
            reqwest::header::ACCEPT_LANGUAGE,
            "zh-TW,zh;q=0.9,en;q=0.8".parse().unwrap(),
        );
        reqwest::Client::builder()
            .user_agent(BROWSER_UA)
            .default_headers(headers)
            .connect_timeout(Duration::from_secs(10))
            .timeout(Duration::from_secs(30))
            .redirect(reqwest::redirect::Policy::custom(|attempt| {
                if attempt.previous().len() >= 5 {
                    return attempt.error("too many redirects");
                }
                match check_public(attempt.url()) {
                    Ok(()) => attempt.follow(),
                    Err(error) => attempt.error(error.to_string()),
                }
            }))
            .build()
            .expect("reqwest client")
    })
}

fn fetch_cache() -> &'static Mutex<HashMap<String, (Instant, Value)>> {
    static CACHE: OnceLock<Mutex<HashMap<String, (Instant, Value)>>> = OnceLock::new();
    CACHE.get_or_init(Default::default)
}

fn cache_get(url: &str) -> Option<Value> {
    let mut cache = fetch_cache().lock().ok()?;
    cache.retain(|_, (at, _)| at.elapsed() < FETCH_CACHE_TTL);
    cache.get(url).map(|(_, value)| value.clone())
}

fn cache_put(url: &str, value: &Value) {
    if let Ok(mut cache) = fetch_cache().lock() {
        if cache.len() >= FETCH_CACHE_CAP {
            let oldest = cache
                .iter()
                .min_by_key(|(_, (at, _))| *at)
                .map(|(key, _)| key.clone());
            if let Some(key) = oldest {
                cache.remove(&key);
            }
        }
        cache.insert(url.to_string(), (Instant::now(), value.clone()));
    }
}

fn charset_label(content_type: &str, body: &[u8]) -> Option<String> {
    let from_header = content_type
        .split(';')
        .map(str::trim)
        .find_map(|part| part.strip_prefix("charset="))
        .map(|label| label.trim_matches('"').to_string());
    from_header.or_else(|| {
        let head = String::from_utf8_lossy(&body[..body.len().min(4096)]).to_ascii_lowercase();
        let index = head.find("charset=")?;
        let rest = &head[index + "charset=".len()..];
        let label: String = rest
            .trim_start_matches(['"', '\''])
            .chars()
            .take_while(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | ':'))
            .collect();
        (!label.is_empty()).then_some(label)
    })
}

fn decode_body(content_type: &str, body: &[u8]) -> String {
    let encoding = charset_label(content_type, body)
        .and_then(|label| encoding_rs::Encoding::for_label(label.as_bytes()))
        .unwrap_or(encoding_rs::UTF_8);
    encoding.decode(body).0.into_owned()
}

fn html_title(html: &str) -> Option<String> {
    let lower = html.to_ascii_lowercase();
    let start = lower.find("<title")?;
    let open_end = lower[start..].find('>')? + start + 1;
    let close = lower[open_end..].find("</title>")? + open_end;
    let title = html2text::from_read(&html.as_bytes()[open_end..close], usize::MAX)
        .ok()?
        .split_whitespace()
        .collect::<Vec<_>>()
        .join(" ");
    (!title.is_empty()).then_some(title)
}

fn looks_like_js_shell(content: &str) -> bool {
    let trimmed = content.trim();
    if trimmed.starts_with("Hi, 需要幫忙嗎") || trimmed.starts_with("# Hi, 需要幫忙嗎") {
        return trimmed.chars().filter(|c| !c.is_whitespace()).count() < 1500;
    }
    false
}

fn html_to_text(html: &str) -> Result<String> {
    let slice = if html.len() > FETCH_HTML_CONVERT_LIMIT {
        let mut end = FETCH_HTML_CONVERT_LIMIT;
        while end > 0 && !html.is_char_boundary(end) {
            end -= 1;
        }
        &html[..end]
    } else {
        html
    };
    let text = html2text::config::plain()
        .string_from_read(slice.as_bytes(), 120)
        .map_err(|error| anyhow!("HTML to text failed: {error}"))?;
    let mut out = String::with_capacity(text.len());
    let mut blank = 0;
    for line in text.lines() {
        let line = line.trim_end();
        if line.trim().is_empty() {
            blank += 1;
            if blank > 1 {
                continue;
            }
        } else {
            blank = 0;
        }
        out.push_str(line);
        out.push('\n');
    }
    Ok(out.trim().to_string())
}

fn is_textual(content_type: &str) -> bool {
    let mime = content_type
        .split(';')
        .next()
        .unwrap_or("")
        .trim()
        .to_ascii_lowercase();
    mime.starts_with("text/")
        || mime.ends_with("+xml")
        || mime.ends_with("+json")
        || matches!(
            mime.as_str(),
            "application/json"
                | "application/xml"
                | "application/javascript"
                | "application/x-yaml"
                | "application/yaml"
                | "application/x-ndjson"
                | ""
        )
}

fn is_html(content_type: &str, body: &[u8]) -> bool {
    let mime = content_type.split(';').next().unwrap_or("").trim().to_ascii_lowercase();
    if mime == "text/html" || mime == "application/xhtml+xml" {
        return true;
    }
    if !mime.is_empty() && mime != "text/plain" {
        return false;
    }
    let head = String::from_utf8_lossy(&body[..body.len().min(512)]).to_ascii_lowercase();
    head.contains("<html") || head.contains("<!doctype html")
}

#[derive(Debug)]
enum Direct {
    Text(Value),
    /// Server answered but plain HTTP cannot yield usable text (bot wall, JS-only shell).
    Unusable(String),
}

async fn direct_fetch(url: &reqwest::Url) -> Result<Direct> {
    let response = fetch_client()
        .get(url.clone())
        .send()
        .await
        .with_context(|| format!("GET {url}"))?;
    let status = response.status();
    let final_url = response.url().to_string();
    let content_type = response
        .headers()
        .get(reqwest::header::CONTENT_TYPE)
        .and_then(|value| value.to_str().ok())
        .unwrap_or("")
        .to_string();
    let mut response = response;
    let mut body = Vec::new();
    let mut body_truncated = false;
    while let Some(chunk) = response.chunk().await.context("reading response body")? {
        if body.len() + chunk.len() > FETCH_BODY_LIMIT {
            body.extend_from_slice(&chunk[..FETCH_BODY_LIMIT - body.len()]);
            body_truncated = true;
            break;
        }
        body.extend_from_slice(&chunk);
    }
    if matches!(status.as_u16(), 401 | 403 | 405 | 406 | 429 | 503) {
        return Ok(Direct::Unusable(format!("HTTP {status}")));
    }
    if !status.is_success() {
        return Err(anyhow!("{url} returned HTTP {status}"));
    }
    if !is_textual(&content_type) {
        return Err(anyhow!(
            "{url} is {content_type} ({} bytes); web_fetch only extracts text. Use shell (curl -o) on the box to download binary files.",
            body.len()
        ));
    }
    let decoded = decode_body(&content_type, &body);
    let html = is_html(&content_type, &body);
    let (title, content) = if html {
        (html_title(&decoded), html_to_text(&decoded)?)
    } else {
        (None, decoded.trim().to_string())
    };
    if html && content.chars().filter(|c| !c.is_whitespace()).count() < 80 {
        return Ok(Direct::Unusable(
            "page body has no readable text (likely JavaScript-rendered or a bot wall)".into(),
        ));
    }
    if html && looks_like_js_shell(&content) {
        return Ok(Direct::Unusable(
            "page is a JavaScript help-center shell (Hi, 需要幫忙嗎); use browser_* if the article body is required".into(),
        ));
    }
    Ok(Direct::Text(json!({
        "surface":"direct_http",
        "browser_profile":null,
        "url":final_url,
        "status":status.as_u16(),
        "content_type":content_type,
        "title":title,
        "content":content,
        "content_kind":if html {"extracted_text"} else {"raw_text"},
        "body_truncated":body_truncated,
        "provider":"direct",
    })))
}

async fn rendered_fetch(url: &reqwest::Url, reason: &str) -> Result<Value> {
    let config = crate::Config::from_env().map_err(|error| anyhow!(error))?;
    if !official_xai(&config) {
        return Err(anyhow!(
            "{url}: {reason}; no model-rendered fallback is configured for this provider"
        ));
    }
    let result = xai_call(&config, "RunWebFetch", json!({"url":url.as_str()})).await?;
    Ok(json!({
        "surface":"remote_web",
        "browser_profile":null,
        "url":url.as_str(),
        "content":result["content"],
        "content_kind":result["content_kind"],
        "provider":result["provider"],
        "citations":result["documents"],
        "fallback_reason":reason,
    }))
}

fn truncate_content(mut value: Value, max: usize) -> Value {
    let content = value["content"].as_str().unwrap_or("").to_string();
    let mut end = content.len().min(max);
    while !content.is_char_boundary(end) {
        end -= 1;
    }
    value["truncated"] = json!(end < content.len());
    value["content"] = json!(&content[..end]);
    value
}

pub async fn fetch(args: &Value) -> Result<Value> {
    let _timing = crate::timing::Timing::new("web_fetch");
    let url = public_url(args)?;
    let max = args["max_bytes"]
        .as_u64()
        .unwrap_or(100_000)
        .clamp(1, 1_048_576) as usize;
    let key = url.to_string();
    if let Some(mut hit) = cache_get(&key) {
        hit["cached"] = json!(true);
        return Ok(truncate_content(hit, max));
    }
    let mut value = match direct_fetch(&url).await? {
        Direct::Text(value) => value,
        Direct::Unusable(reason) => {
            if std::env::var("LAZYBOY_WEB_FETCH_FALLBACK").as_deref() == Ok("xai") {
                rendered_fetch(&url, &reason).await?
            } else {
                json!({
                    "surface":"direct_http",
                    "browser_profile":null,
                    "url":url.as_str(),
                    "content":"",
                    "content_kind":"blocked_plain_http",
                    "provider":"direct",
                    "fallback_reason":reason,
                    "hint":"Anonymous HTTP did not yield readable text. Use browser_* for JS/login pages, or set LAZYBOY_WEB_FETCH_FALLBACK=xai for a slow model-rendered summary.",
                })
            }
        }
    };
    value["cached"] = json!(false);
    cache_put(&key, &value);
    Ok(truncate_content(value, max))
}

#[cfg(test)]
mod tests {
    use super::*;
    #[test]
    fn xai_results_require_completed_search_and_preserve_citations() {
        let mut response = json!({"output":[{"type":"message","content":[{"type":"output_text","text":"fixture","annotations":[{"type":"url_citation","url":"https://example.com","title":"Example"}]}]}]});
        assert!(xai_result("RunWebSearch", &response).is_err());
        response["output"]
            .as_array_mut()
            .unwrap()
            .push(json!({"type":"web_search_call","status":"completed"}));
        let result = xai_result("RunWebSearch", &response).unwrap();
        assert_eq!(result["documents"][0]["url"], "https://example.com");
        assert_eq!(result["answer"], "fixture");
        assert_eq!(
            xai_result("RunWebFetch", &response).unwrap()["content_kind"],
            "model_rendered_web_content"
        );
    }
    #[test]
    fn web_is_public_and_without_credentials() {
        for url in [
            "http://localhost/a",
            "http://127.0.0.1",
            "http://10.1.2.3",
            "http://[::1]",
            "http://[fc00::1]",
            "https://user:pass@example.com",
            "file:///tmp/x",
        ] {
            assert!(public_url(&json!({"url":url})).is_err(), "{url}");
        }
        assert!(public_url(&json!({"url":"https://example.com"})).is_ok());
    }
    #[test]
    fn help_center_shell_is_unusable() {
        let shell = "# Hi, 需要幫忙嗎？\n\n## 蝦皮分潤計畫介紹\n";
        assert!(looks_like_js_shell(shell));
        let article = format!(
            "# Hi, 需要幫忙嗎？\n\n{}",
            "正文內容。".repeat(400)
        );
        assert!(!looks_like_js_shell(&article));
    }

    #[test]
    fn html_is_reduced_to_readable_text() {
        let html = "<html><head><title> Hello  &amp; World </title><style>p{}</style><script>var x=1;</script></head><body><h1>標題</h1><p>first <b>para</b></p><nav><a href=\"/x\">link</a></nav><p>second</p></body></html>";
        assert_eq!(html_title(html).as_deref(), Some("Hello & World"));
        let text = html_to_text(html).unwrap();
        assert!(text.contains("標題"), "{text}");
        assert!(text.contains("first **para**"), "{text}");
        assert!(text.contains("[1]: /x"), "{text}");
        assert!(text.contains("second"), "{text}");
        assert!(!text.contains("var x"), "{text}");
        assert!(!text.contains("p{}"), "{text}");
    }
    #[test]
    fn charset_is_taken_from_header_then_meta() {
        assert_eq!(
            charset_label("text/html; charset=Big5", b"").as_deref(),
            Some("Big5")
        );
        assert_eq!(
            charset_label("text/html", b"<meta charset=\"gbk\">").as_deref(),
            Some("gbk")
        );
        let big5 = encoding_rs::BIG5.encode("中文").0;
        assert_eq!(decode_body("text/html; charset=big5", &big5), "中文");
    }
}

#[cfg(test)]
mod service_tests {
    use super::*;
    use std::io::{Read, Write};
    #[tokio::test]
    async fn remote_contract_has_no_browser_cookie_and_no_fallback() {
        let _guard = crate::test_env::lock_async().await;
        let old_url = std::env::var("LAZYBOY_WEB_BACKEND_URL").ok();
        let old_token = std::env::var("LAZYBOY_WEB_TOKEN").ok();
        let server = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        std::env::set_var(
            "LAZYBOY_WEB_BACKEND_URL",
            format!("http://{}", server.local_addr().unwrap()),
        );
        std::env::set_var("LAZYBOY_WEB_TOKEN", "fixture-token");
        let thread = std::thread::spawn(move || {
            for (method, response) in [(
                "RunWebSearch",
                r#"{"answer":"found","documents":[{"url":"https://example.com","title":"Example","text":"fixture"}]}"#,
            )] {
                let (mut socket, _) = server.accept().unwrap();
                socket
                    .set_read_timeout(Some(std::time::Duration::from_secs(5)))
                    .unwrap();
                let mut bytes = Vec::new();
                loop {
                    let mut chunk = [0; 4096];
                    let n = socket.read(&mut chunk).unwrap();
                    assert!(n > 0);
                    bytes.extend_from_slice(&chunk[..n]);
                    let text = String::from_utf8_lossy(&bytes);
                    if let Some((headers, body)) = text.split_once("\r\n\r\n") {
                        let len = headers
                            .lines()
                            .find_map(|line| {
                                line.to_lowercase()
                                    .strip_prefix("content-length: ")
                                    .and_then(|s| s.parse::<usize>().ok())
                            })
                            .unwrap();
                        if body.len() >= len {
                            break;
                        }
                    }
                }
                let request = String::from_utf8(bytes).unwrap();
                assert!(request.starts_with(&format!("POST /aiserver.v1.AiService/{method} ")));
                assert!(request
                    .to_lowercase()
                    .contains("authorization: bearer fixture-token"));
                assert!(!request.to_lowercase().contains("\r\ncookie:"));
                write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{response}", response.len()).unwrap();
            }
        });
        let search_result = search(&json!({"searchTerm":"fixture"})).await;
        std::env::remove_var("LAZYBOY_WEB_BACKEND_URL");
        let old_model_url = std::env::var("LAZYBOY_BASE_URL").ok();
        std::env::set_var("LAZYBOY_BASE_URL", "http://unconfigured.invalid");
        let unavailable = search(&json!({"searchTerm":"fixture"})).await;
        if let Some(value) = old_model_url {
            std::env::set_var("LAZYBOY_BASE_URL", value);
        } else {
            std::env::remove_var("LAZYBOY_BASE_URL");
        }
        if let Some(value) = old_url {
            std::env::set_var("LAZYBOY_WEB_BACKEND_URL", value);
        }
        if let Some(value) = old_token {
            std::env::set_var("LAZYBOY_WEB_TOKEN", value);
        } else {
            std::env::remove_var("LAZYBOY_WEB_TOKEN");
        }
        thread.join().unwrap();
        assert_eq!(search_result.unwrap()["answer"], "found");
        assert!(unavailable
            .unwrap_err()
            .to_string()
            .contains("not configured"));
    }

    fn serve_once(server: std::net::TcpListener, response: String) -> std::thread::JoinHandle<String> {
        std::thread::spawn(move || {
            let (mut socket, _) = server.accept().unwrap();
            socket
                .set_read_timeout(Some(std::time::Duration::from_secs(5)))
                .unwrap();
            let mut bytes = Vec::new();
            loop {
                let mut chunk = [0; 4096];
                let n = socket.read(&mut chunk).unwrap();
                assert!(n > 0);
                bytes.extend_from_slice(&chunk[..n]);
                if bytes.windows(4).any(|w| w == b"\r\n\r\n") {
                    break;
                }
            }
            socket.write_all(response.as_bytes()).unwrap();
            String::from_utf8(bytes).unwrap()
        })
    }

    #[tokio::test]
    async fn direct_fetch_is_anonymous_plain_http_reduced_to_text() {
        let server = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let url = reqwest::Url::parse(&format!("http://{}/page", server.local_addr().unwrap())).unwrap();
        let body = format!(
            "<html><head><title>Fixture</title><script>x()</script></head><body><h1>你好</h1>{}</body></html>",
            "<p>readable paragraph text</p>".repeat(6)
        );
        let thread = serve_once(
            server,
            format!("HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()),
        );
        let result = match direct_fetch(&url).await.unwrap() {
            Direct::Text(value) => value,
            Direct::Unusable(reason) => panic!("{reason}"),
        };
        let request = thread.join().unwrap().to_lowercase();
        assert!(request.starts_with("get /page "));
        assert!(!request.contains("\r\ncookie:"));
        assert!(!request.contains("authorization:"));
        assert!(request.contains("user-agent: mozilla/5.0"));
        assert_eq!(result["title"], "Fixture");
        assert_eq!(result["content_kind"], "extracted_text");
        assert_eq!(result["surface"], "direct_http");
        let content = result["content"].as_str().unwrap();
        assert!(content.contains("你好"), "{content}");
        assert!(content.contains("readable paragraph text"), "{content}");
        assert!(!content.contains("x()"), "{content}");
        let cut = truncate_content(result, 6);
        assert_eq!(cut["content"], "# 你");
        assert_eq!(cut["truncated"], true);
    }

    #[tokio::test]
    async fn direct_fetch_refuses_redirect_to_private_host_and_flags_bot_walls() {
        let server = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = server.local_addr().unwrap();
        let url = reqwest::Url::parse(&format!("http://{addr}/start")).unwrap();
        let thread = serve_once(
            server,
            format!("HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:{}/internal\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", addr.port()),
        );
        let error = format!("{:#}", direct_fetch(&url).await.unwrap_err());
        thread.join().unwrap();
        assert!(error.contains("public hosts") || error.contains("redirect"), "{error}");

        let server = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let url = reqwest::Url::parse(&format!("http://{}/blocked", server.local_addr().unwrap())).unwrap();
        let thread = serve_once(
            server,
            "HTTP/1.1 403 Forbidden\r\nContent-Type: text/html\r\nContent-Length: 9\r\nConnection: close\r\n\r\n<p>no</p>".to_string(),
        );
        let outcome = direct_fetch(&url).await.unwrap();
        thread.join().unwrap();
        assert!(matches!(outcome, Direct::Unusable(reason) if reason.contains("403")));
    }
}