thread-master/apps/backend/crawler/src/server.ts

import { chromium, type Page } from "playwright";
import { createServer } from "node:http";

type SearchRequest = { storage_state?: string; terms?: string[]; limit?: number };
type ResolveRequest = { storage_state?: string; permalink?: string };
type Post = { permalink: string; author: string; text: string };

const port = Number(process.env.SCOUT_CRAWLER_PORT || 8891);
const token = process.env.SCOUT_CRAWLER_TOKEN || "";

function isThreadsURL(value: string): boolean {
  try {
    const host = new URL(value).hostname.toLowerCase();
    return host === "threads.com" || host.endsWith(".threads.com") || host === "threads.net" || host.endsWith(".threads.net");
  } catch {
    return false;
  }
}

async function readPosts(page: Page, limit: number): Promise<Post[]> {
  const posts = new Map<string, Post>();
  const links = page.locator('a[href*="/post/"]');
  const count = Math.min(await links.count(), 50);
  for (let i = 0; i < count && posts.size < limit; i++) {
    const link = links.nth(i);
    const href = await link.getAttribute("href").catch(() => null);
    if (!href) continue;
    const permalink = href.startsWith("http") ? href : `https://www.threads.com${href}`;
    if (!isThreadsURL(permalink)) continue;
    const author = href.match(/@([^/]+)\/post/)?.[1] || "";
    const scope = link.locator("xpath=ancestor::div[position()<=6]").first();
    const text = (await scope.innerText().catch(() => "")).trim();
    if (text.length < 5) continue;
    posts.set(permalink, { permalink, author, text: text.slice(0, 2000) });
  }
  return [...posts.values()];
}

async function search(storageState: string, terms: string[], limit: number): Promise<Post[]> {
  const state = JSON.parse(storageState) as { cookies?: unknown[] };
  if (!Array.isArray(state.cookies) || state.cookies.length === 0) throw new Error("crawler session is invalid");
  const browser = await chromium.launch({ headless: true });
  try {
    const context = await browser.newContext({ storageState: state, locale: "zh-TW", timezoneId: "Asia/Taipei" });
    const page = await context.newPage();
    const query = terms.filter(Boolean).join(" ").slice(0, 180);
    await page.goto(`https://www.threads.com/search?q=${encodeURIComponent(query)}&serp_type=default`, { waitUntil: "domcontentloaded", timeout: 45_000 });
    const body = await page.locator("body").innerText().catch(() => "");
    if (page.url().includes("/login") || body.includes("登入")) throw new Error("crawler session expired");
    await page.waitForSelector('a[href*="/post/"]', { timeout: 12_000 }).catch(() => undefined);
    await page.mouse.wheel(0, 900);
    await page.waitForTimeout(1000);
    const posts = await readPosts(page, limit);
    await context.close();
    return posts;
  } finally {
    await browser.close();
  }
}

function shortcodeFromPermalink(permalink: string): string {
  return permalink.match(/\/post\/([^/?]+)/)?.[1] || "";
}

function findMediaID(value: unknown, shortcode: string): string {
  if (!value || typeof value !== "object") return "";
  if (Array.isArray(value)) { for (const item of value) { const id = findMediaID(item, shortcode); if (id) return id; } return ""; }
  const record = value as Record<string, unknown>;
  const code = String(record.code || record.shortcode || "");
  const id = String(record.id || record.pk || record.media_id || "");
  if (code === shortcode && /^\d{10,}$/.test(id)) return id;
  for (const child of Object.values(record)) { const found = findMediaID(child, shortcode); if (found) return found; }
  return "";
}

function findMediaIDInText(raw: string, shortcode: string): string {
  const escaped = shortcode.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
  const patterns = [
    new RegExp(`(?:code|shortcode)["']?\\s*[:=]\\s*["']${escaped}["'][\\s\\S]{0,800}?(?:id|pk|media_id)["']?\\s*[:=]\\s*["']?(\\d{10,})`, "i"),
    new RegExp(`(?:id|pk|media_id)["']?\\s*[:=]\\s*["']?(\\d{10,})[\\s\\S]{0,800}?(?:code|shortcode)["']?\\s*[:=]\\s*["']${escaped}["']`, "i"),
  ];
  for (const pattern of patterns) {
    const match = raw.match(pattern);
    if (match?.[1]) return match[1];
  }
  return "";
}

async function resolve(storageState: string, permalink: string): Promise<string> {
  if (!isThreadsURL(permalink)) throw new Error("invalid Threads permalink");
  const state = JSON.parse(storageState) as { cookies?: unknown[] };
  if (!Array.isArray(state.cookies) || state.cookies.length === 0) throw new Error("crawler session is invalid");
  const shortcode = shortcodeFromPermalink(permalink);
  if (!shortcode) throw new Error("permalink has no Threads shortcode");
  const browser = await chromium.launch({ headless: true });
  try {
    const context = await browser.newContext({ storageState: state, locale: "zh-TW", timezoneId: "Asia/Taipei" });
    const page = await context.newPage();
    let mediaID = "";
    const responseReads: Promise<void>[] = [];
    page.on("response", async (response) => {
      if (mediaID || !/graphql|threads|instagram/.test(response.url())) return;
      responseReads.push((async () => {
        try {
          const raw = await response.text();
          if (!mediaID) {
            try { mediaID = findMediaID(JSON.parse(raw), shortcode); } catch { /* non-JSON response */ }
          }
          if (!mediaID) mediaID = findMediaIDInText(raw, shortcode);
        } catch { /* ignored */ }
      })());
    });
    await page.goto(permalink, { waitUntil: "domcontentloaded", timeout: 45_000 });
    const body = await page.locator("body").innerText().catch(() => "");
    if (page.url().includes("/login") || body.includes("登入")) throw new Error("crawler session expired");
    await page.waitForTimeout(2500);
    await Promise.allSettled(responseReads);
    if (!mediaID) {
      mediaID = findMediaIDInText(await page.content(), shortcode);
    }
    await context.close();
    if (!mediaID) throw new Error("Threads media ID could not be resolved")
    return mediaID;
  } finally { await browser.close(); }
}

if (!token) throw new Error("SCOUT_CRAWLER_TOKEN is required");

createServer(async (req, res) => {
  const path = new URL(req.url || "/", "http://127.0.0.1").pathname;
  if (req.method !== "POST" || (path !== "/v1/threads/search" && path !== "/v1/threads/resolve")) {
    res.writeHead(404).end();
    return;
  }
  if (req.headers.authorization !== `Bearer ${token}`) {
    res.writeHead(401).end();
    return;
  }
  const body = await new Promise<string>((resolve, reject) => {
    let raw = "";
    req.setEncoding("utf8");
    req.on("data", (chunk) => { raw += chunk; if (raw.length > 300_000) req.destroy(); });
    req.on("end", () => resolve(raw));
    req.on("error", reject);
  });
  try {
    if (path === "/v1/threads/resolve") {
      const input = JSON.parse(body) as ResolveRequest;
      const mediaID = await resolve(String(input.storage_state || ""), String(input.permalink || ""));
      res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ media_id: mediaID }));
    } else {
      const input = JSON.parse(body) as SearchRequest;
      const posts = await search(String(input.storage_state || ""), Array.isArray(input.terms) ? input.terms.slice(0, 12) : [], Math.min(Math.max(Number(input.limit) || 10, 1), 30));
      res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ posts }));
    }
  } catch (error) {
    res.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ error: error instanceof Error ? error.message : "crawler failed" }));
  }
}).listen(port, "127.0.0.1");