thread-master/apps/backend/scripts/threads-profile/scrape.mjs

#!/usr/bin/env node
/**
 * 抓取 Threads 公開個人頁貼文文字。
 * Usage:
 *   node scrape.mjs --username zuck [--storage /path/state.json] [--limit 12]
 * stdout: JSON { "ok": true, "username": "...", "posts": ["..."] }
 */
import { chromium } from "playwright";
import fs from "node:fs";
import path from "node:path";

function arg(name, fallback = "") {
  const i = process.argv.indexOf(name);
  if (i >= 0 && process.argv[i + 1]) return process.argv[i + 1];
  return fallback;
}

function fail(msg) {
  process.stdout.write(JSON.stringify({ ok: false, error: msg, posts: [] }) + "\n");
  process.exit(0);
}

function walkJson(data, visit) {
  if (!data || typeof data !== "object") return;
  if (Array.isArray(data)) {
    for (const item of data) walkJson(item, visit);
    return;
  }
  visit(data);
  for (const v of Object.values(data)) {
    if (v && typeof v === "object") walkJson(v, visit);
  }
}

function postText(obj) {
  return (
    obj?.caption?.text ??
    obj?.text_post_app_info?.text ??
    (typeof obj?.text === "string" ? obj.text : undefined)
  );
}

function collectFromJson(data, out, seen) {
  walkJson(data, (obj) => {
    const t = postText(obj);
    if (!t || typeof t !== "string") return;
    const text = t.trim();
    if (text.length < 8 || text.length > 2500) return;
    if (text.startsWith("http://") || text.startsWith("https://")) return;
    if (seen.has(text)) return;
    seen.add(text);
    out.push(text);
  });
}

async function main() {
  const username = arg("--username", "").replace(/^@/, "").trim();
  const storagePath = arg("--storage", "");
  const limit = Math.max(2, Math.min(30, Number(arg("--limit", "12")) || 12));
  if (!username || /[\s/]/.test(username) || username.includes("threads.")) {
    fail("username 無效：請只填帳號名，例如 kupi.cat.31");
  }

  let storageState;
  if (storagePath) {
    try {
      const raw = fs.readFileSync(path.resolve(storagePath), "utf8");
      storageState = JSON.parse(raw);
      // Playwright expects cookies/origins shape
      if (!storageState.cookies) storageState = undefined;
    } catch {
      storageState = undefined;
    }
  }

  const browser = await chromium.launch({
    headless: true,
    args: ["--disable-blink-features=AutomationControlled", "--no-sandbox"],
  });

  try {
    const context = await browser.newContext({
      userAgent:
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
      locale: "zh-TW",
      viewport: { width: 1280, height: 900 },
      storageState: storageState || undefined,
    });
    await context.addInitScript(() => {
      Object.defineProperty(navigator, "webdriver", { get: () => false });
    });

    const page = await context.newPage();
    const collected = [];
    const seen = new Set();

    page.on("response", async (response) => {
      try {
        const url = response.url();
        if (!/graphql|threads|instagram/i.test(url)) return;
        const ct = response.headers()["content-type"] || "";
        if (!ct.includes("json")) return;
        const json = await response.json();
        collectFromJson(json, collected, seen);
      } catch {
        // ignore
      }
    });

    const profileUrl = `https://www.threads.com/@${encodeURIComponent(username)}`;
    await page.goto(profileUrl, { waitUntil: "domcontentloaded", timeout: 45_000 });
    await page.waitForTimeout(1200);

    // not found?
    const bodyText = await page.locator("body").innerText().catch(() => "");
    if (/走丟|找不到|Page not found|isn't available|無法使用/i.test(bodyText) && collected.length === 0) {
      fail(`找不到 @${username}（帳號可能不存在或頁面不可用）`);
    }

    await page.waitForSelector('a[href*="/post/"]', { timeout: 12_000 }).catch(() => undefined);

    // scroll to trigger more network
    for (let i = 0; i < 4; i++) {
      await page.mouse.wheel(0, 1400);
      await page.waitForTimeout(700);
      if (collected.length >= limit) break;
    }

    // embedded JSON scripts
    if (collected.length < 2) {
      const scripts = await page.locator('script[type="application/json"]').allTextContents();
      for (const raw of scripts) {
        if (!raw || raw.length < 20) continue;
        try {
          collectFromJson(JSON.parse(raw), collected, seen);
        } catch {
          // ignore
        }
      }
    }

    // DOM fallback: post link cards text
    if (collected.length < 2) {
      const links = page.locator('a[href*="/post/"]');
      const n = await links.count();
      for (let i = 0; i < Math.min(n, 25); i++) {
        try {
          const t = (await links.nth(i).innerText()).trim();
          if (t.length >= 8 && !seen.has(t)) {
            seen.add(t);
            collected.push(t);
          }
        } catch {
          // ignore
        }
        if (collected.length >= limit) break;
      }
    }

    const posts = collected.slice(0, limit);
    process.stdout.write(
      JSON.stringify({
        ok: posts.length >= 2,
        username,
        posts,
        error:
          posts.length >= 2
            ? ""
            : `可讀取貼文不足 2 篇（目前 ${posts.length}）。可先在設定同步 Chrome Session，或改貼參考文字分析。`,
      }) + "\n",
    );
  } finally {
    await browser.close();
  }
}

main().catch((e) => fail(e instanceof Error ? e.message : String(e)));