thread-master/apps/backend/internal/module/studio/usecase/profile_scrape.go

package usecase

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

// profileFetchFunc is overridable in tests.
var profileFetchFunc = defaultFetchProfilePostTexts

// SetProfileFetchForTest overrides profile scrape (tests). Restore with returned cleanup.
func SetProfileFetchForTest(fn func(ctx context.Context, username, storageStateJSON string, limit int) ([]string, error)) (restore func()) {
	old := profileFetchFunc
	if fn == nil {
		profileFetchFunc = defaultFetchProfilePostTexts
	} else {
		profileFetchFunc = fn
	}
	return func() { profileFetchFunc = old }
}

// fetchProfilePostTexts tries to read public Threads posts for @username.
// storageStateJSON is optional Playwright storageState (from Chrome extension sync).
func fetchProfilePostTexts(ctx context.Context, username, storageStateJSON string, limit int) ([]string, error) {
	return profileFetchFunc(ctx, username, storageStateJSON, limit)
}

// defaultFetchProfilePostTexts uses Node Playwright scraper (real browser).
// Pure HTTP cannot read Threads profile posts (logged-out shell / JS-rendered).
func defaultFetchProfilePostTexts(ctx context.Context, username, storageStateJSON string, limit int) ([]string, error) {
	username = strings.TrimPrefix(strings.TrimSpace(username), "@")
	if username == "" {
		return nil, fmt.Errorf("empty username")
	}
	if limit <= 0 {
		limit = 12
	}

	script, err := findProfileScrapeScript()
	if err != nil {
		return nil, err
	}

	// optional storage state temp file
	var storagePath string
	if strings.TrimSpace(storageStateJSON) != "" {
		// ensure valid JSON object with cookies
		var probe struct {
			Cookies []json.RawMessage `json:"cookies"`
		}
		if json.Unmarshal([]byte(storageStateJSON), &probe) == nil && len(probe.Cookies) > 0 {
			f, ferr := os.CreateTemp("", "haixun-storage-*.json")
			if ferr == nil {
				_, _ = f.WriteString(storageStateJSON)
				_ = f.Close()
				storagePath = f.Name()
				defer os.Remove(storagePath)
			}
		}
	}

	// timeout for browser scrape
	if _, ok := ctx.Deadline(); !ok {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, 55*time.Second)
		defer cancel()
	}

	args := []string{script, "--username", username, "--limit", fmt.Sprintf("%d", limit)}
	if storagePath != "" {
		args = append(args, "--storage", storagePath)
	}

	cmd := exec.CommandContext(ctx, "node", args...)
	browsersPath, err := playwrightBrowsersPath(script)
	if err != nil {
		return nil, err
	}
	cmd.Env = append(os.Environ(),
		"PLAYWRIGHT_BROWSERS_PATH="+browsersPath,
		"PLAYWRIGHT_HOST_PLATFORM_OVERRIDE=ubuntu24.04-x64",
	)
	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		msg := strings.TrimSpace(stderr.String())
		if msg == "" {
			msg = err.Error()
		}
		return nil, fmt.Errorf("profile scrape failed: %s", truncate(msg, 240))
	}

	var out struct {
		OK       bool     `json:"ok"`
		Posts    []string `json:"posts"`
		Error    string   `json:"error"`
		Username string   `json:"username"`
	}
	if err := json.Unmarshal(stdout.Bytes(), &out); err != nil {
		return nil, fmt.Errorf("profile scrape bad json: %w", err)
	}
	if len(out.Posts) > 0 {
		return out.Posts, nil
	}
	if out.Error != "" {
		return nil, fmt.Errorf("%s", out.Error)
	}
	return nil, fmt.Errorf("找不到 @%s 的公開貼文", username)
}

func findProfileScrapeScript() (string, error) {
	var candidates []string
	// release layout: <release>/bin/{gateway,worker} + <release>/scripts/threads-profile/scrape.mjs
	if exe, err := os.Executable(); err == nil {
		if resolved, rerr := filepath.EvalSymlinks(exe); rerr == nil {
			exe = resolved
		}
		releaseDir := filepath.Dir(filepath.Dir(exe)) // <release>/bin/worker -> <release>
		candidates = append(candidates,
			filepath.Join(releaseDir, "scripts", "threads-profile", "scrape.mjs"),
		)
	}
	// relative to process cwd and source file location (local dev)
	candidates = append(candidates,
		"scripts/threads-profile/scrape.mjs",
		"apps/backend/scripts/threads-profile/scrape.mjs",
		filepath.Join("..", "scripts", "threads-profile", "scrape.mjs"),
	)
	// from this source file: internal/module/studio/usecase -> ../../../../scripts/...
	if _, file, _, ok := runtime.Caller(0); ok {
		base := filepath.Dir(file)
		candidates = append(candidates,
			filepath.Join(base, "..", "..", "..", "..", "scripts", "threads-profile", "scrape.mjs"),
		)
	}
	if wd, err := os.Getwd(); err == nil {
		candidates = append(candidates,
			filepath.Join(wd, "scripts", "threads-profile", "scrape.mjs"),
			filepath.Join(wd, "apps", "backend", "scripts", "threads-profile", "scrape.mjs"),
		)
	}
	for _, c := range candidates {
		if st, err := os.Stat(c); err == nil && !st.IsDir() {
			abs, _ := filepath.Abs(c)
			return abs, nil
		}
	}
	return "", fmt.Errorf("threads profile scrape script not found (scripts/threads-profile/scrape.mjs)")
}

func playwrightBrowsersPath(script string) (string, error) {
	scriptDir := filepath.Dir(script)
	archive := filepath.Join(scriptDir, "playwright-browsers.tar.gz")
	if st, err := os.Stat(archive); err == nil && !st.IsDir() {
		runtimeRoot := os.Getenv("PLAYWRIGHT_RUNTIME_BROWSERS_PATH")
		if runtimeRoot == "" {
			runtimeRoot = filepath.Join("/var/lib/harbor", "playwright")
		}
		target := filepath.Join(runtimeRoot, "1.55.1-ubuntu24.04-x64")
		ready := filepath.Join(target, ".ready")
		if _, err := os.Stat(ready); err == nil {
			return target, nil
		}
		if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil {
			return "", fmt.Errorf("prepare Playwright browser directory: %w", err)
		}
		tmp := fmt.Sprintf("%s.tmp-%d", target, os.Getpid())
		_ = os.RemoveAll(tmp)
		if err := os.Mkdir(tmp, 0o750); err != nil {
			return "", fmt.Errorf("prepare Playwright browser extraction: %w", err)
		}
		defer os.RemoveAll(tmp)
		if output, err := exec.Command("tar", "-xzf", archive, "-C", tmp).CombinedOutput(); err != nil {
			return "", fmt.Errorf("extract Playwright browsers: %s", truncate(strings.TrimSpace(string(output)), 240))
		}
		if err := os.WriteFile(filepath.Join(tmp, ".ready"), []byte("1.55.1\n"), 0o640); err != nil {
			return "", fmt.Errorf("mark Playwright browsers ready: %w", err)
		}
		if err := os.Rename(tmp, target); err != nil {
			if _, statErr := os.Stat(ready); statErr != nil {
				return "", fmt.Errorf("activate Playwright browsers: %w", err)
			}
		}
		return target, nil
	}
	local := filepath.Join(filepath.Dir(script), "node_modules", "playwright-core", ".local-browsers")
	if st, err := os.Stat(local); err == nil && st.IsDir() {
		return "0", nil
	}
	if v := os.Getenv("PLAYWRIGHT_BROWSERS_PATH"); v != "" {
		return v, nil
	}
	// default cache used by npx playwright install
	home, _ := os.UserHomeDir()
	if home != "" {
		return filepath.Join(home, ".cache", "ms-playwright"), nil
	}
	return "", nil
}