thread-master/apps/backend/internal/module/scout/usecase/chrome_crawler_provider.go

package usecase

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"

	"apps/backend/internal/module/scout/domain"
)

// ChromeCrawlerProvider is the private worker-to-browser boundary. The
// browser service receives decrypted state only for this request.
type ChromeCrawlerProvider interface {
	SearchChrome(ctx context.Context, storageState string, terms []string, limit int) ([]ThreadSearchResult, error)
	ResolveMediaID(ctx context.Context, storageState, permalink string) (string, error)
}

func (p *HTTPCrawlerProvider) ResolveMediaID(ctx context.Context, storageState, permalink string) (string, error) {
	if p == nil || p.Endpoint == "" || p.Token == "" {
		return "", fmt.Errorf("Chrome crawler is not configured")
	}
	body, err := json.Marshal(map[string]string{"storage_state": storageState, "permalink": permalink})
	if err != nil {
		return "", err
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.Endpoint+"/v1/threads/resolve", bytes.NewReader(body))
	if err != nil {
		return "", err
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", "Bearer "+p.Token)
	client := p.HTTP
	if client == nil {
		client = &http.Client{Timeout: 95 * time.Second}
	}
	res, err := client.Do(req)
	if err != nil {
		return "", err
	}
	defer res.Body.Close()
	raw, _ := io.ReadAll(io.LimitReader(res.Body, 64<<10))
	if res.StatusCode < 200 || res.StatusCode >= 300 {
		body := truncate(string(raw), 160)
		err := fmt.Errorf("Chrome resolver status %d: %s", res.StatusCode, body)
		if isCrawlerSessionDead(err) {
			return "", fmt.Errorf("%w (%s)", domain.ErrCrawlerSessionExpired, body)
		}
		return "", err
	}
	var out struct {
		MediaID string `json:"media_id"`
	}
	if err := json.Unmarshal(raw, &out); err != nil {
		return "", err
	}
	if !isNumericMediaID(out.MediaID) {
		return "", fmt.Errorf("Chrome resolver returned no numeric media ID")
	}
	return out.MediaID, nil
}

type HTTPCrawlerProvider struct {
	Endpoint string
	Token    string
	HTTP     *http.Client
}

func NewHTTPCrawlerProvider(endpoint, token string) *HTTPCrawlerProvider {
	return &HTTPCrawlerProvider{Endpoint: strings.TrimRight(strings.TrimSpace(endpoint), "/"), Token: strings.TrimSpace(token)}
}

func (p *HTTPCrawlerProvider) SearchChrome(ctx context.Context, storageState string, terms []string, limit int) ([]ThreadSearchResult, error) {
	if p == nil || p.Endpoint == "" || p.Token == "" {
		return nil, fmt.Errorf("Chrome crawler is not configured")
	}
	if strings.TrimSpace(storageState) == "" {
		return nil, fmt.Errorf("crawler session required")
	}
	if limit < 1 {
		limit = 10
	}
	if limit > 30 {
		limit = 30
	}
	body, err := json.Marshal(map[string]any{"storage_state": storageState, "terms": nonEmptyTerms(terms), "limit": limit})
	if err != nil {
		return nil, err
	}
	// 雙軌（熱門+最新）約兩倍時間
	runCtx, cancel := context.WithTimeout(ctx, 150*time.Second)
	defer cancel()
	req, err := http.NewRequestWithContext(runCtx, http.MethodPost, p.Endpoint+"/v1/threads/search", bytes.NewReader(body))
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", "Bearer "+p.Token)
	client := p.HTTP
	if client == nil {
		client = &http.Client{Timeout: 160 * time.Second}
	}
	res, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("Chrome crawler request failed: %w", err)
	}
	defer res.Body.Close()
	raw, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20))
	if res.StatusCode < 200 || res.StatusCode >= 300 {
		body := truncate(string(raw), 160)
		err := fmt.Errorf("Chrome crawler status %d: %s", res.StatusCode, body)
		if isCrawlerSessionDead(err) {
			return nil, fmt.Errorf("%w (%s)", domain.ErrCrawlerSessionExpired, body)
		}
		return nil, err
	}
	var out struct {
		Posts []struct {
			Permalink      string `json:"permalink"`
			Author         string `json:"author"`
			Text           string `json:"text"`
			Track          string `json:"track"`
			SerpRank       int    `json:"serp_rank"`
			PublishedAt    string `json:"published_at"`
			PublishedLabel string `json:"published_label"`
		} `json:"posts"`
	}
	if err := json.Unmarshal(raw, &out); err != nil {
		return nil, fmt.Errorf("Chrome crawler response invalid: %w", err)
	}
	results := make([]ThreadSearchResult, 0, len(out.Posts))
	for _, post := range out.Posts {
		if !isThreadsURL(post.Permalink) || strings.TrimSpace(post.Text) == "" {
			continue
		}
		track := normalizeSearchTrack(post.Track)
		pub := parsePublishedDateNano(post.PublishedAt)
		results = append(results, ThreadSearchResult{
			URL:         post.Permalink,
			Title:       post.Author,
			Snippet:     post.Text,
			PublishedAt: pub,
			Track:       track,
			SerpRank:    post.SerpRank,
		})
	}
	return results, nil
}

func normalizeSearchTrack(s string) string {
	switch strings.ToLower(strings.TrimSpace(s)) {
	case "both", "top+recent", "top_recent":
		return "both"
	case "recent", "latest", "new":
		return "recent"
	case "top", "hot", "default":
		return "top"
	default:
		return ""
	}
}

// defaultScoutSoftAgeDays：超過此天數只在 score 中降權，不淘汰候選。
const defaultScoutSoftAgeDays = 45

// minCrediblePublishedNano：早於 2020-01-01 的時間戳視為假資料／測試 stub，不套用時效過濾。
var minCrediblePublishedNano = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).UnixNano()

func isSoftAged(publishedAtNano int64, softDays int) bool {
	if publishedAtNano < minCrediblePublishedNano || softDays <= 0 {
		return false
	}
	cutoff := time.Now().UTC().AddDate(0, 0, -softDays).UnixNano()
	return publishedAtNano < cutoff
}

func isCrawlerSessionDead(err error) bool {
	if err == nil {
		return false
	}
	if errors.Is(err, domain.ErrNoCrawlerSession) || errors.Is(err, domain.ErrCrawlerSessionExpired) {
		return true
	}
	msg := strings.ToLower(err.Error())
	return strings.Contains(msg, "crawler session expired") ||
		strings.Contains(msg, "crawler session is invalid") ||
		strings.Contains(msg, "crawler session required")
}