thread-master/apps/backend/internal/module/scout/usecase/chrome_crawler_provider.go

package usecase

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"
)

// ChromeCrawlerProvider is the private worker-to-browser boundary. The
// browser service receives decrypted state only for this request.
type ChromeCrawlerProvider interface {
	SearchChrome(ctx context.Context, storageState string, terms []string, limit int) ([]ThreadSearchResult, error)
	ResolveMediaID(ctx context.Context, storageState, permalink string) (string, error)
}

func (p *HTTPCrawlerProvider) ResolveMediaID(ctx context.Context, storageState, permalink string) (string, error) {
	if p == nil || p.Endpoint == "" || p.Token == "" {
		return "", fmt.Errorf("Chrome crawler is not configured")
	}
	body, err := json.Marshal(map[string]string{"storage_state": storageState, "permalink": permalink})
	if err != nil {
		return "", err
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.Endpoint+"/v1/threads/resolve", bytes.NewReader(body))
	if err != nil {
		return "", err
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", "Bearer "+p.Token)
	client := p.HTTP
	if client == nil {
		client = &http.Client{Timeout: 95 * time.Second}
	}
	res, err := client.Do(req)
	if err != nil {
		return "", err
	}
	defer res.Body.Close()
	raw, _ := io.ReadAll(io.LimitReader(res.Body, 64<<10))
	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return "", fmt.Errorf("Chrome resolver status %d: %s", res.StatusCode, truncate(string(raw), 160))
	}
	var out struct {
		MediaID string `json:"media_id"`
	}
	if err := json.Unmarshal(raw, &out); err != nil {
		return "", err
	}
	if !isNumericMediaID(out.MediaID) {
		return "", fmt.Errorf("Chrome resolver returned no numeric media ID")
	}
	return out.MediaID, nil
}

type HTTPCrawlerProvider struct {
	Endpoint string
	Token    string
	HTTP     *http.Client
}

func NewHTTPCrawlerProvider(endpoint, token string) *HTTPCrawlerProvider {
	return &HTTPCrawlerProvider{Endpoint: strings.TrimRight(strings.TrimSpace(endpoint), "/"), Token: strings.TrimSpace(token)}
}

func (p *HTTPCrawlerProvider) SearchChrome(ctx context.Context, storageState string, terms []string, limit int) ([]ThreadSearchResult, error) {
	if p == nil || p.Endpoint == "" || p.Token == "" {
		return nil, fmt.Errorf("Chrome crawler is not configured")
	}
	if strings.TrimSpace(storageState) == "" {
		return nil, fmt.Errorf("crawler session required")
	}
	if limit < 1 {
		limit = 10
	}
	if limit > 30 {
		limit = 30
	}
	body, err := json.Marshal(map[string]any{"storage_state": storageState, "terms": nonEmptyTerms(terms), "limit": limit})
	if err != nil {
		return nil, err
	}
	runCtx, cancel := context.WithTimeout(ctx, 90*time.Second)
	defer cancel()
	req, err := http.NewRequestWithContext(runCtx, http.MethodPost, p.Endpoint+"/v1/threads/search", bytes.NewReader(body))
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", "Bearer "+p.Token)
	client := p.HTTP
	if client == nil {
		client = &http.Client{Timeout: 95 * time.Second}
	}
	res, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("Chrome crawler request failed: %w", err)
	}
	defer res.Body.Close()
	raw, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20))
	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return nil, fmt.Errorf("Chrome crawler status %d: %s", res.StatusCode, truncate(string(raw), 160))
	}
	var out struct {
		Posts []struct {
			Permalink string `json:"permalink"`
			Author    string `json:"author"`
			Text      string `json:"text"`
		} `json:"posts"`
	}
	if err := json.Unmarshal(raw, &out); err != nil {
		return nil, fmt.Errorf("Chrome crawler response invalid: %w", err)
	}
	results := make([]ThreadSearchResult, 0, len(out.Posts))
	for _, post := range out.Posts {
		if !isThreadsURL(post.Permalink) || strings.TrimSpace(post.Text) == "" {
			continue
		}
		results = append(results, ThreadSearchResult{URL: post.Permalink, Title: post.Author, Snippet: post.Text})
	}
	return results, nil
}