127 lines
4.0 KiB
Go
127 lines
4.0 KiB
Go
package usecase
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// ChromeCrawlerProvider is the private worker-to-browser boundary. The
|
|
// browser service receives decrypted state only for this request.
|
|
type ChromeCrawlerProvider interface {
|
|
SearchChrome(ctx context.Context, storageState string, terms []string, limit int) ([]ThreadSearchResult, error)
|
|
ResolveMediaID(ctx context.Context, storageState, permalink string) (string, error)
|
|
}
|
|
|
|
func (p *HTTPCrawlerProvider) ResolveMediaID(ctx context.Context, storageState, permalink string) (string, error) {
|
|
if p == nil || p.Endpoint == "" || p.Token == "" {
|
|
return "", fmt.Errorf("Chrome crawler is not configured")
|
|
}
|
|
body, err := json.Marshal(map[string]string{"storage_state": storageState, "permalink": permalink})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.Endpoint+"/v1/threads/resolve", bytes.NewReader(body))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Authorization", "Bearer "+p.Token)
|
|
client := p.HTTP
|
|
if client == nil {
|
|
client = &http.Client{Timeout: 95 * time.Second}
|
|
}
|
|
res, err := client.Do(req)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer res.Body.Close()
|
|
raw, _ := io.ReadAll(io.LimitReader(res.Body, 64<<10))
|
|
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
|
return "", fmt.Errorf("Chrome resolver status %d: %s", res.StatusCode, truncate(string(raw), 160))
|
|
}
|
|
var out struct {
|
|
MediaID string `json:"media_id"`
|
|
}
|
|
if err := json.Unmarshal(raw, &out); err != nil {
|
|
return "", err
|
|
}
|
|
if !isNumericMediaID(out.MediaID) {
|
|
return "", fmt.Errorf("Chrome resolver returned no numeric media ID")
|
|
}
|
|
return out.MediaID, nil
|
|
}
|
|
|
|
type HTTPCrawlerProvider struct {
|
|
Endpoint string
|
|
Token string
|
|
HTTP *http.Client
|
|
}
|
|
|
|
func NewHTTPCrawlerProvider(endpoint, token string) *HTTPCrawlerProvider {
|
|
return &HTTPCrawlerProvider{Endpoint: strings.TrimRight(strings.TrimSpace(endpoint), "/"), Token: strings.TrimSpace(token)}
|
|
}
|
|
|
|
func (p *HTTPCrawlerProvider) SearchChrome(ctx context.Context, storageState string, terms []string, limit int) ([]ThreadSearchResult, error) {
|
|
if p == nil || p.Endpoint == "" || p.Token == "" {
|
|
return nil, fmt.Errorf("Chrome crawler is not configured")
|
|
}
|
|
if strings.TrimSpace(storageState) == "" {
|
|
return nil, fmt.Errorf("crawler session required")
|
|
}
|
|
if limit < 1 {
|
|
limit = 10
|
|
}
|
|
if limit > 30 {
|
|
limit = 30
|
|
}
|
|
body, err := json.Marshal(map[string]any{"storage_state": storageState, "terms": nonEmptyTerms(terms), "limit": limit})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
runCtx, cancel := context.WithTimeout(ctx, 90*time.Second)
|
|
defer cancel()
|
|
req, err := http.NewRequestWithContext(runCtx, http.MethodPost, p.Endpoint+"/v1/threads/search", bytes.NewReader(body))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Authorization", "Bearer "+p.Token)
|
|
client := p.HTTP
|
|
if client == nil {
|
|
client = &http.Client{Timeout: 95 * time.Second}
|
|
}
|
|
res, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Chrome crawler request failed: %w", err)
|
|
}
|
|
defer res.Body.Close()
|
|
raw, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20))
|
|
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
|
return nil, fmt.Errorf("Chrome crawler status %d: %s", res.StatusCode, truncate(string(raw), 160))
|
|
}
|
|
var out struct {
|
|
Posts []struct {
|
|
Permalink string `json:"permalink"`
|
|
Author string `json:"author"`
|
|
Text string `json:"text"`
|
|
} `json:"posts"`
|
|
}
|
|
if err := json.Unmarshal(raw, &out); err != nil {
|
|
return nil, fmt.Errorf("Chrome crawler response invalid: %w", err)
|
|
}
|
|
results := make([]ThreadSearchResult, 0, len(out.Posts))
|
|
for _, post := range out.Posts {
|
|
if !isThreadsURL(post.Permalink) || strings.TrimSpace(post.Text) == "" {
|
|
continue
|
|
}
|
|
results = append(results, ThreadSearchResult{URL: post.Permalink, Title: post.Author, Snippet: post.Text})
|
|
}
|
|
return results, nil
|
|
}
|