package usecase import ( "context" "fmt" "strings" "apps/backend/internal/module/scout/domain" ) const maxPipelineRawCandidates = 320 type SearchPipelineDiagnostics struct { RawCount int DuplicateCount int IrrelevantCount int EligibleCount int Stages int SourceUnavailable bool ShortfallReasons []string } type SearchPipelineResult struct { Hits []ThreadSearchResult Diagnostics SearchPipelineDiagnostics } type searchPipelineRunner struct { ctx context.Context service *Service brief *domain.RunBrief terms []string target int path string storageState string source func(context.Context, string, int) ([]ThreadSearchResult, error) secondary func(context.Context, string, int) ([]ThreadSearchResult, error) evaluator *CandidateEvaluator hits []ThreadSearchResult diagnostics SearchPipelineDiagnostics lastErr error sourceOK bool } // searchEligiblePipeline executes initial, same-source boost and optional // secondary-source stages. It evaluates each raw candidate immediately, so a // raw hit count can never satisfy target before relevance and dedupe checks. func (s *Service) searchEligiblePipeline( ctx context.Context, ownerUID int64, terms []string, brief *domain.RunBrief, target int, path string, storageState string, initialPerQuery int, ) (SearchPipelineResult, error) { if s == nil || brief == nil || len(terms) == 0 { return SearchPipelineResult{}, fmt.Errorf("%w: search pipeline input required", domain.ErrValidation) } if initialPerQuery < 1 { initialPerQuery = 1 } if initialPerQuery > 20 { initialPerQuery = 20 } runner := &searchPipelineRunner{ ctx: ctx, service: s, brief: brief, terms: nonEmptyTerms(terms), target: target, path: path, storageState: storageState, evaluator: NewCandidateEvaluator(brief), } runner.evaluator.SetHistoricalSeenLookup(func(identity string) (bool, error) { return s.Repo.HasSeenIdentity(ctx, ownerUID, identity) }) if path == domain.PathCrawler && s.Crawler != nil && storageState != "" { runner.source = func(ctx context.Context, term string, limit int) ([]ThreadSearchResult, error) { return s.Crawler.SearchChrome(ctx, storageState, []string{term}, limit) } runner.secondary = func(ctx context.Context, term string, limit int) ([]ThreadSearchResult, error) { if s.Provider == nil { return nil, fmt.Errorf("search provider is not configured") } return s.Provider.SearchThreads(ctx, []string{term}, limit) } } else if s.Provider != nil { runner.source = func(ctx context.Context, term string, limit int) ([]ThreadSearchResult, error) { return s.Provider.SearchThreads(ctx, []string{term}, limit) } } if runner.source == nil { return SearchPipelineResult{}, fmt.Errorf("%w: primary search source unavailable", domain.ErrValidation) } runner.runStage(initialPerQuery, runner.source) if target <= 0 || runner.reachedTarget() || runner.diagnostics.RawCount >= maxPipelineRawCandidates { return runner.finish() } runner.runStage(20, runner.source) if runner.reachedTarget() || runner.diagnostics.RawCount >= maxPipelineRawCandidates { return runner.finish() } // API providers such as Exa do not expose an offset in this adapter, so a // second identical request can return the same first page. When the user // approved only one term, use a small, intent-derived set of conjunctions // before giving up or switching source. The evaluator still applies the // original topic signature, so expansion increases recall without relaxing // relevance. if expansionTerms := searchPipelineExpansionTerms(runner.brief, runner.terms); len(expansionTerms) > 0 { runner.runStageForTerms(20, runner.source, expansionTerms) if runner.reachedTarget() || runner.diagnostics.RawCount >= maxPipelineRawCandidates { return runner.finish() } } if runner.secondary != nil { runner.runStage(20, runner.secondary) } return runner.finish() } func (r *searchPipelineRunner) runStage(perQuery int, source func(context.Context, string, int) ([]ThreadSearchResult, error)) { r.runStageForTerms(perQuery, source, r.terms) } func (r *searchPipelineRunner) runStageForTerms(perQuery int, source func(context.Context, string, int) ([]ThreadSearchResult, error), terms []string) { if source == nil || r.reachedTarget() || r.diagnostics.RawCount >= maxPipelineRawCandidates { return } r.diagnostics.Stages++ for _, term := range terms { if r.reachedTarget() || r.diagnostics.RawCount >= maxPipelineRawCandidates { break } remaining := maxPipelineRawCandidates - r.diagnostics.RawCount limit := perQuery if limit > remaining { limit = remaining } hits, err := source(r.ctx, term, limit) if err != nil { r.lastErr = err r.diagnostics.SourceUnavailable = true if isCrawlerSessionDead(err) { r.source = nil break } } if err == nil { r.sourceOK = true } if len(hits) > remaining { hits = hits[:remaining] } r.diagnostics.RawCount += len(hits) for _, hit := range hits { if hit.MatchedQuery == "" { hit.MatchedQuery = term } if permalink := canonicalPermalink(hit.URL); permalink != "" { hit.URL = permalink } evaluation := r.evaluator.EvaluateAt(hit, domain.NowNano()) switch evaluation.Decision { case CandidateEligible: r.hits = append(r.hits, hit) case CandidateDuplicate: r.diagnostics.DuplicateCount++ default: r.diagnostics.IrrelevantCount++ } if r.reachedTarget() { break } } } } func searchPipelineExpansionTerms(brief *domain.RunBrief, selected []string) []string { if brief == nil || brief.Mode != domain.ModeActivity || len(nonEmptyTerms(selected)) != 1 { return nil } // Keep this bounded: it is a recovery stage for a one-term approval, not a // second unrestricted planner. planActivityTerms is deterministic and uses // the same Threads short-query contract as the review UI. planned := planActivityTerms(brief.Intent) selectedKey := normalizeTopicText(selected[0]) out := make([]string, 0, 4) seen := map[string]struct{}{selectedKey: {}} for _, term := range planned { term = strings.TrimSpace(term) key := normalizeTopicText(term) if key == "" { continue } if _, exists := seen[key]; exists { continue } if len(semanticTopicParts(term)) == 0 { continue } seen[key] = struct{}{} out = append(out, term) if len(out) == 4 { break } } return out } func (r *searchPipelineRunner) reachedTarget() bool { return r.target > 0 && len(r.hits) >= r.target } func (r *searchPipelineRunner) finish() (SearchPipelineResult, error) { r.diagnostics.EligibleCount = len(r.hits) if r.target > 0 && len(r.hits) < r.target { r.diagnostics.ShortfallReasons = shortfallReasons(r.diagnostics) } if !r.sourceOK && r.lastErr != nil { return SearchPipelineResult{Hits: r.hits, Diagnostics: r.diagnostics}, r.lastErr } return SearchPipelineResult{Hits: r.hits, Diagnostics: r.diagnostics}, nil } func shortfallReasons(d SearchPipelineDiagnostics) []string { var reasons []string if d.SourceUnavailable { reasons = append(reasons, domain.ShortfallSourceUnavailable) } if d.DuplicateCount > 0 { reasons = append(reasons, domain.ShortfallDuplicateExhausted) } if d.IrrelevantCount > 0 { reasons = append(reasons, domain.ShortfallRelevanceExhausted) } if d.RawCount >= maxPipelineRawCandidates { reasons = append(reasons, domain.ShortfallLimitReached) } if len(reasons) == 0 { reasons = []string{domain.ShortfallSourceExhausted} } return reasons }