thread-master/backend/internal/library/placement/post_relevance.go

package placement

import (
	"strings"
	"time"
	"unicode"

	libkg "haixun-backend/internal/library/knowledge"
)

const (
	minPostBodyProductFit         = 50
	minPostBodyProductFitDiscover = 28
	minPostProductFitPatrol       = 58
	minPostProductFitDiscover     = 34
	minPostProductFitRelevant     = 62
	minPostProductFitRelaxedFinal = 38
	minPostProductFitIngestFinal  = 28
	minPostPillarAnchor           = 1
	patrolMaxPostAgeDays          = IdealMaxPostAgeDays
	relevanceMaxPostAgeDays       = 14
)

// PostScanContext carries research map + product signals for post-level filtering.
type PostScanContext struct {
	MatchTags       []string
	ProductName     string
	ProductFeatures string
	AudienceSummary string
	TargetAudience  string
	Questions       []string
	Pillars         []string
	Exclusions      []string
}

func PostScanContextFromPatrolInput(in libkg.PatrolTagInput, exclusions []string) PostScanContext {
	return PostScanContext{
		MatchTags:       append([]string{}, in.MatchTags...),
		ProductName:     strings.TrimSpace(in.ProductName),
		ProductFeatures: strings.TrimSpace(in.ProductFeatures),
		AudienceSummary: strings.TrimSpace(in.AudienceSummary),
		TargetAudience:  strings.TrimSpace(in.TargetAudience),
		Questions:       append([]string{}, in.Questions...),
		Pillars:         append([]string{}, in.Pillars...),
		Exclusions:      ExpandExclusionTokens(exclusions),
	}
}

func ScorePostProductFit(text, searchTag string, ctx PostScanContext) int {
	text = strings.ToLower(strings.TrimSpace(text))
	if text == "" {
		return 0
	}
	tagScore := ScoreProductForTag(searchTag, ctx.MatchTags)
	if tagScore == 0 && ctx.ProductName != "" {
		tagScore = ScoreProductForTag(searchTag, []string{ctx.ProductName})
	}
	postScore := ScoreProductForTag(text, ctx.MatchTags)
	if postScore == 0 && ctx.ProductName != "" {
		postScore = scoreTextAgainstPhrase(text, ctx.ProductName)
	}
	if ctx.ProductFeatures != "" {
		postScore = maxInt(postScore, scoreTextAgainstPhrase(text, ctx.ProductFeatures))
	}
	for _, pillar := range ctx.Pillars {
		postScore = maxInt(postScore, scoreTextAgainstPhrase(text, pillar)/2)
	}
	combined := postScore
	if tagScore > 0 && postScore > 0 {
		combined = (tagScore*2 + postScore*3) / 5
	} else if postScore > 0 {
		combined = postScore
	} else if tagScore > 0 {
		combined = tagScore / 2
	}
	if HasCategoryConflict(text, ctx) {
		return minInt(combined, 25)
	}
	if HasTangentialTopicMismatch(text, ctx) {
		return minInt(combined, 20)
	}
	intentScore := LocalIntentScore(text, ctx)
	if intentScore > combined {
		combined = (combined*2 + intentScore*3) / 5
	}
	return combined
}

// PassesDiscoverFilters is the crawl-stage gate: collect candidates from Search/Threads API.
// Stricter ranking happens at finalize.
func PassesDiscoverFilters(text, searchTag, postedAt string, dimension QueryDimension, tagFit, recencyDays int, ctx PostScanContext) bool {
	text = strings.TrimSpace(text)
	if text == "" {
		return false
	}
	if MatchesExclusion(text, ctx.Exclusions) {
		return false
	}
	if HasCategoryConflict(text, ctx) {
		return false
	}
	if HasTangentialTopicMismatch(text, ctx) {
		return false
	}
	if !HasPainPointDemand(text) && !HasPlacementIntent(text) {
		return false
	}
	if !PassesPostAge(postedAt, dimension, recencyDays) {
		return false
	}
	bodyFit := ScorePostBodyProductFit(text, ctx)
	effectiveFit := bodyFit
	if tagFit > 0 && bodyFit > 0 {
		effectiveFit = (tagFit + bodyFit*2) / 3
	} else if tagFit > 0 && HasPlacementIntent(text) {
		effectiveFit = tagFit / 2
	}
	if bodyFit < minPostBodyProductFitDiscover {
		if !(HasPlacementIntent(text) && effectiveFit >= minPostProductFitIngestFinal) {
			return false
		}
	}
	return effectiveFit >= minPostProductFitDiscover ||
		(HasPlacementIntent(text) && effectiveFit >= minPostProductFitIngestFinal)
}

func PassesPostScanFilters(text, searchTag, postedAt string, dimension QueryDimension, tagFit int, ctx PostScanContext) bool {
	text = strings.TrimSpace(text)
	if text == "" {
		return false
	}
	if MatchesExclusion(text, ctx.Exclusions) {
		return false
	}
	if HasCategoryConflict(text, ctx) {
		return false
	}
	if HasTangentialTopicMismatch(text, ctx) {
		return false
	}
	if !HasPainPointDemand(text) {
		return false
	}
	if !PassesPostAge(postedAt, dimension, 0) {
		return false
	}
	bodyFit := ScorePostBodyProductFit(text, ctx)
	if bodyFit < minPostBodyProductFit {
		return false
	}
	if !ProductSolvesPostPain(text, ctx) {
		return false
	}
	effectiveFit := bodyFit
	if tagFit > 0 && bodyFit > 0 {
		effectiveFit = (tagFit + bodyFit*2) / 3
	}
	minFit := minPostProductFitPatrol
	if dimension == QueryRelevance {
		minFit = minPostProductFitRelevant
	}
	return effectiveFit >= minFit
}

// ScorePostBodyProductFit scores only the post body against the product offer (ignores search tag).
func ScorePostBodyProductFit(text string, ctx PostScanContext) int {
	return ScorePostProductFit(text, "", ctx)
}

// ProductSolvesPostPain checks pain in the post aligns with what the product can solve.
func ProductSolvesPostPain(text string, ctx PostScanContext) bool {
	if HasCategoryConflict(text, ctx) {
		return false
	}
	if HasTangentialTopicMismatch(text, ctx) {
		return false
	}
	if matchesTopicAnchor(text, ctx) {
		return true
	}
	bodyFit := ScorePostBodyProductFit(text, ctx)
	if bodyFit < minPostBodyProductFit {
		return false
	}
	for _, q := range ctx.Questions {
		if tokenHits(text, q) >= 2 && bodyFit >= minPostBodyProductFit {
			return true
		}
	}
	if ctx.ProductName != "" && scoreTextAgainstPhrase(text, ctx.ProductName) >= 55 {
		return true
	}
	for _, tag := range ctx.MatchTags {
		if scoreTextAgainstPhrase(text, tag) >= 55 {
			return true
		}
	}
	return LocalIntentScore(text, ctx) >= minPostBodyProductFit
}

// PostSolvedByProduct is the final flag for UI: demand + fit + no mismatch.
func PostSolvedByProduct(text string, effectiveFit int, ctx PostScanContext) bool {
	return effectiveFit >= minPostProductFitPatrol &&
		HasPainPointDemand(text) &&
		ProductSolvesPostPain(text, ctx) &&
		!HasCategoryConflict(text, ctx) &&
		!HasTangentialTopicMismatch(text, ctx)
}

// PassesRelaxedFinalFilters keeps demand + basic fit when strict pass yields zero results.
func PassesRelaxedFinalFilters(text string, effectiveFit int, ctx PostScanContext) bool {
	text = strings.TrimSpace(text)
	if text == "" {
		return false
	}
	if MatchesExclusion(text, ctx.Exclusions) {
		return false
	}
	if HasCategoryConflict(text, ctx) {
		return false
	}
	if HasTangentialTopicMismatch(text, ctx) {
		return false
	}
	if !HasPainPointDemand(text) && !HasPlacementIntent(text) {
		return false
	}
	return effectiveFit >= minPostProductFitRelaxedFinal
}

// PassesIngestFallbackFilters keeps basic searchable posts when strict/relaxed gates yield zero.
func PassesIngestFallbackFilters(text string, effectiveFit int, ctx PostScanContext) bool {
	text = strings.TrimSpace(text)
	if text == "" {
		return false
	}
	if MatchesExclusion(text, ctx.Exclusions) {
		return false
	}
	if HasCategoryConflict(text, ctx) {
		return false
	}
	if !HasPlacementIntent(text) && !HasPainPointDemand(text) {
		return false
	}
	return effectiveFit >= minPostProductFitIngestFinal
}

func matchesTopicAnchor(text string, ctx PostScanContext) bool {
	text = strings.ToLower(text)
	hits := 0
	for _, pillar := range ctx.Pillars {
		if tokenHits(text, pillar) > 0 {
			hits++
		}
	}
	for _, q := range ctx.Questions {
		if tokenHits(text, q) >= 2 {
			hits++
		}
	}
	if hits >= minPostPillarAnchor {
		return true
	}
	for _, tag := range ctx.MatchTags {
		if tokenHits(text, tag) > 0 {
			return true
		}
	}
	if ctx.ProductName != "" && tokenHits(text, ctx.ProductName) > 0 {
		return true
	}
	for _, phrase := range []string{ctx.AudienceSummary, ctx.TargetAudience} {
		if tokenHits(text, phrase) >= 2 {
			return true
		}
	}
	return false
}

func PassesPostAge(postedAt string, dimension QueryDimension, recencyDays int) bool {
	postedAt = strings.TrimSpace(postedAt)
	if postedAt == "" {
		// Web Search snippets often lack dates; do not drop at ingest.
		return true
	}
	maxDays := patrolMaxPostAgeDays
	switch dimension {
	case QueryRelevance:
		maxDays = relevanceMaxPostAgeDays
	case QueryRecency:
		if recencyDays > 0 {
			maxDays = recencyDays
		}
	}
	return IsPostWithinMaxAge(postedAt, maxDays, time.Now())
}

func IsPostWithinMaxAge(postedAt string, maxDays int, now time.Time) bool {
	if maxDays <= 0 {
		return true
	}
	ts, ok := ParsePostedAt(postedAt)
	if !ok {
		return true
	}
	if now.IsZero() {
		now = time.Now()
	}
	cutoff := now.AddDate(0, 0, -maxDays)
	return !ts.Before(cutoff)
}

func ParsePostedAt(raw string) (time.Time, bool) {
	raw = strings.TrimSpace(raw)
	if raw == "" {
		return time.Time{}, false
	}
	layouts := []string{
		time.RFC3339,
		"2006-01-02T15:04:05Z07:00",
		"2006-01-02 15:04:05",
		"2006-01-02",
		"2006/01/02",
	}
	for _, layout := range layouts {
		if ts, err := time.Parse(layout, raw); err == nil {
			return ts, true
		}
	}
	return time.Time{}, false
}

func ExpandExclusionTokens(exclusions []string) []string {
	seen := map[string]struct{}{}
	out := []string{}
	add := func(token string) {
		token = strings.TrimSpace(strings.ToLower(token))
		if len([]rune(token)) < 2 {
			return
		}
		if _, ok := seen[token]; ok {
			return
		}
		seen[token] = struct{}{}
		out = append(out, token)
	}
	for _, rule := range exclusions {
		rule = strings.TrimSpace(rule)
		if rule == "" {
			continue
		}
		add(rule)
		for _, token := range tokenizePhrase(rule) {
			add(token)
		}
		for _, synonym := range exclusionSynonyms(rule) {
			add(synonym)
		}
	}
	return out
}

func exclusionSynonyms(rule string) []string {
	rule = strings.ToLower(rule)
	out := []string{}
	for _, group := range exclusionCategoryGroups {
		if strings.Contains(rule, group.key) {
			out = append(out, group.tokens...)
		}
	}
	return out
}

var exclusionCategoryGroups = []struct {
	key    string
	tokens []string
}{
	{key: "洗碗", tokens: []string{"洗碗", "洗碗精", "洗碗機", "碗盤", "廚房清潔", "油污"}},
	{key: "洗衣", tokens: []string{"洗衣", "洗衣精", "洗衣粉", "衣物", "衣服清洗", "污漬"}},
	{key: "沐浴", tokens: []string{"沐浴", "沐浴乳", "沐浴露", "洗澡"}},
	{key: "洗髮", tokens: []string{"洗髮", "洗髮精", "頭皮", "髮品"}},
	{key: "寵物", tokens: []string{"寵物", "貓砂", "狗糧", "貓咪", "狗狗"}},
	{key: "廚房", tokens: []string{"廚房", "抽油煙機", "鍋具"}},
}

var productCategoryGroups = []struct {
	key    string
	tokens []string
}{
	{key: "洗衣", tokens: []string{"洗衣", "洗衣精", "洗衣粉", "衣物", "衣服", "污漬", "漂白"}},
	{key: "洗碗", tokens: []string{"洗碗", "洗碗精", "洗碗機", "碗盤", "廚房", "油污"}},
	{key: "沐浴", tokens: []string{"沐浴", "沐浴乳", "沐浴露", "洗澡", "沖澡"}},
	{key: "洗髮", tokens: []string{"洗髮", "洗髮精", "洗髮乳", "頭皮", "髮絲"}},
	{key: "清潔", tokens: []string{"清潔", "打掃", "去污"}},
}

func productCategories(ctx PostScanContext) []string {
	blob := strings.ToLower(ctx.ProductName + " " + ctx.ProductFeatures + " " + strings.Join(ctx.MatchTags, " "))
	found := []string{}
	for _, group := range productCategoryGroups {
		for _, token := range group.tokens {
			if strings.Contains(blob, token) {
				found = append(found, group.key)
				break
			}
		}
	}
	return found
}

func postCategories(text string) []string {
	text = strings.ToLower(text)
	found := []string{}
	for _, group := range productCategoryGroups {
		for _, token := range group.tokens {
			if strings.Contains(text, token) {
				found = append(found, group.key)
				break
			}
		}
	}
	return found
}

var genericProductCategories = map[string]struct{}{
	"清潔": {},
}

func HasCategoryConflict(text string, ctx PostScanContext) bool {
	productCats := productCategories(ctx)
	if len(productCats) == 0 {
		return false
	}
	postCats := postCategories(text)
	if len(postCats) == 0 {
		return false
	}
	productSet := map[string]struct{}{}
	for _, c := range productCats {
		productSet[c] = struct{}{}
	}
	for _, c := range postCats {
		if _, ok := genericProductCategories[c]; ok {
			continue
		}
		if _, ok := productSet[c]; !ok {
			return true
		}
	}
	return false
}

func scoreTextAgainstPhrase(text, phrase string) int {
	text = strings.ToLower(text)
	phrase = strings.TrimSpace(strings.ToLower(phrase))
	if text == "" || phrase == "" {
		return 0
	}
	if strings.Contains(text, phrase) {
		return 85
	}
	best := 0
	for _, token := range tokenizePhrase(phrase) {
		if len([]rune(token)) < 2 {
			continue
		}
		if strings.Contains(text, token) {
			best = maxInt(best, 55)
		}
	}
	return best
}

func tokenHits(text, phrase string) int {
	text = strings.ToLower(text)
	hits := 0
	for _, token := range tokenizePhrase(phrase) {
		if len([]rune(token)) < 2 {
			continue
		}
		if strings.Contains(text, token) {
			hits++
		}
	}
	return hits
}

func tokenizePhrase(phrase string) []string {
	phrase = strings.ToLower(strings.TrimSpace(phrase))
	if phrase == "" {
		return nil
	}
	repl := strings.NewReplacer("，", " ", "、", " ", "。", " ", "；", " ", "/", " ", "｜", " ", "|", " ", "（", " ", "）", " ", "(", " ", ")", " ")
	phrase = repl.Replace(phrase)
	var tokens []string
	for _, part := range strings.Fields(phrase) {
		part = strings.Trim(part, `"'「」『』：:`)
		if part != "" {
			tokens = append(tokens, part)
		}
	}
	if len(tokens) == 0 {
		return splitRunes(phrase)
	}
	return tokens
}

func splitRunes(s string) []string {
	var out []string
	var buf []rune
	flush := func() {
		if len(buf) >= 2 {
			out = append(out, string(buf))
		}
		buf = buf[:0]
	}
	for _, r := range s {
		if unicode.Is(unicode.Han, r) {
			buf = append(buf, r)
			continue
		}
		flush()
	}
	flush()
	return out
}

func minInt(a, b int) int {
	if a < b {
		return a
	}
	return b
}