444 lines
13 KiB
Go
444 lines
13 KiB
Go
|
package usecase
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"sort"
|
|||
|
|
"strings"
|
|||
|
|
"unicode"
|
|||
|
|
"unicode/utf8"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
// TopicConcept is one necessary concept group in a topic. A candidate must
|
|||
|
|
// match at least one core or approved alias from every group.
|
|||
|
|
type TopicConcept struct {
|
|||
|
|
Name string
|
|||
|
|
Cores []string
|
|||
|
|
Aliases []string
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// TopicSignature is the hard relevance contract for one scan. Search
|
|||
|
|
// modifiers and conversation anchors are intentionally not concepts.
|
|||
|
|
type TopicSignature struct {
|
|||
|
|
Intent string
|
|||
|
|
Concepts []TopicConcept
|
|||
|
|
Anchors []string
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// TopicMatch is deliberately small: callers need the decision, matched
|
|||
|
|
// concepts for diagnostics, and missing concepts for shortfall reasons.
|
|||
|
|
type TopicMatch struct {
|
|||
|
|
Matched bool
|
|||
|
|
MatchedCores []string
|
|||
|
|
MissingGroups []string
|
|||
|
|
Reason string
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// NewTopicSignature builds a signature from the original intent and the
|
|||
|
|
// approved search terms. Approved terms may expose an explicit alias (for
|
|||
|
|
// example "鬼滅" for "鬼滅之刃") or split a compact intent into concepts (for
|
|||
|
|
// example "台北 市集" for "台北週末市集").
|
|||
|
|
func NewTopicSignature(intent string, approvedTerms []string) TopicSignature {
|
|||
|
|
intent = normalizeTopicTerm(intent)
|
|||
|
|
sig := TopicSignature{Intent: intent, Anchors: append([]string(nil), topicAnchors...)}
|
|||
|
|
intentParts := semanticTopicParts(intent)
|
|||
|
|
naturalIntent := len(intentParts) == 1 && isNaturalTopicIntent(intent)
|
|||
|
|
if naturalIntent {
|
|||
|
|
if derived := deriveNaturalTopicParts(intent, approvedTerms); len(derived) > 0 {
|
|||
|
|
intentParts = derived
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
approvedParts := make([]string, 0)
|
|||
|
|
approvedTermParts := make([][]string, 0, len(approvedTerms))
|
|||
|
|
for _, term := range approvedTerms {
|
|||
|
|
parts := semanticTopicParts(term)
|
|||
|
|
if len(parts) == 0 {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
approvedTermParts = append(approvedTermParts, parts)
|
|||
|
|
approvedParts = append(approvedParts, parts...)
|
|||
|
|
}
|
|||
|
|
approvedParts = uniqueTopicTerms(approvedParts)
|
|||
|
|
|
|||
|
|
// A compact long phrase such as 台北週末市集 is often not present
|
|||
|
|
// contiguously in a post. When approved terms explicitly split it, use the
|
|||
|
|
// split concepts instead of making the unsplittable phrase mandatory. Named
|
|||
|
|
// entities (鬼滅之刃) remain one concept because their approved form is
|
|||
|
|
// still a single term.
|
|||
|
|
splitApproved := !naturalIntent && len(intentParts) == 1 && len(approvedParts) > 1 &&
|
|||
|
|
!containsTopicTerm(approvedParts, intentParts[0])
|
|||
|
|
keepIntentParts := !splitApproved
|
|||
|
|
if keepIntentParts {
|
|||
|
|
for _, part := range intentParts {
|
|||
|
|
sig.addConcept(part, "")
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for _, termParts := range approvedTermParts {
|
|||
|
|
// A multi-concept query ("後端 外包") is a conjunction, not an
|
|||
|
|
// alias declaration. Treating each token as an alias would let a post
|
|||
|
|
// mentioning only "後端" pass a signature that requires "後端工程師".
|
|||
|
|
explicitAlias := len(termParts) == 1
|
|||
|
|
for _, part := range termParts {
|
|||
|
|
if isTopicAnchor(part) {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
matched := false
|
|||
|
|
for i := range sig.Concepts {
|
|||
|
|
concept := &sig.Concepts[i]
|
|||
|
|
for _, core := range concept.Cores {
|
|||
|
|
if topicEquivalent(part, core) {
|
|||
|
|
matched = true
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
// Work-intent vocabulary is a bounded synonym group. A
|
|||
|
|
// user asking for 接案 should still see a post that says
|
|||
|
|
// 外包, but this never turns arbitrary query fragments into
|
|||
|
|
// aliases for a role or named entity.
|
|||
|
|
if isWorkTopicTerm(core) && isWorkTopicTerm(part) {
|
|||
|
|
concept.addAlias(part)
|
|||
|
|
matched = true
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
// Only a single-token approved term explicitly grants an
|
|||
|
|
// alias. Never infer aliases from a conjunction's fragments.
|
|||
|
|
if explicitAlias && topicContains(core, part) && topicRuneCount(part) >= 2 {
|
|||
|
|
concept.addAlias(part)
|
|||
|
|
matched = true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if !matched && splitApproved {
|
|||
|
|
sig.addConcept(part, "")
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// No usable concept means the input contained only generic anchors. It is
|
|||
|
|
// safer to reject every candidate than to turn "推薦" into a topic.
|
|||
|
|
return sig
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// BuildTopicSignature is kept as a descriptive alias for callers that prefer
|
|||
|
|
// a builder-style name.
|
|||
|
|
func BuildTopicSignature(intent string, approvedTerms []string) TopicSignature {
|
|||
|
|
return NewTopicSignature(intent, approvedTerms)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *TopicSignature) addConcept(core, alias string) {
|
|||
|
|
core = normalizeTopicTerm(core)
|
|||
|
|
if core == "" || isTopicAnchor(core) {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
for i := range s.Concepts {
|
|||
|
|
if topicEquivalent(s.Concepts[i].Name, core) {
|
|||
|
|
if alias != "" {
|
|||
|
|
s.Concepts[i].addAlias(alias)
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
c := TopicConcept{Name: core, Cores: []string{core}}
|
|||
|
|
if alias != "" {
|
|||
|
|
c.addAlias(alias)
|
|||
|
|
}
|
|||
|
|
s.Concepts = append(s.Concepts, c)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (c *TopicConcept) addAlias(alias string) {
|
|||
|
|
alias = normalizeTopicTerm(alias)
|
|||
|
|
if alias == "" || isTopicAnchor(alias) || topicEquivalent(alias, c.Name) {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
for _, existing := range c.Aliases {
|
|||
|
|
if topicEquivalent(existing, alias) {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
c.Aliases = append(c.Aliases, alias)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Match applies the all-concept hard gate. Matching is case-insensitive and
|
|||
|
|
// whitespace-insensitive, which handles CJK and Latin terms consistently.
|
|||
|
|
func (s TopicSignature) Match(text string) TopicMatch {
|
|||
|
|
body := normalizeTopicText(text)
|
|||
|
|
match := TopicMatch{Matched: len(s.Concepts) > 0}
|
|||
|
|
for _, concept := range s.Concepts {
|
|||
|
|
matched := ""
|
|||
|
|
candidates := append(append([]string(nil), concept.Cores...), concept.Aliases...)
|
|||
|
|
for _, candidate := range candidates {
|
|||
|
|
candidate = normalizeTopicText(candidate)
|
|||
|
|
if candidate != "" && strings.Contains(body, candidate) {
|
|||
|
|
matched = candidate
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if matched == "" {
|
|||
|
|
match.Matched = false
|
|||
|
|
match.MissingGroups = append(match.MissingGroups, concept.Name)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
match.MatchedCores = append(match.MatchedCores, matched)
|
|||
|
|
}
|
|||
|
|
if len(match.MatchedCores) > 0 {
|
|||
|
|
match.Reason = "matched core: " + strings.Join(match.MatchedCores, ", ")
|
|||
|
|
}
|
|||
|
|
if len(match.MissingGroups) > 0 {
|
|||
|
|
if match.Reason != "" {
|
|||
|
|
match.Reason += "; "
|
|||
|
|
}
|
|||
|
|
match.Reason += "missing core: " + strings.Join(match.MissingGroups, ", ")
|
|||
|
|
}
|
|||
|
|
return match
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s TopicSignature) Matches(text string) bool { return s.Match(text).Matched }
|
|||
|
|
|
|||
|
|
var topicAnchors = []string{
|
|||
|
|
"求推薦", "推薦", "分享", "心得", "活動", "怎麼辦", "詢問", "討論",
|
|||
|
|
"有人知道", "請問", "請問一下", "求助", "有沒有", "有沒有人", "有人也",
|
|||
|
|
"熱門", "最新", "近期",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func isTopicAnchor(term string) bool {
|
|||
|
|
term = normalizeTopicText(term)
|
|||
|
|
for _, anchor := range topicAnchors {
|
|||
|
|
if term == normalizeTopicText(anchor) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func semanticTopicParts(raw string) []string {
|
|||
|
|
raw = normalizeTopicTerm(raw)
|
|||
|
|
if raw == "" {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
fields := strings.FieldsFunc(raw, func(r rune) bool {
|
|||
|
|
return unicode.IsSpace(r) || strings.ContainsRune(",，、/|·。！？!?：:；;（）()【】[]", r)
|
|||
|
|
})
|
|||
|
|
parts := make([]string, 0, len(fields))
|
|||
|
|
for _, field := range fields {
|
|||
|
|
field = normalizeTopicTerm(field)
|
|||
|
|
if field == "" || isTopicAnchor(field) || isTopicModifier(field) {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
parts = append(parts, field)
|
|||
|
|
}
|
|||
|
|
return uniqueTopicTerms(parts)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Natural activity input is often a sentence ("想找後端工程師接案"), while
|
|||
|
|
// approved search terms are short variants. Treating that whole sentence as
|
|||
|
|
// one mandatory contiguous core makes every real post fail the gate. Derive
|
|||
|
|
// concepts from the approved terms instead, but keep the gate conjunctive.
|
|||
|
|
func isNaturalTopicIntent(intent string) bool {
|
|||
|
|
if len(splitTitleCores(intent)) > 0 {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if topicRuneCount(intent) > 8 {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
body := normalizeTopicText(intent)
|
|||
|
|
for _, marker := range naturalSentenceMarkers {
|
|||
|
|
if strings.Contains(body, normalizeTopicText(marker)) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for _, marker := range activityWorkMarkers {
|
|||
|
|
if strings.Contains(body, normalizeTopicText(marker)) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var naturalTopicFillers = []string{
|
|||
|
|
"想找", "想看", "想問", "我想", "我要", "需要", "可以", "有沒有", "有人",
|
|||
|
|
"適合", "什麼", "怎麼", "如何", "最近", "換季", "週末", "這週", "本週",
|
|||
|
|
"真的", "求推薦", "推薦", "分享", "心得", "活動", "討論", "請問", "哪裡", "哪家", "附近", "有", "找", "拍", "看看",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var naturalSentenceMarkers = []string{
|
|||
|
|
"想找", "想看", "想問", "我想", "我要", "需要", "可以", "有沒有", "有人",
|
|||
|
|
"適合", "什麼", "怎麼", "如何", "最近", "換季", "真的", "求推薦", "推薦", "分享", "心得", "活動", "討論", "請問", "哪裡", "哪家", "附近", "有", "找", "拍", "看看",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func deriveNaturalTopicParts(intent string, approvedTerms []string) []string {
|
|||
|
|
body := normalizeTopicText(intent)
|
|||
|
|
type candidate struct {
|
|||
|
|
term string
|
|||
|
|
count int
|
|||
|
|
}
|
|||
|
|
candidates := make([]candidate, 0, 8)
|
|||
|
|
index := make(map[string]int)
|
|||
|
|
add := func(raw string) {
|
|||
|
|
raw = cleanNaturalTopicPart(raw)
|
|||
|
|
key := normalizeTopicText(raw)
|
|||
|
|
if key == "" || topicRuneCount(raw) < 2 || !strings.Contains(body, key) {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
if i, ok := index[key]; ok {
|
|||
|
|
candidates[i].count++
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
index[key] = len(candidates)
|
|||
|
|
candidates = append(candidates, candidate{term: raw, count: 1})
|
|||
|
|
}
|
|||
|
|
for _, region := range activityRegions {
|
|||
|
|
if strings.Contains(body, normalizeTopicText(region)) {
|
|||
|
|
add(region)
|
|||
|
|
if i, ok := index[normalizeTopicText(region)]; ok {
|
|||
|
|
candidates[i].count = 2
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// Preserve explicit work/role vocabulary from the user's sentence even if
|
|||
|
|
// they unchecked the corresponding generated query variant.
|
|||
|
|
knownIntentTerms := append([]string{}, activityWorkMarkers...)
|
|||
|
|
knownIntentTerms = append(knownIntentTerms, activityRoleSpecialties...)
|
|||
|
|
knownIntentTerms = append(knownIntentTerms, "工程師")
|
|||
|
|
for _, term := range knownIntentTerms {
|
|||
|
|
if strings.Contains(body, normalizeTopicText(term)) {
|
|||
|
|
add(term)
|
|||
|
|
if i, ok := index[normalizeTopicText(term)]; ok {
|
|||
|
|
candidates[i].count = 2
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for _, term := range approvedTerms {
|
|||
|
|
for _, part := range semanticTopicParts(term) {
|
|||
|
|
add(part)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Prefer compact, repeated concepts over one-off sliding-window fragments
|
|||
|
|
// generated from a sentence. Known work vocabulary may legitimately occur
|
|||
|
|
// in only one approved conjunction (for example 接案).
|
|||
|
|
out := make([]string, 0, len(candidates))
|
|||
|
|
for _, c := range candidates {
|
|||
|
|
if c.count < 2 && !isKnownNaturalTopicTerm(c.term) {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
out = append(out, c.term)
|
|||
|
|
}
|
|||
|
|
if len(out) == 0 {
|
|||
|
|
for _, part := range semanticTopicParts(intent) {
|
|||
|
|
if cleaned := cleanNaturalTopicPart(part); cleaned != "" {
|
|||
|
|
out = append(out, cleaned)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return pruneContainedTopicParts(uniqueTopicTerms(out))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func cleanNaturalTopicPart(raw string) string {
|
|||
|
|
raw = normalizeTopicTerm(raw)
|
|||
|
|
for _, filler := range naturalTopicFillers {
|
|||
|
|
raw = strings.ReplaceAll(raw, filler, "")
|
|||
|
|
}
|
|||
|
|
for _, modifier := range []string{"熱門", "最新", "近期"} {
|
|||
|
|
raw = strings.ReplaceAll(raw, modifier, "")
|
|||
|
|
}
|
|||
|
|
return normalizeTopicTerm(raw)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func isKnownNaturalTopicTerm(term string) bool {
|
|||
|
|
term = normalizeTopicText(term)
|
|||
|
|
if term == "工程師" {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
for _, group := range [][]string{activityWorkMarkers, activityRoleSpecialties, activityRegions} {
|
|||
|
|
for _, known := range group {
|
|||
|
|
if topicEquivalent(term, known) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func isWorkTopicTerm(term string) bool {
|
|||
|
|
term = normalizeTopicText(term)
|
|||
|
|
for _, known := range activityWorkMarkers {
|
|||
|
|
if normalizeTopicText(known) == term {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func pruneContainedTopicParts(parts []string) []string {
|
|||
|
|
keep := make([]string, 0, len(parts))
|
|||
|
|
for _, part := range parts {
|
|||
|
|
partKey := normalizeTopicText(part)
|
|||
|
|
contained := false
|
|||
|
|
for _, other := range parts {
|
|||
|
|
otherKey := normalizeTopicText(other)
|
|||
|
|
if otherKey == partKey || topicRuneCount(other) <= topicRuneCount(part) {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if strings.Contains(otherKey, partKey) && !isKnownNaturalTopicTerm(part) {
|
|||
|
|
contained = true
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if !contained {
|
|||
|
|
keep = append(keep, part)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return keep
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func isTopicModifier(term string) bool {
|
|||
|
|
term = normalizeTopicText(term)
|
|||
|
|
return term == "熱門" || term == "最新" || term == "近期"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func normalizeTopicTerm(raw string) string {
|
|||
|
|
raw = strings.ReplaceAll(raw, "\u3000", " ")
|
|||
|
|
return strings.Join(strings.Fields(strings.TrimSpace(raw)), " ")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func normalizeTopicText(raw string) string {
|
|||
|
|
return strings.ReplaceAll(strings.ToLower(normalizeTopicTerm(raw)), " ", "")
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func uniqueTopicTerms(in []string) []string {
|
|||
|
|
seen := make(map[string]struct{}, len(in))
|
|||
|
|
out := make([]string, 0, len(in))
|
|||
|
|
for _, term := range in {
|
|||
|
|
term = normalizeTopicTerm(term)
|
|||
|
|
key := normalizeTopicText(term)
|
|||
|
|
if key == "" {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if _, ok := seen[key]; ok {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
seen[key] = struct{}{}
|
|||
|
|
out = append(out, term)
|
|||
|
|
}
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func containsTopicTerm(terms []string, want string) bool {
|
|||
|
|
want = normalizeTopicText(want)
|
|||
|
|
for _, term := range terms {
|
|||
|
|
if normalizeTopicText(term) == want {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func topicEquivalent(a, b string) bool { return normalizeTopicText(a) == normalizeTopicText(b) }
|
|||
|
|
|
|||
|
|
func topicContains(container, part string) bool {
|
|||
|
|
return strings.Contains(normalizeTopicText(container), normalizeTopicText(part))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func topicRuneCount(term string) int { return utf8.RuneCountInString(normalizeTopicText(term)) }
|
|||
|
|
|
|||
|
|
// Keep deterministic output if a caller serializes concepts for diagnostics.
|
|||
|
|
func (s TopicSignature) SortConcepts() {
|
|||
|
|
sort.SliceStable(s.Concepts, func(i, j int) bool { return s.Concepts[i].Name < s.Concepts[j].Name })
|
|||
|
|
}
|