535 lines
14 KiB
Go
535 lines
14 KiB
Go
|
package placement
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"strings"
|
|||
|
|
"time"
|
|||
|
|
"unicode"
|
|||
|
|
|
|||
|
|
libkg "haixun-backend/internal/library/knowledge"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
const (
|
|||
|
|
minPostBodyProductFit = 50
|
|||
|
|
minPostBodyProductFitDiscover = 28
|
|||
|
|
minPostProductFitPatrol = 58
|
|||
|
|
minPostProductFitDiscover = 34
|
|||
|
|
minPostProductFitRelevant = 62
|
|||
|
|
minPostProductFitRelaxedFinal = 38
|
|||
|
|
minPostProductFitIngestFinal = 28
|
|||
|
|
minPostPillarAnchor = 1
|
|||
|
|
patrolMaxPostAgeDays = IdealMaxPostAgeDays
|
|||
|
|
relevanceMaxPostAgeDays = 14
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
// PostScanContext carries research map + product signals for post-level filtering.
|
|||
|
|
type PostScanContext struct {
|
|||
|
|
MatchTags []string
|
|||
|
|
ProductName string
|
|||
|
|
ProductFeatures string
|
|||
|
|
AudienceSummary string
|
|||
|
|
TargetAudience string
|
|||
|
|
Questions []string
|
|||
|
|
Pillars []string
|
|||
|
|
Exclusions []string
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func PostScanContextFromPatrolInput(in libkg.PatrolTagInput, exclusions []string) PostScanContext {
|
|||
|
|
return PostScanContext{
|
|||
|
|
MatchTags: append([]string{}, in.MatchTags...),
|
|||
|
|
ProductName: strings.TrimSpace(in.ProductName),
|
|||
|
|
ProductFeatures: strings.TrimSpace(in.ProductFeatures),
|
|||
|
|
AudienceSummary: strings.TrimSpace(in.AudienceSummary),
|
|||
|
|
TargetAudience: strings.TrimSpace(in.TargetAudience),
|
|||
|
|
Questions: append([]string{}, in.Questions...),
|
|||
|
|
Pillars: append([]string{}, in.Pillars...),
|
|||
|
|
Exclusions: ExpandExclusionTokens(exclusions),
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func ScorePostProductFit(text, searchTag string, ctx PostScanContext) int {
|
|||
|
|
text = strings.ToLower(strings.TrimSpace(text))
|
|||
|
|
if text == "" {
|
|||
|
|
return 0
|
|||
|
|
}
|
|||
|
|
tagScore := ScoreProductForTag(searchTag, ctx.MatchTags)
|
|||
|
|
if tagScore == 0 && ctx.ProductName != "" {
|
|||
|
|
tagScore = ScoreProductForTag(searchTag, []string{ctx.ProductName})
|
|||
|
|
}
|
|||
|
|
postScore := ScoreProductForTag(text, ctx.MatchTags)
|
|||
|
|
if postScore == 0 && ctx.ProductName != "" {
|
|||
|
|
postScore = scoreTextAgainstPhrase(text, ctx.ProductName)
|
|||
|
|
}
|
|||
|
|
if ctx.ProductFeatures != "" {
|
|||
|
|
postScore = maxInt(postScore, scoreTextAgainstPhrase(text, ctx.ProductFeatures))
|
|||
|
|
}
|
|||
|
|
for _, pillar := range ctx.Pillars {
|
|||
|
|
postScore = maxInt(postScore, scoreTextAgainstPhrase(text, pillar)/2)
|
|||
|
|
}
|
|||
|
|
combined := postScore
|
|||
|
|
if tagScore > 0 && postScore > 0 {
|
|||
|
|
combined = (tagScore*2 + postScore*3) / 5
|
|||
|
|
} else if postScore > 0 {
|
|||
|
|
combined = postScore
|
|||
|
|
} else if tagScore > 0 {
|
|||
|
|
combined = tagScore / 2
|
|||
|
|
}
|
|||
|
|
if HasCategoryConflict(text, ctx) {
|
|||
|
|
return minInt(combined, 25)
|
|||
|
|
}
|
|||
|
|
if HasTangentialTopicMismatch(text, ctx) {
|
|||
|
|
return minInt(combined, 20)
|
|||
|
|
}
|
|||
|
|
intentScore := LocalIntentScore(text, ctx)
|
|||
|
|
if intentScore > combined {
|
|||
|
|
combined = (combined*2 + intentScore*3) / 5
|
|||
|
|
}
|
|||
|
|
return combined
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// PassesDiscoverFilters is the crawl-stage gate: collect candidates from Search/Threads API.
|
|||
|
|
// Stricter ranking happens at finalize.
|
|||
|
|
func PassesDiscoverFilters(text, searchTag, postedAt string, dimension QueryDimension, tagFit, recencyDays int, ctx PostScanContext) bool {
|
|||
|
|
text = strings.TrimSpace(text)
|
|||
|
|
if text == "" {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if MatchesExclusion(text, ctx.Exclusions) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if HasCategoryConflict(text, ctx) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if HasTangentialTopicMismatch(text, ctx) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if !HasPainPointDemand(text) && !HasPlacementIntent(text) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if !PassesPostAge(postedAt, dimension, recencyDays) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
bodyFit := ScorePostBodyProductFit(text, ctx)
|
|||
|
|
effectiveFit := bodyFit
|
|||
|
|
if tagFit > 0 && bodyFit > 0 {
|
|||
|
|
effectiveFit = (tagFit + bodyFit*2) / 3
|
|||
|
|
} else if tagFit > 0 && HasPlacementIntent(text) {
|
|||
|
|
effectiveFit = tagFit / 2
|
|||
|
|
}
|
|||
|
|
if bodyFit < minPostBodyProductFitDiscover {
|
|||
|
|
if !(HasPlacementIntent(text) && effectiveFit >= minPostProductFitIngestFinal) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return effectiveFit >= minPostProductFitDiscover ||
|
|||
|
|
(HasPlacementIntent(text) && effectiveFit >= minPostProductFitIngestFinal)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func PassesPostScanFilters(text, searchTag, postedAt string, dimension QueryDimension, tagFit int, ctx PostScanContext) bool {
|
|||
|
|
text = strings.TrimSpace(text)
|
|||
|
|
if text == "" {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if MatchesExclusion(text, ctx.Exclusions) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if HasCategoryConflict(text, ctx) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if HasTangentialTopicMismatch(text, ctx) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if !HasPainPointDemand(text) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if !PassesPostAge(postedAt, dimension, 0) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
bodyFit := ScorePostBodyProductFit(text, ctx)
|
|||
|
|
if bodyFit < minPostBodyProductFit {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if !ProductSolvesPostPain(text, ctx) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
effectiveFit := bodyFit
|
|||
|
|
if tagFit > 0 && bodyFit > 0 {
|
|||
|
|
effectiveFit = (tagFit + bodyFit*2) / 3
|
|||
|
|
}
|
|||
|
|
minFit := minPostProductFitPatrol
|
|||
|
|
if dimension == QueryRelevance {
|
|||
|
|
minFit = minPostProductFitRelevant
|
|||
|
|
}
|
|||
|
|
return effectiveFit >= minFit
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ScorePostBodyProductFit scores only the post body against the product offer (ignores search tag).
|
|||
|
|
func ScorePostBodyProductFit(text string, ctx PostScanContext) int {
|
|||
|
|
return ScorePostProductFit(text, "", ctx)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ProductSolvesPostPain checks pain in the post aligns with what the product can solve.
|
|||
|
|
func ProductSolvesPostPain(text string, ctx PostScanContext) bool {
|
|||
|
|
if HasCategoryConflict(text, ctx) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if HasTangentialTopicMismatch(text, ctx) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if matchesTopicAnchor(text, ctx) {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
bodyFit := ScorePostBodyProductFit(text, ctx)
|
|||
|
|
if bodyFit < minPostBodyProductFit {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
for _, q := range ctx.Questions {
|
|||
|
|
if tokenHits(text, q) >= 2 && bodyFit >= minPostBodyProductFit {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if ctx.ProductName != "" && scoreTextAgainstPhrase(text, ctx.ProductName) >= 55 {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
for _, tag := range ctx.MatchTags {
|
|||
|
|
if scoreTextAgainstPhrase(text, tag) >= 55 {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return LocalIntentScore(text, ctx) >= minPostBodyProductFit
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// PostSolvedByProduct is the final flag for UI: demand + fit + no mismatch.
|
|||
|
|
func PostSolvedByProduct(text string, effectiveFit int, ctx PostScanContext) bool {
|
|||
|
|
return effectiveFit >= minPostProductFitPatrol &&
|
|||
|
|
HasPainPointDemand(text) &&
|
|||
|
|
ProductSolvesPostPain(text, ctx) &&
|
|||
|
|
!HasCategoryConflict(text, ctx) &&
|
|||
|
|
!HasTangentialTopicMismatch(text, ctx)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// PassesRelaxedFinalFilters keeps demand + basic fit when strict pass yields zero results.
|
|||
|
|
func PassesRelaxedFinalFilters(text string, effectiveFit int, ctx PostScanContext) bool {
|
|||
|
|
text = strings.TrimSpace(text)
|
|||
|
|
if text == "" {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if MatchesExclusion(text, ctx.Exclusions) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if HasCategoryConflict(text, ctx) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if HasTangentialTopicMismatch(text, ctx) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if !HasPainPointDemand(text) && !HasPlacementIntent(text) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
return effectiveFit >= minPostProductFitRelaxedFinal
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// PassesIngestFallbackFilters keeps basic searchable posts when strict/relaxed gates yield zero.
|
|||
|
|
func PassesIngestFallbackFilters(text string, effectiveFit int, ctx PostScanContext) bool {
|
|||
|
|
text = strings.TrimSpace(text)
|
|||
|
|
if text == "" {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if MatchesExclusion(text, ctx.Exclusions) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if HasCategoryConflict(text, ctx) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
if !HasPlacementIntent(text) && !HasPainPointDemand(text) {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
return effectiveFit >= minPostProductFitIngestFinal
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func matchesTopicAnchor(text string, ctx PostScanContext) bool {
|
|||
|
|
text = strings.ToLower(text)
|
|||
|
|
hits := 0
|
|||
|
|
for _, pillar := range ctx.Pillars {
|
|||
|
|
if tokenHits(text, pillar) > 0 {
|
|||
|
|
hits++
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for _, q := range ctx.Questions {
|
|||
|
|
if tokenHits(text, q) >= 2 {
|
|||
|
|
hits++
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if hits >= minPostPillarAnchor {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
for _, tag := range ctx.MatchTags {
|
|||
|
|
if tokenHits(text, tag) > 0 {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if ctx.ProductName != "" && tokenHits(text, ctx.ProductName) > 0 {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
for _, phrase := range []string{ctx.AudienceSummary, ctx.TargetAudience} {
|
|||
|
|
if tokenHits(text, phrase) >= 2 {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func PassesPostAge(postedAt string, dimension QueryDimension, recencyDays int) bool {
|
|||
|
|
postedAt = strings.TrimSpace(postedAt)
|
|||
|
|
if postedAt == "" {
|
|||
|
|
// Web Search snippets often lack dates; do not drop at ingest.
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
maxDays := patrolMaxPostAgeDays
|
|||
|
|
switch dimension {
|
|||
|
|
case QueryRelevance:
|
|||
|
|
maxDays = relevanceMaxPostAgeDays
|
|||
|
|
case QueryRecency:
|
|||
|
|
if recencyDays > 0 {
|
|||
|
|
maxDays = recencyDays
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return IsPostWithinMaxAge(postedAt, maxDays, time.Now())
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func IsPostWithinMaxAge(postedAt string, maxDays int, now time.Time) bool {
|
|||
|
|
if maxDays <= 0 {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
ts, ok := ParsePostedAt(postedAt)
|
|||
|
|
if !ok {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
if now.IsZero() {
|
|||
|
|
now = time.Now()
|
|||
|
|
}
|
|||
|
|
cutoff := now.AddDate(0, 0, -maxDays)
|
|||
|
|
return !ts.Before(cutoff)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func ParsePostedAt(raw string) (time.Time, bool) {
|
|||
|
|
raw = strings.TrimSpace(raw)
|
|||
|
|
if raw == "" {
|
|||
|
|
return time.Time{}, false
|
|||
|
|
}
|
|||
|
|
layouts := []string{
|
|||
|
|
time.RFC3339,
|
|||
|
|
"2006-01-02T15:04:05Z07:00",
|
|||
|
|
"2006-01-02 15:04:05",
|
|||
|
|
"2006-01-02",
|
|||
|
|
"2006/01/02",
|
|||
|
|
}
|
|||
|
|
for _, layout := range layouts {
|
|||
|
|
if ts, err := time.Parse(layout, raw); err == nil {
|
|||
|
|
return ts, true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return time.Time{}, false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func ExpandExclusionTokens(exclusions []string) []string {
|
|||
|
|
seen := map[string]struct{}{}
|
|||
|
|
out := []string{}
|
|||
|
|
add := func(token string) {
|
|||
|
|
token = strings.TrimSpace(strings.ToLower(token))
|
|||
|
|
if len([]rune(token)) < 2 {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
if _, ok := seen[token]; ok {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
seen[token] = struct{}{}
|
|||
|
|
out = append(out, token)
|
|||
|
|
}
|
|||
|
|
for _, rule := range exclusions {
|
|||
|
|
rule = strings.TrimSpace(rule)
|
|||
|
|
if rule == "" {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
add(rule)
|
|||
|
|
for _, token := range tokenizePhrase(rule) {
|
|||
|
|
add(token)
|
|||
|
|
}
|
|||
|
|
for _, synonym := range exclusionSynonyms(rule) {
|
|||
|
|
add(synonym)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func exclusionSynonyms(rule string) []string {
|
|||
|
|
rule = strings.ToLower(rule)
|
|||
|
|
out := []string{}
|
|||
|
|
for _, group := range exclusionCategoryGroups {
|
|||
|
|
if strings.Contains(rule, group.key) {
|
|||
|
|
out = append(out, group.tokens...)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var exclusionCategoryGroups = []struct {
|
|||
|
|
key string
|
|||
|
|
tokens []string
|
|||
|
|
}{
|
|||
|
|
{key: "洗碗", tokens: []string{"洗碗", "洗碗精", "洗碗機", "碗盤", "廚房清潔", "油污"}},
|
|||
|
|
{key: "洗衣", tokens: []string{"洗衣", "洗衣精", "洗衣粉", "衣物", "衣服清洗", "污漬"}},
|
|||
|
|
{key: "沐浴", tokens: []string{"沐浴", "沐浴乳", "沐浴露", "洗澡"}},
|
|||
|
|
{key: "洗髮", tokens: []string{"洗髮", "洗髮精", "頭皮", "髮品"}},
|
|||
|
|
{key: "寵物", tokens: []string{"寵物", "貓砂", "狗糧", "貓咪", "狗狗"}},
|
|||
|
|
{key: "廚房", tokens: []string{"廚房", "抽油煙機", "鍋具"}},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var productCategoryGroups = []struct {
|
|||
|
|
key string
|
|||
|
|
tokens []string
|
|||
|
|
}{
|
|||
|
|
{key: "洗衣", tokens: []string{"洗衣", "洗衣精", "洗衣粉", "衣物", "衣服", "污漬", "漂白"}},
|
|||
|
|
{key: "洗碗", tokens: []string{"洗碗", "洗碗精", "洗碗機", "碗盤", "廚房", "油污"}},
|
|||
|
|
{key: "沐浴", tokens: []string{"沐浴", "沐浴乳", "沐浴露", "洗澡", "沖澡"}},
|
|||
|
|
{key: "洗髮", tokens: []string{"洗髮", "洗髮精", "洗髮乳", "頭皮", "髮絲"}},
|
|||
|
|
{key: "清潔", tokens: []string{"清潔", "打掃", "去污"}},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func productCategories(ctx PostScanContext) []string {
|
|||
|
|
blob := strings.ToLower(ctx.ProductName + " " + ctx.ProductFeatures + " " + strings.Join(ctx.MatchTags, " "))
|
|||
|
|
found := []string{}
|
|||
|
|
for _, group := range productCategoryGroups {
|
|||
|
|
for _, token := range group.tokens {
|
|||
|
|
if strings.Contains(blob, token) {
|
|||
|
|
found = append(found, group.key)
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return found
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func postCategories(text string) []string {
|
|||
|
|
text = strings.ToLower(text)
|
|||
|
|
found := []string{}
|
|||
|
|
for _, group := range productCategoryGroups {
|
|||
|
|
for _, token := range group.tokens {
|
|||
|
|
if strings.Contains(text, token) {
|
|||
|
|
found = append(found, group.key)
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return found
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var genericProductCategories = map[string]struct{}{
|
|||
|
|
"清潔": {},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func HasCategoryConflict(text string, ctx PostScanContext) bool {
|
|||
|
|
productCats := productCategories(ctx)
|
|||
|
|
if len(productCats) == 0 {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
postCats := postCategories(text)
|
|||
|
|
if len(postCats) == 0 {
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
productSet := map[string]struct{}{}
|
|||
|
|
for _, c := range productCats {
|
|||
|
|
productSet[c] = struct{}{}
|
|||
|
|
}
|
|||
|
|
for _, c := range postCats {
|
|||
|
|
if _, ok := genericProductCategories[c]; ok {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if _, ok := productSet[c]; !ok {
|
|||
|
|
return true
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func scoreTextAgainstPhrase(text, phrase string) int {
|
|||
|
|
text = strings.ToLower(text)
|
|||
|
|
phrase = strings.TrimSpace(strings.ToLower(phrase))
|
|||
|
|
if text == "" || phrase == "" {
|
|||
|
|
return 0
|
|||
|
|
}
|
|||
|
|
if strings.Contains(text, phrase) {
|
|||
|
|
return 85
|
|||
|
|
}
|
|||
|
|
best := 0
|
|||
|
|
for _, token := range tokenizePhrase(phrase) {
|
|||
|
|
if len([]rune(token)) < 2 {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if strings.Contains(text, token) {
|
|||
|
|
best = maxInt(best, 55)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return best
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func tokenHits(text, phrase string) int {
|
|||
|
|
text = strings.ToLower(text)
|
|||
|
|
hits := 0
|
|||
|
|
for _, token := range tokenizePhrase(phrase) {
|
|||
|
|
if len([]rune(token)) < 2 {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if strings.Contains(text, token) {
|
|||
|
|
hits++
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return hits
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func tokenizePhrase(phrase string) []string {
|
|||
|
|
phrase = strings.ToLower(strings.TrimSpace(phrase))
|
|||
|
|
if phrase == "" {
|
|||
|
|
return nil
|
|||
|
|
}
|
|||
|
|
repl := strings.NewReplacer("，", " ", "、", " ", "。", " ", "；", " ", "/", " ", "｜", " ", "|", " ", "（", " ", "）", " ", "(", " ", ")", " ")
|
|||
|
|
phrase = repl.Replace(phrase)
|
|||
|
|
var tokens []string
|
|||
|
|
for _, part := range strings.Fields(phrase) {
|
|||
|
|
part = strings.Trim(part, `"'「」『』：:`)
|
|||
|
|
if part != "" {
|
|||
|
|
tokens = append(tokens, part)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if len(tokens) == 0 {
|
|||
|
|
return splitRunes(phrase)
|
|||
|
|
}
|
|||
|
|
return tokens
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func splitRunes(s string) []string {
|
|||
|
|
var out []string
|
|||
|
|
var buf []rune
|
|||
|
|
flush := func() {
|
|||
|
|
if len(buf) >= 2 {
|
|||
|
|
out = append(out, string(buf))
|
|||
|
|
}
|
|||
|
|
buf = buf[:0]
|
|||
|
|
}
|
|||
|
|
for _, r := range s {
|
|||
|
|
if unicode.Is(unicode.Han, r) {
|
|||
|
|
buf = append(buf, r)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
flush()
|
|||
|
|
}
|
|||
|
|
flush()
|
|||
|
|
return out
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func minInt(a, b int) int {
|
|||
|
|
if a < b {
|
|||
|
|
return a
|
|||
|
|
}
|
|||
|
|
return b
|
|||
|
|
}
|