122 lines
3.0 KiB
Go
122 lines
3.0 KiB
Go
|
package viral
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"encoding/xml"
|
||
|
|
"html"
|
||
|
|
"net/http"
|
||
|
|
"net/url"
|
||
|
|
"regexp"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
var rssTagRE = regexp.MustCompile(`<[^>]+>`)
|
||
|
|
|
||
|
|
type freeRSS struct {
|
||
|
|
Channel struct {
|
||
|
|
Items []freeRSSItem `xml:"item"`
|
||
|
|
} `xml:"channel"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type freeRSSItem struct {
|
||
|
|
Title string `xml:"title"`
|
||
|
|
Description string `xml:"description"`
|
||
|
|
Link string `xml:"link"`
|
||
|
|
}
|
||
|
|
|
||
|
|
func CollectFreeTopicTrends(ctx context.Context, keyword, personaBrief string) []MissionInspireTrendSnippet {
|
||
|
|
queries := freeTrendFeedQueries(keyword, personaBrief)
|
||
|
|
client := &http.Client{Timeout: 7 * time.Second}
|
||
|
|
out := make([]MissionInspireTrendSnippet, 0, 12)
|
||
|
|
seen := map[string]struct{}{}
|
||
|
|
|
||
|
|
add := func(query, title, snippet, link string) {
|
||
|
|
title = cleanRSSField(title)
|
||
|
|
snippet = cleanRSSField(snippet)
|
||
|
|
link = strings.TrimSpace(link)
|
||
|
|
if title == "" && snippet == "" {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
key := strings.ToLower(title + "|" + snippet)
|
||
|
|
if _, ok := seen[key]; ok {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
seen[key] = struct{}{}
|
||
|
|
out = append(out, MissionInspireTrendSnippet{Query: query, Title: title, Snippet: snippet, URL: link})
|
||
|
|
}
|
||
|
|
|
||
|
|
for _, feed := range queries {
|
||
|
|
if len(out) >= 12 {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, feed.url, nil)
|
||
|
|
if err != nil {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
req.Header.Set("User-Agent", "HaixunTopicBot/1.0")
|
||
|
|
resp, err := client.Do(req)
|
||
|
|
if err != nil || resp == nil {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
func() {
|
||
|
|
defer resp.Body.Close()
|
||
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
var parsed freeRSS
|
||
|
|
if err := xml.NewDecoder(resp.Body).Decode(&parsed); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
for _, item := range parsed.Channel.Items {
|
||
|
|
if len(out) >= 12 {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
add(feed.label, item.Title, item.Description, item.Link)
|
||
|
|
}
|
||
|
|
}()
|
||
|
|
}
|
||
|
|
return out
|
||
|
|
}
|
||
|
|
|
||
|
|
type freeTrendFeed struct {
|
||
|
|
label string
|
||
|
|
url string
|
||
|
|
}
|
||
|
|
|
||
|
|
func freeTrendFeedQueries(keyword, personaBrief string) []freeTrendFeed {
|
||
|
|
feeds := []freeTrendFeed{
|
||
|
|
{label: "Google Trends 台灣每日趨勢", url: "https://trends.google.com/trends/trendingsearches/daily/rss?geo=TW"},
|
||
|
|
{label: "Google Trends 台灣即時趨勢", url: "https://trends.google.com/trends/trendingsearches/realtime/rss?geo=TW&category=all"},
|
||
|
|
}
|
||
|
|
terms := []string{strings.TrimSpace(keyword)}
|
||
|
|
brief := strings.TrimSpace(personaBrief)
|
||
|
|
if brief != "" {
|
||
|
|
if len([]rune(brief)) > 18 {
|
||
|
|
brief = string([]rune(brief)[:18])
|
||
|
|
}
|
||
|
|
terms = append(terms, brief)
|
||
|
|
}
|
||
|
|
for _, term := range terms {
|
||
|
|
if term == "" {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
q := url.QueryEscape(term + " 共鳴 OR 痛點 OR 趨勢")
|
||
|
|
feeds = append(feeds, freeTrendFeed{
|
||
|
|
label: "Google News 搜尋：" + term,
|
||
|
|
url: "https://news.google.com/rss/search?q=" + q + "&hl=zh-TW&gl=TW&ceid=TW:zh-Hant",
|
||
|
|
})
|
||
|
|
}
|
||
|
|
return feeds
|
||
|
|
}
|
||
|
|
|
||
|
|
func cleanRSSField(raw string) string {
|
||
|
|
raw = html.UnescapeString(raw)
|
||
|
|
raw = rssTagRE.ReplaceAllString(raw, " ")
|
||
|
|
raw = strings.Join(strings.Fields(raw), " ")
|
||
|
|
if len([]rune(raw)) > 220 {
|
||
|
|
return string([]rune(raw)[:220]) + "…"
|
||
|
|
}
|
||
|
|
return raw
|
||
|
|
}
|