package webpage import ( "context" "fmt" "io" "net/http" "net/url" "strings" "time" "github.com/go-shiori/go-readability" ) const ( DefaultMaxPages = 6 DefaultMaxMarkdown = 6000 DefaultPerPageChars = 2500 DefaultFetchTimeout = 18 * time.Second DefaultMaxBodyBytes = 2 << 20 ) type FetchOptions struct { MaxPages int MaxMarkdown int PerPageChars int Timeout time.Duration UserAgent string } func (o FetchOptions) normalized() FetchOptions { out := o if out.MaxPages <= 0 { out.MaxPages = DefaultMaxPages } if out.MaxMarkdown <= 0 { out.MaxMarkdown = DefaultMaxMarkdown } if out.PerPageChars <= 0 { out.PerPageChars = DefaultPerPageChars } if out.Timeout <= 0 { out.Timeout = DefaultFetchTimeout } if strings.TrimSpace(out.UserAgent) == "" { out.UserAgent = "ThreadMasterKnowledgeBot/1.0 (+https://thread-master.local)" } return out } type Digest struct { URL string Title string Markdown string Error string } // FetchDigests fetches readable page content and formats each page as lightweight markdown. func FetchDigests(ctx context.Context, urls []string, opts FetchOptions) []Digest { opts = opts.normalized() seen := map[string]struct{}{} out := make([]Digest, 0, min(len(urls), opts.MaxPages)) for _, raw := range urls { if len(out) >= opts.MaxPages { break } u := strings.TrimSpace(raw) if u == "" { continue } key := strings.ToLower(u) if _, ok := seen[key]; ok { continue } seen[key] = struct{}{} if !FetchableURL(u) { continue } out = append(out, fetchOne(ctx, u, opts)) } return out } func fetchOne(ctx context.Context, pageURL string, opts FetchOptions) Digest { d := Digest{URL: pageURL} reqCtx, cancel := context.WithTimeout(ctx, opts.Timeout) defer cancel() req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, pageURL, nil) if err != nil { d.Error = err.Error() return d } req.Header.Set("User-Agent", opts.UserAgent) req.Header.Set("Accept", "text/html,application/xhtml+xml") resp, err := http.DefaultClient.Do(req) if err != nil { d.Error = err.Error() return d } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { d.Error = fmt.Sprintf("http %d", resp.StatusCode) return d } body, err := io.ReadAll(io.LimitReader(resp.Body, DefaultMaxBodyBytes)) if err != nil { d.Error = err.Error() return d } parsed, err := url.Parse(pageURL) if err != nil { d.Error = err.Error() return d } article, err := readability.FromReader(strings.NewReader(string(body)), parsed) if err != nil { d.Error = err.Error() return d } title := strings.TrimSpace(article.Title) text := strings.TrimSpace(article.TextContent) if title == "" && text == "" { d.Error = "empty article" return d } d.Title = title d.Markdown = toMarkdown(title, text, opts.PerPageChars) return d } func toMarkdown(title, text string, maxChars int) string { var b strings.Builder if title != "" { b.WriteString("# ") b.WriteString(title) b.WriteString("\n\n") } if text != "" { b.WriteString(text) } out := strings.TrimSpace(b.String()) if maxChars > 0 && len([]rune(out)) > maxChars { runes := []rune(out) out = strings.TrimSpace(string(runes[:maxChars])) + "…" } return out } // FetchableURL returns false for unsupported or crawler-owned hosts. func FetchableURL(raw string) bool { u, err := url.Parse(strings.TrimSpace(raw)) if err != nil || u.Scheme == "" || u.Host == "" { return false } switch strings.ToLower(u.Scheme) { case "http", "https": default: return false } host := strings.ToLower(u.Hostname()) if host == "localhost" || strings.HasSuffix(host, ".local") { return false } if strings.Contains(host, "threads.net") || strings.Contains(host, "threads.com") { return false } path := strings.ToLower(u.Path) if strings.HasSuffix(path, ".pdf") || strings.HasSuffix(path, ".zip") { return false } return true } func min(a, b int) int { if a < b { return a } return b }