thread-master/backend/internal/library/webpage/digest.go

package webpage

import (
	"context"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/go-shiori/go-readability"
)

const (
	DefaultMaxPages     = 6
	DefaultMaxMarkdown  = 6000
	DefaultPerPageChars = 2500
	DefaultFetchTimeout = 18 * time.Second
	DefaultMaxBodyBytes = 2 << 20
)

type FetchOptions struct {
	MaxPages     int
	MaxMarkdown  int
	PerPageChars int
	Timeout      time.Duration
	UserAgent    string
}

func (o FetchOptions) normalized() FetchOptions {
	out := o
	if out.MaxPages <= 0 {
		out.MaxPages = DefaultMaxPages
	}
	if out.MaxMarkdown <= 0 {
		out.MaxMarkdown = DefaultMaxMarkdown
	}
	if out.PerPageChars <= 0 {
		out.PerPageChars = DefaultPerPageChars
	}
	if out.Timeout <= 0 {
		out.Timeout = DefaultFetchTimeout
	}
	if strings.TrimSpace(out.UserAgent) == "" {
		out.UserAgent = "ThreadMasterKnowledgeBot/1.0 (+https://thread-master.local)"
	}
	return out
}

type Digest struct {
	URL      string
	Title    string
	Markdown string
	Error    string
}

// FetchDigests fetches readable page content and formats each page as lightweight markdown.
func FetchDigests(ctx context.Context, urls []string, opts FetchOptions) []Digest {
	opts = opts.normalized()
	seen := map[string]struct{}{}
	out := make([]Digest, 0, min(len(urls), opts.MaxPages))
	for _, raw := range urls {
		if len(out) >= opts.MaxPages {
			break
		}
		u := strings.TrimSpace(raw)
		if u == "" {
			continue
		}
		key := strings.ToLower(u)
		if _, ok := seen[key]; ok {
			continue
		}
		seen[key] = struct{}{}
		if !FetchableURL(u) {
			continue
		}
		out = append(out, fetchOne(ctx, u, opts))
	}
	return out
}

func fetchOne(ctx context.Context, pageURL string, opts FetchOptions) Digest {
	d := Digest{URL: pageURL}
	reqCtx, cancel := context.WithTimeout(ctx, opts.Timeout)
	defer cancel()

	req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, pageURL, nil)
	if err != nil {
		d.Error = err.Error()
		return d
	}
	req.Header.Set("User-Agent", opts.UserAgent)
	req.Header.Set("Accept", "text/html,application/xhtml+xml")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		d.Error = err.Error()
		return d
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		d.Error = fmt.Sprintf("http %d", resp.StatusCode)
		return d
	}

	body, err := io.ReadAll(io.LimitReader(resp.Body, DefaultMaxBodyBytes))
	if err != nil {
		d.Error = err.Error()
		return d
	}
	parsed, err := url.Parse(pageURL)
	if err != nil {
		d.Error = err.Error()
		return d
	}
	article, err := readability.FromReader(strings.NewReader(string(body)), parsed)
	if err != nil {
		d.Error = err.Error()
		return d
	}
	title := strings.TrimSpace(article.Title)
	text := strings.TrimSpace(article.TextContent)
	if title == "" && text == "" {
		d.Error = "empty article"
		return d
	}
	d.Title = title
	d.Markdown = toMarkdown(title, text, opts.PerPageChars)
	return d
}

func toMarkdown(title, text string, maxChars int) string {
	var b strings.Builder
	if title != "" {
		b.WriteString("# ")
		b.WriteString(title)
		b.WriteString("\n\n")
	}
	if text != "" {
		b.WriteString(text)
	}
	out := strings.TrimSpace(b.String())
	if maxChars > 0 && len([]rune(out)) > maxChars {
		runes := []rune(out)
		out = strings.TrimSpace(string(runes[:maxChars])) + "…"
	}
	return out
}

// FetchableURL returns false for unsupported or crawler-owned hosts.
func FetchableURL(raw string) bool {
	u, err := url.Parse(strings.TrimSpace(raw))
	if err != nil || u.Scheme == "" || u.Host == "" {
		return false
	}
	switch strings.ToLower(u.Scheme) {
	case "http", "https":
	default:
		return false
	}
	host := strings.ToLower(u.Hostname())
	if host == "localhost" || strings.HasSuffix(host, ".local") {
		return false
	}
	if strings.Contains(host, "threads.net") || strings.Contains(host, "threads.com") {
		return false
	}
	path := strings.ToLower(u.Path)
	if strings.HasSuffix(path, ".pdf") || strings.HasSuffix(path, ".zip") {
		return false
	}
	return true
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}