380 lines
13 KiB
Python
380 lines
13 KiB
Python
|
#!/usr/bin/env python3
|
|||
|
|
"""
|
|||
|
|
web_to_md.py - 將網頁轉換為 Markdown
|
|||
|
|
|
|||
|
|
使用方式：
|
|||
|
|
# 單頁模式
|
|||
|
|
python web_to_md.py --url https://example.com/features --output output.md
|
|||
|
|
|
|||
|
|
# 爬蟲模式（追蹤同域連結）
|
|||
|
|
python web_to_md.py --url https://example.com --crawl --depth 2 --output-dir ./scraped/
|
|||
|
|
|
|||
|
|
# 爬蟲模式（只爬特定路徑前綴）
|
|||
|
|
python web_to_md.py --url https://example.com/docs --crawl --depth 3 --same-path --output-dir ./docs/
|
|||
|
|
"""
|
|||
|
|
import argparse
|
|||
|
|
import os
|
|||
|
|
import re
|
|||
|
|
import sys
|
|||
|
|
import time
|
|||
|
|
from collections import deque
|
|||
|
|
from datetime import datetime
|
|||
|
|
from urllib.parse import urljoin, urlparse
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
import requests
|
|||
|
|
from bs4 import BeautifulSoup
|
|||
|
|
import html2text
|
|||
|
|
except ImportError:
|
|||
|
|
print("缺少必要套件，請執行：pip install requests beautifulsoup4 html2text lxml")
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ── 設定 ──────────────────────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
DEFAULT_HEADERS = {
|
|||
|
|
"User-Agent": (
|
|||
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
|||
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|||
|
|
"Chrome/120.0.0.0 Safari/537.36"
|
|||
|
|
),
|
|||
|
|
"Accept-Language": "zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# 自動排除的 URL 路徑關鍵字（不爬這些）
|
|||
|
|
DEFAULT_EXCLUDE_PATTERNS = [
|
|||
|
|
"/login", "/logout", "/signup", "/register",
|
|||
|
|
"/cdn-cgi/", "/__", "/static/", "/assets/",
|
|||
|
|
".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg",
|
|||
|
|
".pdf", ".zip", ".tar", ".gz",
|
|||
|
|
".css", ".js", ".woff", ".woff2",
|
|||
|
|
"/api/", "/feed/", "/rss", "/atom",
|
|||
|
|
"#", # 錨點連結
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
# 雜訊元素（移除這些以得到乾淨內文）
|
|||
|
|
NOISE_TAGS = [
|
|||
|
|
"nav", "header", "footer", "aside",
|
|||
|
|
"script", "style", "noscript", "iframe",
|
|||
|
|
"[class*='cookie']", "[class*='banner']", "[class*='popup']",
|
|||
|
|
"[class*='modal']", "[class*='overlay']", "[id*='cookie']",
|
|||
|
|
"[class*='nav']", "[class*='footer']", "[class*='header']",
|
|||
|
|
"[class*='sidebar']", "[class*='advertisement']", "[class*='ad-']",
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ── URL 工具 ──────────────────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
def normalize_url(url: str) -> str:
|
|||
|
|
"""移除 fragment，標準化 URL"""
|
|||
|
|
parsed = urlparse(url)
|
|||
|
|
return parsed._replace(fragment="").geturl().rstrip("/")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def is_same_domain(url: str, base_url: str) -> bool:
|
|||
|
|
return urlparse(url).netloc == urlparse(base_url).netloc
|
|||
|
|
|
|||
|
|
|
|||
|
|
def is_same_path_prefix(url: str, base_url: str) -> bool:
|
|||
|
|
base_path = urlparse(base_url).path
|
|||
|
|
url_path = urlparse(url).path
|
|||
|
|
return url_path.startswith(base_path)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def should_skip(url: str, exclude_patterns: list[str]) -> bool:
|
|||
|
|
url_lower = url.lower()
|
|||
|
|
return any(p in url_lower for p in exclude_patterns)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def url_to_filename(url: str) -> str:
|
|||
|
|
"""將 URL 轉成合適的檔名"""
|
|||
|
|
parsed = urlparse(url)
|
|||
|
|
path = parsed.path.strip("/") or "index"
|
|||
|
|
path = re.sub(r"[^\w\-/]", "-", path)
|
|||
|
|
path = path.replace("/", "__")
|
|||
|
|
if parsed.query:
|
|||
|
|
query = re.sub(r"[^\w\-]", "-", parsed.query)[:50]
|
|||
|
|
path = f"{path}_{query}"
|
|||
|
|
return path[:100] + ".md"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ── HTML → Markdown ───────────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
def clean_html(html_content: str, base_url: str) -> BeautifulSoup:
|
|||
|
|
"""移除雜訊元素，保留主要內容"""
|
|||
|
|
soup = BeautifulSoup(html_content, "lxml")
|
|||
|
|
|
|||
|
|
# 嘗試找到主要內容區域
|
|||
|
|
main_content = (
|
|||
|
|
soup.find("main") or
|
|||
|
|
soup.find(attrs={"role": "main"}) or
|
|||
|
|
soup.find("article") or
|
|||
|
|
soup.find(id=re.compile(r"(content|main|body)", re.I)) or
|
|||
|
|
soup.find(class_=re.compile(r"(content|main|body|post)", re.I)) or
|
|||
|
|
soup.body or
|
|||
|
|
soup
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# 移除雜訊
|
|||
|
|
for tag in NOISE_TAGS:
|
|||
|
|
if tag.startswith("["):
|
|||
|
|
# CSS 選擇器形式
|
|||
|
|
try:
|
|||
|
|
for el in main_content.select(tag):
|
|||
|
|
el.decompose()
|
|||
|
|
except Exception:
|
|||
|
|
pass
|
|||
|
|
else:
|
|||
|
|
for el in main_content.find_all(tag):
|
|||
|
|
el.decompose()
|
|||
|
|
|
|||
|
|
return main_content
|
|||
|
|
|
|||
|
|
|
|||
|
|
def html_to_markdown(html_content: str, url: str, title: str = "") -> str:
|
|||
|
|
"""將 HTML 轉換為乾淨的 Markdown"""
|
|||
|
|
cleaned = clean_html(html_content, url)
|
|||
|
|
|
|||
|
|
# 設定 html2text
|
|||
|
|
converter = html2text.HTML2Text()
|
|||
|
|
converter.ignore_links = False
|
|||
|
|
converter.ignore_images = False
|
|||
|
|
converter.ignore_tables = False
|
|||
|
|
converter.body_width = 0 # 不自動換行
|
|||
|
|
converter.protect_links = True
|
|||
|
|
converter.wrap_links = False
|
|||
|
|
converter.mark_code = True
|
|||
|
|
converter.ul_item_mark = "-"
|
|||
|
|
converter.emphasis_mark = "*"
|
|||
|
|
converter.strong_mark = "**"
|
|||
|
|
converter.baseurl = url
|
|||
|
|
|
|||
|
|
md = converter.handle(str(cleaned))
|
|||
|
|
|
|||
|
|
# 基本清理
|
|||
|
|
md = re.sub(r"\n{3,}", "\n\n", md) # 移除多餘空行
|
|||
|
|
md = re.sub(r" +\n", "\n", md) # 移除行尾空格
|
|||
|
|
md = md.strip()
|
|||
|
|
|
|||
|
|
# 加上 Frontmatter
|
|||
|
|
crawled_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|||
|
|
frontmatter = f"""---
|
|||
|
|
title: {title or '（無標題）'}
|
|||
|
|
url: {url}
|
|||
|
|
crawled_at: {crawled_at}
|
|||
|
|
---
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
return frontmatter + md
|
|||
|
|
|
|||
|
|
|
|||
|
|
def fetch_page(url: str, session: requests.Session, timeout: int = 15) -> tuple[str, str]:
|
|||
|
|
"""
|
|||
|
|
抓取頁面，回傳 (html_content, page_title)
|
|||
|
|
失敗時回傳 ("", "")
|
|||
|
|
"""
|
|||
|
|
try:
|
|||
|
|
resp = session.get(url, timeout=timeout, allow_redirects=True)
|
|||
|
|
resp.raise_for_status()
|
|||
|
|
|
|||
|
|
content_type = resp.headers.get("Content-Type", "")
|
|||
|
|
if "html" not in content_type:
|
|||
|
|
return "", ""
|
|||
|
|
|
|||
|
|
resp.encoding = resp.apparent_encoding or "utf-8"
|
|||
|
|
html = resp.text
|
|||
|
|
|
|||
|
|
# 取得標題
|
|||
|
|
soup = BeautifulSoup(html, "lxml")
|
|||
|
|
title = ""
|
|||
|
|
if soup.title:
|
|||
|
|
title = soup.title.string or ""
|
|||
|
|
if not title and soup.find("h1"):
|
|||
|
|
title = soup.find("h1").get_text(strip=True)
|
|||
|
|
|
|||
|
|
return html, title.strip()
|
|||
|
|
|
|||
|
|
except requests.exceptions.RequestException as e:
|
|||
|
|
print(f" ⚠️ 抓取失敗 {url}: {e}", file=sys.stderr)
|
|||
|
|
return "", ""
|
|||
|
|
|
|||
|
|
|
|||
|
|
def extract_links(html: str, base_url: str) -> list[str]:
|
|||
|
|
"""從 HTML 中提取所有連結"""
|
|||
|
|
soup = BeautifulSoup(html, "lxml")
|
|||
|
|
links = []
|
|||
|
|
for tag in soup.find_all("a", href=True):
|
|||
|
|
href = tag["href"].strip()
|
|||
|
|
if not href or href.startswith("javascript:") or href.startswith("mailto:"):
|
|||
|
|
continue
|
|||
|
|
full_url = urljoin(base_url, href)
|
|||
|
|
links.append(normalize_url(full_url))
|
|||
|
|
return links
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ── 主要功能 ──────────────────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
def convert_single(url: str, output_path: str, session: requests.Session) -> bool:
|
|||
|
|
"""單頁模式：轉換一個 URL"""
|
|||
|
|
print(f"📄 抓取：{url}")
|
|||
|
|
html, title = fetch_page(url, session)
|
|||
|
|
if not html:
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
md = html_to_markdown(html, url, title)
|
|||
|
|
|
|||
|
|
os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
|
|||
|
|
with open(output_path, "w", encoding="utf-8") as f:
|
|||
|
|
f.write(md)
|
|||
|
|
|
|||
|
|
lines = md.count("\n")
|
|||
|
|
print(f" ✅ 已儲存：{output_path}（約 {lines} 行）")
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
|
|||
|
|
def crawl(
|
|||
|
|
start_url: str,
|
|||
|
|
output_dir: str,
|
|||
|
|
max_depth: int = 2,
|
|||
|
|
same_path: bool = False,
|
|||
|
|
max_pages: int = 50,
|
|||
|
|
delay: float = 1.0,
|
|||
|
|
exclude_patterns: list[str] = None,
|
|||
|
|
session: requests.Session = None,
|
|||
|
|
) -> dict:
|
|||
|
|
"""
|
|||
|
|
爬蟲模式：從 start_url 出發，追蹤同域連結
|
|||
|
|
回傳 {url: output_file_path} 的 mapping
|
|||
|
|
"""
|
|||
|
|
if exclude_patterns is None:
|
|||
|
|
exclude_patterns = DEFAULT_EXCLUDE_PATTERNS
|
|||
|
|
|
|||
|
|
queue = deque([(normalize_url(start_url), 0)]) # (url, depth)
|
|||
|
|
visited = set()
|
|||
|
|
results = {}
|
|||
|
|
|
|||
|
|
os.makedirs(output_dir, exist_ok=True)
|
|||
|
|
log_path = os.path.join(output_dir, "_crawl_log.md")
|
|||
|
|
|
|||
|
|
print(f"\n🕷️ 開始爬取：{start_url}")
|
|||
|
|
print(f" 深度上限：{max_depth}，頁面上限：{max_pages}，延遲：{delay}s")
|
|||
|
|
if same_path:
|
|||
|
|
print(f" 路徑限制：只爬 {urlparse(start_url).path} 底下的頁面")
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
with open(log_path, "w", encoding="utf-8") as log:
|
|||
|
|
log.write(f"# 爬蟲記錄\n\n")
|
|||
|
|
log.write(f"- **起始 URL**：{start_url}\n")
|
|||
|
|
log.write(f"- **執行時間**：{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
|
|||
|
|
log.write("## 已爬取頁面\n\n")
|
|||
|
|
|
|||
|
|
while queue and len(results) < max_pages:
|
|||
|
|
url, depth = queue.popleft()
|
|||
|
|
|
|||
|
|
if url in visited:
|
|||
|
|
continue
|
|||
|
|
visited.add(url)
|
|||
|
|
|
|||
|
|
if should_skip(url, exclude_patterns):
|
|||
|
|
print(f" ⏭️ 跳過（排除規則）：{url}")
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
if not is_same_domain(url, start_url):
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
if same_path and not is_same_path_prefix(url, start_url):
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# 抓取
|
|||
|
|
print(f" {' ' * depth}📄 [深度{depth}] {url}")
|
|||
|
|
html, title = fetch_page(url, session)
|
|||
|
|
|
|||
|
|
if not html:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# 轉換並儲存
|
|||
|
|
filename = url_to_filename(url)
|
|||
|
|
output_path = os.path.join(output_dir, filename)
|
|||
|
|
md = html_to_markdown(html, url, title)
|
|||
|
|
|
|||
|
|
with open(output_path, "w", encoding="utf-8") as f:
|
|||
|
|
f.write(md)
|
|||
|
|
|
|||
|
|
results[url] = output_path
|
|||
|
|
lines = md.count("\n")
|
|||
|
|
print(f" {' ' * depth} ✅ → {filename}（{lines} 行）")
|
|||
|
|
log.write(f"| {url} | [{filename}](./{filename}) | {title} |\n")
|
|||
|
|
|
|||
|
|
# 如果還沒到最大深度，繼續追蹤連結
|
|||
|
|
if depth < max_depth:
|
|||
|
|
links = extract_links(html, url)
|
|||
|
|
for link in links:
|
|||
|
|
if link not in visited and not should_skip(link, exclude_patterns):
|
|||
|
|
queue.append((link, depth + 1))
|
|||
|
|
|
|||
|
|
if queue and delay > 0:
|
|||
|
|
time.sleep(delay)
|
|||
|
|
|
|||
|
|
log.write(f"\n---\n共爬取 **{len(results)}** 頁\n")
|
|||
|
|
|
|||
|
|
print(f"\n✅ 爬取完成：{len(results)} 頁 → {output_dir}")
|
|||
|
|
print(f"📋 爬取記錄：{log_path}")
|
|||
|
|
|
|||
|
|
# 輸出索引
|
|||
|
|
index_path = os.path.join(output_dir, "index.md")
|
|||
|
|
with open(index_path, "w", encoding="utf-8") as f:
|
|||
|
|
f.write(f"# 爬取索引：{start_url}\n\n")
|
|||
|
|
f.write(f"爬取時間：{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
|
|||
|
|
f.write("## 所有頁面\n\n")
|
|||
|
|
for url, path in results.items():
|
|||
|
|
fname = os.path.basename(path)
|
|||
|
|
f.write(f"- [{url}](./{fname})\n")
|
|||
|
|
|
|||
|
|
print(f"📑 索引檔案：{index_path}")
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ── Entry Point ────────────────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
parser = argparse.ArgumentParser(
|
|||
|
|
description="將網頁轉換為 Markdown（單頁或爬蟲模式）"
|
|||
|
|
)
|
|||
|
|
parser.add_argument("--url", required=True, help="起始 URL")
|
|||
|
|
parser.add_argument("--crawl", action="store_true", help="啟用爬蟲模式（追蹤同域連結）")
|
|||
|
|
parser.add_argument("--depth", type=int, default=2, help="爬蟲深度（預設 2）")
|
|||
|
|
parser.add_argument("--same-path", action="store_true", help="只爬相同路徑前綴下的頁面")
|
|||
|
|
parser.add_argument("--max-pages", type=int, default=50, help="最多爬取頁數（預設 50）")
|
|||
|
|
parser.add_argument("--delay", type=float, default=1.0, help="請求間隔秒數（預設 1.0）")
|
|||
|
|
parser.add_argument("--output", default="output.md", help="單頁模式輸出路徑（預設 output.md）")
|
|||
|
|
parser.add_argument("--output-dir", default="./scraped/", help="爬蟲模式輸出目錄（預設 ./scraped/）")
|
|||
|
|
parser.add_argument("--exclude", action="append", default=[], help="排除含有此字串的 URL（可多次使用）")
|
|||
|
|
|
|||
|
|
args = parser.parse_args()
|
|||
|
|
|
|||
|
|
# 組合排除規則
|
|||
|
|
exclude_patterns = DEFAULT_EXCLUDE_PATTERNS + args.exclude
|
|||
|
|
|
|||
|
|
# 建立 Session
|
|||
|
|
session = requests.Session()
|
|||
|
|
session.headers.update(DEFAULT_HEADERS)
|
|||
|
|
|
|||
|
|
if args.crawl:
|
|||
|
|
crawl(
|
|||
|
|
start_url=args.url,
|
|||
|
|
output_dir=args.output_dir,
|
|||
|
|
max_depth=args.depth,
|
|||
|
|
same_path=args.same_path,
|
|||
|
|
max_pages=args.max_pages,
|
|||
|
|
delay=args.delay,
|
|||
|
|
exclude_patterns=exclude_patterns,
|
|||
|
|
session=session,
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
success = convert_single(args.url, args.output, session)
|
|||
|
|
sys.exit(0 if success else 1)
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|