127 lines
4.3 KiB
Python
127 lines
4.3 KiB
Python
"""缠论《教你炒股票》108 课抓取入口。
|
||
|
||
用法
|
||
----
|
||
python main.py # 抓取全部并生成数据（使用本地缓存）
|
||
python main.py --refresh # 忽略缓存重新抓取
|
||
python main.py --limit 5 # 仅抓前 5 篇（联调测试）
|
||
python main.py --no-images # 不下载图片，正文保留远程链接
|
||
|
||
产物
|
||
----
|
||
data/raw/p001.html ... 原始 HTML 归档
|
||
data/markdown/ke-001.md ... 每篇 Markdown 归档
|
||
data/articles.json 全量结构化数据
|
||
D:\\code\\chan\\src\\data\\chanlun.ts Vue 数据源（覆盖示例）
|
||
D:\\code\\chan\\public\\chanlun-img\\... 本地化图片
|
||
"""
|
||
|
||
import argparse
|
||
import json
|
||
import shutil
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
from config import (
|
||
CHAN_DATA_FILE,
|
||
INDEX_URL,
|
||
JSON_PATH,
|
||
MARKDOWN_DIR,
|
||
RAW_DIR,
|
||
article_url,
|
||
)
|
||
from fetcher import fetch_text
|
||
from parser import parse_article, parse_index
|
||
from writer import make_image_resolver, remote_image_resolver, write_ts_file
|
||
|
||
GEN_MARKER = "自动生成"
|
||
|
||
|
||
def cached_html(p_num: int, refresh: bool) -> str:
|
||
"""读取/抓取单篇 HTML，落盘缓存以便重跑。"""
|
||
path = RAW_DIR / f"p{p_num:03d}.html"
|
||
if not refresh and path.exists():
|
||
return path.read_text(encoding="utf-8")
|
||
html = fetch_text(article_url(p_num))
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(html, encoding="utf-8")
|
||
return html
|
||
|
||
|
||
def main() -> None:
|
||
ap = argparse.ArgumentParser(description="抓取缠论《教你炒股票》系列")
|
||
ap.add_argument("--refresh", action="store_true", help="忽略缓存重新抓取")
|
||
ap.add_argument("--limit", type=int, default=0, help="仅抓取前 N 篇（测试）")
|
||
ap.add_argument("--no-images", action="store_true", help="不下载图片，保留远程链接")
|
||
args = ap.parse_args()
|
||
|
||
print("① 抓取目录页 …")
|
||
index_html = fetch_text(INDEX_URL)
|
||
pages = parse_index(index_html)
|
||
if args.limit:
|
||
pages = pages[: args.limit]
|
||
if not pages:
|
||
raise SystemExit("目录页未解析到任何文章链接")
|
||
print(f" 发现 {len(pages)} 篇文章：p={pages[0]}…{pages[-1]}")
|
||
|
||
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
||
MARKDOWN_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
articles: list[dict] = []
|
||
for i, p in enumerate(pages, 1):
|
||
print(f"② [{i}/{len(pages)}] 解析 p={p} …")
|
||
html = cached_html(p, args.refresh)
|
||
resolver = remote_image_resolver if args.no_images else make_image_resolver(p, args.refresh)
|
||
art = parse_article(html, p, resolver)
|
||
(MARKDOWN_DIR / f"{art['slug']}.md").write_text(
|
||
f"# {art['title']}\n\n{art['content']}\n", encoding="utf-8"
|
||
)
|
||
articles.append(art)
|
||
|
||
# 源站少数篇目日期残缺/缺失（如 h5 仅写"2006"），按课次顺序用上一篇日期兜底
|
||
last_date = "2006-06-07"
|
||
fixed = 0
|
||
for a in articles:
|
||
if not a["date"]:
|
||
a["date"] = last_date
|
||
fixed += 1
|
||
else:
|
||
last_date = a["date"]
|
||
if fixed:
|
||
print(f" ℹ {fixed} 篇源站日期缺失，已用相邻课次日期兜底")
|
||
|
||
print("③ 写入归档 JSON …")
|
||
JSON_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||
JSON_PATH.write_text(
|
||
json.dumps(articles, ensure_ascii=False, indent=2), encoding="utf-8"
|
||
)
|
||
|
||
print("④ 生成 Vue 数据文件 …")
|
||
_backup_sample_once(CHAN_DATA_FILE)
|
||
stamp = datetime.now().strftime("%Y-%m-%d %H:%M")
|
||
write_ts_file(articles, CHAN_DATA_FILE, stamp)
|
||
print(f" → {CHAN_DATA_FILE}")
|
||
|
||
print("✅ 完成。")
|
||
print(f" 文章数：{len(articles)}")
|
||
print(" 归档目录：data/")
|
||
print(" Vue 路由：http://localhost:5173/chanlun")
|
||
|
||
|
||
def _backup_sample_once(path: Path) -> None:
|
||
"""首次覆盖前，把原始示例数据备份为 chanlun.sample.ts。"""
|
||
if not path.exists():
|
||
return
|
||
head = "\n".join(path.read_text(encoding="utf-8").splitlines()[:6])
|
||
if GEN_MARKER in head:
|
||
return # 已是生成文件，无需备份
|
||
backup = path.with_name("chanlun.sample.ts")
|
||
if backup.exists():
|
||
return
|
||
shutil.copy2(path, backup)
|
||
print(f" 已备份示例数据 → {backup.name}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|