first commit

main.py

@@ -0,0 +1,126 @@
+"""缠论《教你炒股票》108 课抓取入口。
+
+用法
+----
+    python main.py              # 抓取全部并生成数据（使用本地缓存）
+    python main.py --refresh    # 忽略缓存重新抓取
+    python main.py --limit 5    # 仅抓前 5 篇（联调测试）
+    python main.py --no-images  # 不下载图片，正文保留远程链接
+
+产物
+----
+    data/raw/p001.html ...          原始 HTML 归档
+    data/markdown/ke-001.md ...     每篇 Markdown 归档
+    data/articles.json              全量结构化数据
+    D:\\code\\chan\\src\\data\\chanlun.ts        Vue 数据源（覆盖示例）
+    D:\\code\\chan\\public\\chanlun-img\\...     本地化图片
+"""
+
+import argparse
+import json
+import shutil
+from datetime import datetime
+from pathlib import Path
+
+from config import (
+    CHAN_DATA_FILE,
+    INDEX_URL,
+    JSON_PATH,
+    MARKDOWN_DIR,
+    RAW_DIR,
+    article_url,
+)
+from fetcher import fetch_text
+from parser import parse_article, parse_index
+from writer import make_image_resolver, remote_image_resolver, write_ts_file
+
+GEN_MARKER = "自动生成"
+
+
+def cached_html(p_num: int, refresh: bool) -> str:
+    """读取/抓取单篇 HTML，落盘缓存以便重跑。"""
+    path = RAW_DIR / f"p{p_num:03d}.html"
+    if not refresh and path.exists():
+        return path.read_text(encoding="utf-8")
+    html = fetch_text(article_url(p_num))
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(html, encoding="utf-8")
+    return html
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description="抓取缠论《教你炒股票》系列")
+    ap.add_argument("--refresh", action="store_true", help="忽略缓存重新抓取")
+    ap.add_argument("--limit", type=int, default=0, help="仅抓取前 N 篇（测试）")
+    ap.add_argument("--no-images", action="store_true", help="不下载图片，保留远程链接")
+    args = ap.parse_args()
+
+    print("① 抓取目录页 …")
+    index_html = fetch_text(INDEX_URL)
+    pages = parse_index(index_html)
+    if args.limit:
+        pages = pages[: args.limit]
+    if not pages:
+        raise SystemExit("目录页未解析到任何文章链接")
+    print(f"   发现 {len(pages)} 篇文章：p={pages[0]}…{pages[-1]}")
+
+    RAW_DIR.mkdir(parents=True, exist_ok=True)
+    MARKDOWN_DIR.mkdir(parents=True, exist_ok=True)
+
+    articles: list[dict] = []
+    for i, p in enumerate(pages, 1):
+        print(f"② [{i}/{len(pages)}] 解析 p={p} …")
+        html = cached_html(p, args.refresh)
+        resolver = remote_image_resolver if args.no_images else make_image_resolver(p, args.refresh)
+        art = parse_article(html, p, resolver)
+        (MARKDOWN_DIR / f"{art['slug']}.md").write_text(
+            f"# {art['title']}\n\n{art['content']}\n", encoding="utf-8"
+        )
+        articles.append(art)
+
+    # 源站少数篇目日期残缺/缺失（如 h5 仅写"2006"），按课次顺序用上一篇日期兜底
+    last_date = "2006-06-07"
+    fixed = 0
+    for a in articles:
+        if not a["date"]:
+            a["date"] = last_date
+            fixed += 1
+        else:
+            last_date = a["date"]
+    if fixed:
+        print(f"   ℹ {fixed} 篇源站日期缺失，已用相邻课次日期兜底")
+
+    print("③ 写入归档 JSON …")
+    JSON_PATH.parent.mkdir(parents=True, exist_ok=True)
+    JSON_PATH.write_text(
+        json.dumps(articles, ensure_ascii=False, indent=2), encoding="utf-8"
+    )
+
+    print("④ 生成 Vue 数据文件 …")
+    _backup_sample_once(CHAN_DATA_FILE)
+    stamp = datetime.now().strftime("%Y-%m-%d %H:%M")
+    write_ts_file(articles, CHAN_DATA_FILE, stamp)
+    print(f"   → {CHAN_DATA_FILE}")
+
+    print("✅ 完成。")
+    print(f"   文章数：{len(articles)}")
+    print("   归档目录：data/")
+    print("   Vue 路由：http://localhost:5173/chanlun")
+
+
+def _backup_sample_once(path: Path) -> None:
+    """首次覆盖前，把原始示例数据备份为 chanlun.sample.ts。"""
+    if not path.exists():
+        return
+    head = "\n".join(path.read_text(encoding="utf-8").splitlines()[:6])
+    if GEN_MARKER in head:
+        return  # 已是生成文件，无需备份
+    backup = path.with_name("chanlun.sample.ts")
+    if backup.exists():
+        return
+    shutil.copy2(path, backup)
+    print(f"   已备份示例数据 → {backup.name}")
+
+
+if __name__ == "__main__":
+    main()