"""缠论《教你炒股票》108 课抓取入口。 用法 ---- python main.py # 抓取全部并生成数据(使用本地缓存) python main.py --refresh # 忽略缓存重新抓取 python main.py --limit 5 # 仅抓前 5 篇(联调测试) python main.py --no-images # 不下载图片,正文保留远程链接 产物 ---- data/raw/p001.html ... 原始 HTML 归档 data/markdown/ke-001.md ... 每篇 Markdown 归档 data/articles.json 全量结构化数据 D:\\code\\chan\\src\\data\\chanlun.ts Vue 数据源(覆盖示例) D:\\code\\chan\\public\\chanlun-img\\... 本地化图片 """ import argparse import json import shutil from datetime import datetime from pathlib import Path from config import ( CHAN_DATA_FILE, INDEX_URL, JSON_PATH, MARKDOWN_DIR, RAW_DIR, article_url, ) from fetcher import fetch_text from parser import parse_article, parse_index from writer import make_image_resolver, remote_image_resolver, write_ts_file GEN_MARKER = "自动生成" def cached_html(p_num: int, refresh: bool) -> str: """读取/抓取单篇 HTML,落盘缓存以便重跑。""" path = RAW_DIR / f"p{p_num:03d}.html" if not refresh and path.exists(): return path.read_text(encoding="utf-8") html = fetch_text(article_url(p_num)) path.parent.mkdir(parents=True, exist_ok=True) path.write_text(html, encoding="utf-8") return html def main() -> None: ap = argparse.ArgumentParser(description="抓取缠论《教你炒股票》系列") ap.add_argument("--refresh", action="store_true", help="忽略缓存重新抓取") ap.add_argument("--limit", type=int, default=0, help="仅抓取前 N 篇(测试)") ap.add_argument("--no-images", action="store_true", help="不下载图片,保留远程链接") args = ap.parse_args() print("① 抓取目录页 …") index_html = fetch_text(INDEX_URL) pages = parse_index(index_html) if args.limit: pages = pages[: args.limit] if not pages: raise SystemExit("目录页未解析到任何文章链接") print(f" 发现 {len(pages)} 篇文章:p={pages[0]}…{pages[-1]}") RAW_DIR.mkdir(parents=True, exist_ok=True) MARKDOWN_DIR.mkdir(parents=True, exist_ok=True) articles: list[dict] = [] for i, p in enumerate(pages, 1): print(f"② [{i}/{len(pages)}] 解析 p={p} …") html = cached_html(p, args.refresh) resolver = remote_image_resolver if args.no_images else make_image_resolver(p, args.refresh) art = parse_article(html, p, resolver) (MARKDOWN_DIR / f"{art['slug']}.md").write_text( f"# {art['title']}\n\n{art['content']}\n", encoding="utf-8" ) articles.append(art) # 源站少数篇目日期残缺/缺失(如 h5 仅写"2006"),按课次顺序用上一篇日期兜底 last_date = "2006-06-07" fixed = 0 for a in articles: if not a["date"]: a["date"] = last_date fixed += 1 else: last_date = a["date"] if fixed: print(f" ℹ {fixed} 篇源站日期缺失,已用相邻课次日期兜底") print("③ 写入归档 JSON …") JSON_PATH.parent.mkdir(parents=True, exist_ok=True) JSON_PATH.write_text( json.dumps(articles, ensure_ascii=False, indent=2), encoding="utf-8" ) print("④ 生成 Vue 数据文件 …") _backup_sample_once(CHAN_DATA_FILE) stamp = datetime.now().strftime("%Y-%m-%d %H:%M") write_ts_file(articles, CHAN_DATA_FILE, stamp) print(f" → {CHAN_DATA_FILE}") print("✅ 完成。") print(f" 文章数:{len(articles)}") print(" 归档目录:data/") print(" Vue 路由:http://localhost:5173/chanlun") def _backup_sample_once(path: Path) -> None: """首次覆盖前,把原始示例数据备份为 chanlun.sample.ts。""" if not path.exists(): return head = "\n".join(path.read_text(encoding="utf-8").splitlines()[:6]) if GEN_MARKER in head: return # 已是生成文件,无需备份 backup = path.with_name("chanlun.sample.ts") if backup.exists(): return shutil.copy2(path, backup) print(f" 已备份示例数据 → {backup.name}") if __name__ == "__main__": main()