提交

backend/app/data/index_global.py

@@ -0,0 +1,420 @@
+"""指数专题数据源（tushare 指数接口族）。
+
+- 列表层：21 个国际指数最新收盘 + 45 日 spark（pro.index_global），SWR 整包缓存
+  （模式同 market_overview：进程内新鲜期直返 -> Redis 兜底 -> 过期先返旧值后台刷新）。
+- K 线层：单指数日线全量。国内指数（000001.SH 形式）复用 index_series.get_index_daily；
+  国际指数走本模块 index_global 分页拉全量（单次 4000），进程内 + Redis 两级缓存。
+- 元数据：国内指数 pro.index_basic 按需拉（24h 缓存）；国际指数 tushare 无元数据，
+  内置静态表（名称/地区/国家）。
+- 估值（pro.index_dailybasic，仅 8 大国内指数有数据）与成分权重（pro.index_weight，
+  月度，仅国内指数）：按需拉 + Redis 中长 TTL 缓存。
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import math
+import time
+from datetime import date, datetime, timedelta
+
+from .. import cache
+from ..config import settings
+from ..domain import Bar
+
+# ---- 静态元数据表（tushare index_global 支持的全部 21 个指数，展示顺序即文档顺序）----
+# region: americas 美洲 / europe 欧洲 / asia 亚太（含港股与富时A50）
+GLOBAL_INDEXES: list[dict] = [
+    {"code": "DJI", "name": "道琼斯工业指数", "region": "americas", "country": "美国"},
+    {"code": "SPX", "name": "标普500", "region": "americas", "country": "美国"},
+    {"code": "IXIC", "name": "纳斯达克综合指数", "region": "americas", "country": "美国"},
+    {"code": "RUT", "name": "罗素2000", "region": "americas", "country": "美国"},
+    {"code": "SPTSX", "name": "加拿大S&P/TSX", "region": "americas", "country": "加拿大"},
+    {"code": "IBOVESPA", "name": "巴西IBOVESPA", "region": "americas", "country": "巴西"},
+    {"code": "FTSE", "name": "富时100", "region": "europe", "country": "英国"},
+    {"code": "FCHI", "name": "法国CAC40", "region": "europe", "country": "法国"},
+    {"code": "GDAXI", "name": "德国DAX", "region": "europe", "country": "德国"},
+    {"code": "CSX5P", "name": "STOXX欧洲50", "region": "europe", "country": "欧洲"},
+    {"code": "RTS", "name": "俄罗斯RTS", "region": "europe", "country": "俄罗斯"},
+    {"code": "HSI", "name": "恒生指数", "region": "asia", "country": "中国香港"},
+    {"code": "HKTECH", "name": "恒生科技指数", "region": "asia", "country": "中国香港"},
+    {"code": "HKAH", "name": "恒生AH股H指数", "region": "asia", "country": "中国香港"},
+    {"code": "XIN9", "name": "富时中国A50", "region": "asia", "country": "新加坡"},
+    {"code": "N225", "name": "日经225", "region": "asia", "country": "日本"},
+    {"code": "KS11", "name": "韩国综合指数", "region": "asia", "country": "韩国"},
+    {"code": "TWII", "name": "台湾加权指数", "region": "asia", "country": "中国台湾"},
+    {"code": "AS51", "name": "澳大利亚标普200", "region": "asia", "country": "澳大利亚"},
+    {"code": "SENSEX", "name": "印度孟买SENSEX", "region": "asia", "country": "印度"},
+    {"code": "CKLSE", "name": "马来西亚指数", "region": "asia", "country": "马来西亚"},
+]
+GLOBAL_META = {g["code"]: g for g in GLOBAL_INDEXES}
+
+# 国内指数白名单（K 线 / 详情 / 权重可用范围，防止任意 code 打爆 tushare）
+CN_INDEXES: dict[str, str] = {
+    "000001.SH": "上证指数",
+    "399001.SZ": "深证成指",
+    "399006.SZ": "创业板指",
+    "000688.SH": "科创50",
+    "000300.SH": "沪深300",
+    "000016.SH": "上证50",
+    "000905.SH": "中证500",
+    "000852.SH": "中证1000",
+    "399016.SZ": "深证100",
+}
+
+# index_dailybasic 实际有数据的指数（接口文档写 6 个，实测含 000300/399016 共 8 个）
+DAILYBASIC_CODES = {
+    "000001.SH", "000016.SH", "000300.SH", "000905.SH",
+    "399001.SZ", "399005.SZ", "399006.SZ", "399016.SZ",
+}
+
+_SPARK_DAYS = 45
+_HISTORY_DAYS = 150        # 日历日窗口（约 100 交易日，够取 spark）
+_LIST_KEY = "global_indexes:eod:v1"
+_BARS_KEY = "idxgb:"       # + code（全量日线紧凑 JSON）
+_BASIC_KEY = "idxbm:"      # + code（index_basic 元数据）
+_VAL_KEY = "idxvm:"        # + code（dailybasic 估值序列）
+_W_KEY = "idxwm:"          # + code（index_weight 最近月度）
+_PAGE = 4000               # index_global 单次返回上限
+_CALL_INTERVAL = 0.12      # 顺序调用间隔（秒），对 tushare 控频
+
+
+class GlobalIndexError(RuntimeError):
+    """全部国际指数都拉不到（token/网络故障）——接口层转 503。"""
+
+
+def _f(v) -> float | None:
+    """pandas 值 -> float；NaN/None -> None。"""
+    if v is None:
+        return None
+    try:
+        f = float(v)
+    except (TypeError, ValueError):
+        return None
+    return None if math.isnan(f) else f
+
+
+def _d(v) -> str | None:
+    """YYYYMMDD -> 'YYYY-MM-DD'（字符串便于 JSON 缓存）。"""
+    return datetime.strptime(str(v), "%Y%m%d").date().isoformat() if v else None
+
+
+def is_cn_index(code: str) -> bool:
+    return "." in code
+
+
+def ensure_known(code: str) -> bool:
+    """详情/K线/权重接口只放行白名单内的 code。"""
+    return code in CN_INDEXES or code in GLOBAL_META
+
+
+# ======================= 列表层：21 个国际指数最新行情（SWR 整包） =======================
+
+def _get_pro():
+    from .tushare_provider import get_pro
+
+    return get_pro()
+
+
+def _fetch_quote_sync(pro, ts_code: str) -> dict:
+    """单个指数近 _HISTORY_DAYS 日行情 -> 最新一根 + spark（旧 -> 新）。"""
+    start = (datetime.now() - timedelta(days=_HISTORY_DAYS)).strftime("%Y%m%d")
+    if is_cn_index(ts_code):
+        df = pro.index_daily(ts_code=ts_code, start_date=start)
+    else:
+        df = pro.index_global(ts_code=ts_code, start_date=start)
+    if df is None or df.empty:
+        raise GlobalIndexError("无数据")
+    df = df.sort_values("trade_date")
+    tail = df.tail(_SPARK_DAYS)
+    last = df.iloc[-1]
+    return {
+        "close": _f(last["close"]),
+        "change": _f(last.get("change")),
+        "pct_chg": _f(last.get("pct_chg")),
+        "open": _f(last.get("open")),
+        "high": _f(last.get("high")),
+        "low": _f(last.get("low")),
+        "pre_close": _f(last.get("pre_close")),
+        "trade_date": _d(last["trade_date"]),
+        "spark": [round(float(c), 4) for c in tail["close"]],
+        "spark_dates": [str(d) for d in tail["trade_date"]],
+    }
+
+
+_list_state: dict = {"payload": None}
+_list_refreshing = False
+_list_refresh_error: str | None = None
+_bg_tasks: set[asyncio.Task] = set()
+
+
+async def _refresh_list() -> dict:
+    """拉全量 21 个国际指数 EOD（顺序控频 ~8s），写进程内 state + Redis。"""
+    pro = await asyncio.to_thread(_get_pro)
+
+    items: list[dict] = []
+    errors: list[str] = []
+    for g in GLOBAL_INDEXES:
+        try:
+            q = await asyncio.to_thread(_fetch_quote_sync, pro, g["code"])
+            items.append({**g, **q})
+        except Exception as e:  # noqa: BLE001 —— 单指数失败不拖垮整包
+            errors.append(f"{g['name']}: {str(e)[:60]}")
+        await asyncio.sleep(_CALL_INTERVAL)
+
+    if not items:
+        raise GlobalIndexError("国际指数全部拉取失败: " + "; ".join(errors)[:200])
+
+    payload = {
+        "fetched_at": datetime.now().isoformat(),
+        "fetched_ts": time.time(),
+        "items": items,
+        "errors": errors,
+    }
+    _list_state["payload"] = payload
+    await cache.cache_set(_LIST_KEY, payload, ttl=settings.market_eod_redis_ttl)
+    return payload
+
+
+async def _refresh_list_wrapped() -> None:
+    global _list_refresh_error, _list_refreshing
+    try:
+        await _refresh_list()
+        _list_refresh_error = None
+    except Exception as e:  # noqa: BLE001
+        _list_refresh_error = f"国际指数后台刷新: {str(e)[:60]}"
+    finally:
+        _list_refreshing = False
+
+
+def _spawn_refresh() -> None:
+    global _list_refreshing
+    if _list_refreshing:
+        return
+    _list_refreshing = True
+    task = asyncio.create_task(_refresh_list_wrapped())
+    _bg_tasks.add(task)
+    task.add_done_callback(_bg_tasks.discard)
+
+
+async def fetch_global_list() -> dict:
+    """国际指数列表：内存新鲜直返 -> Redis 回填 -> 有旧值先返 + SWR 后台刷新 -> 冷启动同步拉。"""
+    p = _list_state["payload"]
+    if p is not None and time.time() - p["fetched_ts"] < settings.market_eod_fresh_ttl:
+        return p
+    if p is None:
+        cached = await cache.cache_get(_LIST_KEY)
+        if cached:
+            p = cached
+            _list_state["payload"] = p
+    if p is not None:
+        _spawn_refresh()
+        return p
+    return await _refresh_list()
+
+
+async def fetch_index_quote(code: str) -> dict:
+    """单指数最新行情（详情页头部）。国内走 index_daily、国际走 index_global；
+    Redis 短缓存 2h（收盘口径一天一变）。"""
+    key = f"idxqt:{code}"
+    raw = await cache.cache_get(key)
+    if isinstance(raw, dict):
+        return raw
+    pro = await asyncio.to_thread(_get_pro)
+    q = await asyncio.to_thread(_fetch_quote_sync, pro, code)
+    await cache.cache_set(key, q, ttl=7200)
+    return q
+
+
+# ======================= K 线层：单指数日线全量（两级缓存） =======================
+
+def _fetch_global_bars_sync(ts_code: str) -> list[Bar]:
+    """国际指数全量日线（分页）。vol/amount 大部分指数缺失 -> volume 0 / amount None。"""
+    pro = _get_pro()
+    frames = []
+    offset = 0
+    while True:
+        df = pro.index_global(ts_code=ts_code, offset=offset, limit=_PAGE)
+        if df is None or df.empty:
+            break
+        frames.append(df)
+        if len(df) < _PAGE:
+            break
+        offset += _PAGE
+        time.sleep(_CALL_INTERVAL)
+    if not frames:
+        raise GlobalIndexError(f"Tushare index_global 无数据: {ts_code}")
+
+    import pandas as pd
+
+    df = pd.concat(frames).drop_duplicates(subset="trade_date").sort_values("trade_date")
+    bars: list[Bar] = []
+    for _, r in df.iterrows():
+        vol = _f(r.get("vol"))
+        amt = _f(r.get("amount"))
+        bars.append(
+            Bar(
+                ts=datetime.strptime(str(r["trade_date"]), "%Y%m%d"),
+                open=float(r["open"]), high=float(r["high"]),
+                low=float(r["low"]), close=float(r["close"]),
+                volume=vol or 0.0,
+                amount=amt,
+            )
+        )
+    return bars
+
+
+# 进程内缓存（与 index_series 同款）：code -> (bars, 过期时刻)
+_mem: dict[str, tuple[list[Bar], float]] = {}
+_BARS_TTL = 7200
+
+
+def _bars_to_raw(bars: list[Bar]) -> str:
+    return json.dumps(
+        [[b.ts.isoformat(), b.open, b.high, b.low, b.close, b.volume, b.amount] for b in bars],
+        ensure_ascii=False, separators=(",", ":"),
+    )
+
+
+def _bars_from_raw(raw: str) -> list[Bar]:
+    return [
+        Bar(ts=datetime.fromisoformat(row[0]), open=row[1], high=row[2], low=row[3],
+            close=row[4], volume=row[5], amount=row[6])
+        for row in json.loads(raw)
+    ]
+
+
+async def get_index_bars(code: str) -> list[Bar]:
+    """单指数全量日线（升序）：国内复用 index_series（上证已有热缓存），国际本模块分页。"""
+    if is_cn_index(code):
+        from .index_series import get_index_daily
+
+        return await get_index_daily(code)
+
+    hit = _mem.get(code)
+    if hit and hit[1] > time.monotonic():
+        return hit[0]
+    key = f"{_BARS_KEY}{code}"
+    raw = await cache.cache_get(key)
+    if isinstance(raw, str):
+        bars = _bars_from_raw(raw)
+        _mem[code] = (bars, time.monotonic() + _BARS_TTL)
+        return bars
+    bars = await asyncio.to_thread(_fetch_global_bars_sync, code)
+    _mem[code] = (bars, time.monotonic() + _BARS_TTL)
+    await cache.cache_set(key, _bars_to_raw(bars), ttl=_BARS_TTL)
+    return bars
+
+
+# ======================= 元数据：index_basic（国内）/ 静态表（国际） =======================
+
+def _fetch_basic_sync(ts_code: str) -> dict:
+    df = _get_pro().index_basic(ts_code=ts_code)
+    if df is None or df.empty:
+        raise GlobalIndexError("index_basic 无数据")
+    r = df.iloc[-1]  # 同 code 理论唯一，防御性取末行
+    return {
+        "ts_code": str(r["ts_code"]),
+        "name": str(r.get("name") or ""),
+        "market": r.get("market"),
+        "publisher": r.get("publisher"),
+        "category": r.get("category"),
+        "base_date": _d(r.get("base_date")),
+        "base_point": _f(r.get("base_point")),
+        "list_date": _d(r.get("list_date")),
+    }
+
+
+async def get_index_basic(code: str) -> dict | None:
+    """指数基本信息：国内 index_basic（24h 缓存，拉不到返 None 不阻塞）；
+    国际直接由静态表合成。"""
+    if code in GLOBAL_META:
+        g = GLOBAL_META[code]
+        return {"ts_code": code, "name": g["name"], "market": None, "publisher": None,
+                "category": None, "base_date": None, "base_point": None, "list_date": None,
+                "country": g["country"], "region": g["region"]}
+    key = f"{_BASIC_KEY}{code}"
+    raw = await cache.cache_get(key)
+    if isinstance(raw, dict):
+        return raw
+    try:
+        basic = await asyncio.to_thread(_fetch_basic_sync, code)
+    except Exception:  # noqa: BLE001 —— 元数据缺失时详情页行情照常
+        return None
+    await cache.cache_set(key, basic, ttl=86400)
+    return basic
+
+
+# ======================= 估值：index_dailybasic（仅部分国内指数） =======================
+
+def _fetch_valuation_sync(ts_code: str, days: int) -> list[dict]:
+    start = (datetime.now() - timedelta(days=days)).strftime("%Y%m%d")
+    df = _get_pro().index_dailybasic(ts_code=ts_code, start_date=start)
+    if df is None or df.empty:
+        return []
+    rows = []
+    for _, r in df.sort_values("trade_date").iterrows():
+        rows.append({
+            "trade_date": _d(r["trade_date"]),
+            "pe": _f(r.get("pe")), "pe_ttm": _f(r.get("pe_ttm")), "pb": _f(r.get("pb")),
+            "turnover_rate": _f(r.get("turnover_rate")),
+            "total_mv": _f(r.get("total_mv")), "float_mv": _f(r.get("float_mv")),
+        })
+    return rows
+
+
+async def get_index_valuation(code: str, days: int = 400) -> list[dict]:
+    """近 N 日估值序列（升序）。接口只覆盖 DAILYBASIC_CODES 内的指数，其余不调接口直接空。"""
+    if code not in DAILYBASIC_CODES:
+        return []
+    key = f"{_VAL_KEY}{code}:{days}"
+    raw = await cache.cache_get(key)
+    if isinstance(raw, list):
+        return raw
+    rows = await asyncio.to_thread(_fetch_valuation_sync, code, days)
+    await cache.cache_set(key, rows, ttl=43200)
+    return rows
+
+
+# ======================= 成分权重：index_weight（月度，仅国内指数） =======================
+
+def _fetch_weights_sync(ts_code: str) -> dict | None:
+    """index_weight 是月度快照，官方建议按整月窗口查询：本月 -> 上月 -> 前月，取首个有数据的月份。"""
+    pro = _get_pro()
+    today = date.today()
+    for back in range(3):
+        first = (today.replace(day=1) - timedelta(days=31 * back)).replace(day=1)
+        last_day = (first + timedelta(days=42)).replace(day=1) - timedelta(days=1)
+        df = pro.index_weight(
+            index_code=ts_code,
+            start_date=first.strftime("%Y%m%d"),
+            end_date=last_day.strftime("%Y%m%d"),
+        )
+        if df is None or df.empty:
+            continue
+        df = df.sort_values(["trade_date", "weight"], ascending=[False, False])
+        latest_date = df.iloc[0]["trade_date"]
+        rows = df[df["trade_date"] == latest_date]
+        return {
+            "trade_date": _d(latest_date),
+            "total": int(len(rows)),
+            "items": [
+                {"con_code": str(r["con_code"]), "weight": round(float(r["weight"]), 4)}
+                for _, r in rows.sort_values("weight", ascending=False).iterrows()
+            ],
+        }
+    return None
+
+
+async def get_index_weights(code: str) -> dict | None:
+    """最近月度成分权重（全量，按权重降序）。国际指数无此数据，直接 None。"""
+    if not is_cn_index(code):
+        return None
+    key = f"{_W_KEY}{code}"
+    raw = await cache.cache_get(key)
+    if isinstance(raw, dict):
+        return raw
+    result = await asyncio.to_thread(_fetch_weights_sync, code)
+    if result is None:
+        return None
+    await cache.cache_set(key, result, ttl=43200)
+    return result