405 lines
16 KiB
Python
405 lines
16 KiB
Python
"""指数专题数据源（tushare 指数接口族）。
|
||
|
||
- 列表层：21 个国际指数最新收盘 + 45 日 spark（pro.index_global），SWR 整包缓存
|
||
（模式同 market_overview：进程内新鲜期直返 -> Redis 兜底 -> 过期先返旧值后台刷新）。
|
||
- K 线层：单指数日线全量。国内指数（000001.SH 形式）复用 index_series.get_index_daily；
|
||
国际指数走本模块 index_global 分页拉全量（单次 4000），进程内 + Redis 两级缓存。
|
||
- 元数据：国内指数 pro.index_basic 按需拉（24h 缓存）；国际指数 tushare 无元数据，
|
||
内置静态表（名称/地区/国家）。
|
||
- 估值（pro.index_dailybasic，仅 8 大国内指数有数据）与成分权重（pro.index_weight，
|
||
月度，仅国内指数）：按需拉 + Redis 中长 TTL 缓存。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import time
|
||
from datetime import date, datetime, timedelta
|
||
|
||
from .. import cache
|
||
from ..config import settings
|
||
from ..domain import Bar
|
||
from .sync_utils import d8_iso, f_clean
|
||
|
||
# ---- 静态元数据表（tushare index_global 支持的全部 21 个指数，展示顺序即文档顺序）----
|
||
# region: americas 美洲 / europe 欧洲 / asia 亚太（含港股与富时A50）
|
||
GLOBAL_INDEXES: list[dict] = [
|
||
{"code": "DJI", "name": "道琼斯工业指数", "region": "americas", "country": "美国"},
|
||
{"code": "SPX", "name": "标普500", "region": "americas", "country": "美国"},
|
||
{"code": "IXIC", "name": "纳斯达克综合指数", "region": "americas", "country": "美国"},
|
||
{"code": "RUT", "name": "罗素2000", "region": "americas", "country": "美国"},
|
||
{"code": "SPTSX", "name": "加拿大S&P/TSX", "region": "americas", "country": "加拿大"},
|
||
{"code": "IBOVESPA", "name": "巴西IBOVESPA", "region": "americas", "country": "巴西"},
|
||
{"code": "FTSE", "name": "富时100", "region": "europe", "country": "英国"},
|
||
{"code": "FCHI", "name": "法国CAC40", "region": "europe", "country": "法国"},
|
||
{"code": "GDAXI", "name": "德国DAX", "region": "europe", "country": "德国"},
|
||
{"code": "CSX5P", "name": "STOXX欧洲50", "region": "europe", "country": "欧洲"},
|
||
{"code": "RTS", "name": "俄罗斯RTS", "region": "europe", "country": "俄罗斯"},
|
||
{"code": "HSI", "name": "恒生指数", "region": "asia", "country": "中国香港"},
|
||
{"code": "HKTECH", "name": "恒生科技指数", "region": "asia", "country": "中国香港"},
|
||
{"code": "HKAH", "name": "恒生AH股H指数", "region": "asia", "country": "中国香港"},
|
||
{"code": "XIN9", "name": "富时中国A50", "region": "asia", "country": "新加坡"},
|
||
{"code": "N225", "name": "日经225", "region": "asia", "country": "日本"},
|
||
{"code": "KS11", "name": "韩国综合指数", "region": "asia", "country": "韩国"},
|
||
{"code": "TWII", "name": "台湾加权指数", "region": "asia", "country": "中国台湾"},
|
||
{"code": "AS51", "name": "澳大利亚标普200", "region": "asia", "country": "澳大利亚"},
|
||
{"code": "SENSEX", "name": "印度孟买SENSEX", "region": "asia", "country": "印度"},
|
||
{"code": "CKLSE", "name": "马来西亚指数", "region": "asia", "country": "马来西亚"},
|
||
]
|
||
GLOBAL_META = {g["code"]: g for g in GLOBAL_INDEXES}
|
||
|
||
# 国内指数白名单（K 线 / 详情 / 权重可用范围，防止任意 code 打爆 tushare）
|
||
CN_INDEXES: dict[str, str] = {
|
||
"000001.SH": "上证指数",
|
||
"399001.SZ": "深证成指",
|
||
"399006.SZ": "创业板指",
|
||
"000688.SH": "科创50",
|
||
"000300.SH": "沪深300",
|
||
"000016.SH": "上证50",
|
||
"000905.SH": "中证500",
|
||
"000852.SH": "中证1000",
|
||
"399016.SZ": "深证100",
|
||
}
|
||
|
||
# index_dailybasic 实际有数据的指数（接口文档写 6 个，实测含 000300/399016 共 8 个）
|
||
DAILYBASIC_CODES = {
|
||
"000001.SH", "000016.SH", "000300.SH", "000905.SH",
|
||
"399001.SZ", "399005.SZ", "399006.SZ", "399016.SZ",
|
||
}
|
||
|
||
_SPARK_DAYS = 45
|
||
_HISTORY_DAYS = 150 # 日历日窗口（约 100 交易日，够取 spark）
|
||
_LIST_KEY = "global_indexes:eod:v1"
|
||
_BARS_KEY = "idxgb:" # + code（全量日线紧凑 JSON）
|
||
_BASIC_KEY = "idxbm:" # + code（index_basic 元数据）
|
||
_VAL_KEY = "idxvm:" # + code（dailybasic 估值序列）
|
||
_W_KEY = "idxwm:" # + code（index_weight 最近月度）
|
||
_PAGE = 4000 # index_global 单次返回上限
|
||
_CALL_INTERVAL = 0.12 # 顺序调用间隔（秒），对 tushare 控频
|
||
|
||
|
||
class GlobalIndexError(RuntimeError):
|
||
"""全部国际指数都拉不到（token/网络故障）——接口层转 503。"""
|
||
|
||
|
||
def is_cn_index(code: str) -> bool:
|
||
return "." in code
|
||
|
||
|
||
def ensure_known(code: str) -> bool:
|
||
"""详情/K线/权重接口只放行白名单内的 code。"""
|
||
return code in CN_INDEXES or code in GLOBAL_META
|
||
|
||
|
||
# ======================= 列表层：21 个国际指数最新行情（SWR 整包） =======================
|
||
|
||
def _get_pro():
|
||
from .tushare_provider import get_pro
|
||
|
||
return get_pro()
|
||
|
||
|
||
def _fetch_quote_sync(pro, ts_code: str) -> dict:
|
||
"""单个指数近 _HISTORY_DAYS 日行情 -> 最新一根 + spark（旧 -> 新）。"""
|
||
start = (datetime.now() - timedelta(days=_HISTORY_DAYS)).strftime("%Y%m%d")
|
||
if is_cn_index(ts_code):
|
||
df = pro.index_daily(ts_code=ts_code, start_date=start)
|
||
else:
|
||
df = pro.index_global(ts_code=ts_code, start_date=start)
|
||
if df is None or df.empty:
|
||
raise GlobalIndexError("无数据")
|
||
df = df.sort_values("trade_date")
|
||
tail = df.tail(_SPARK_DAYS)
|
||
last = df.iloc[-1]
|
||
return {
|
||
"close": f_clean(last["close"]),
|
||
"change": f_clean(last.get("change")),
|
||
"pct_chg": f_clean(last.get("pct_chg")),
|
||
"open": f_clean(last.get("open")),
|
||
"high": f_clean(last.get("high")),
|
||
"low": f_clean(last.get("low")),
|
||
"pre_close": f_clean(last.get("pre_close")),
|
||
"trade_date": d8_iso(last["trade_date"]),
|
||
"spark": [round(float(c), 4) for c in tail["close"]],
|
||
"spark_dates": [str(d) for d in tail["trade_date"]],
|
||
}
|
||
|
||
|
||
_list_state: dict = {"payload": None}
|
||
_list_refreshing = False
|
||
_list_refresh_error: str | None = None
|
||
_bg_tasks: set[asyncio.Task] = set()
|
||
|
||
|
||
async def _refresh_list() -> dict:
|
||
"""拉全量 21 个国际指数 EOD（顺序控频 ~8s），写进程内 state + Redis。"""
|
||
pro = await asyncio.to_thread(_get_pro)
|
||
|
||
items: list[dict] = []
|
||
errors: list[str] = []
|
||
for g in GLOBAL_INDEXES:
|
||
try:
|
||
q = await asyncio.to_thread(_fetch_quote_sync, pro, g["code"])
|
||
items.append({**g, **q})
|
||
except Exception as e: # noqa: BLE001 —— 单指数失败不拖垮整包
|
||
errors.append(f"{g['name']}: {str(e)[:60]}")
|
||
await asyncio.sleep(_CALL_INTERVAL)
|
||
|
||
if not items:
|
||
raise GlobalIndexError("国际指数全部拉取失败: " + "; ".join(errors)[:200])
|
||
|
||
payload = {
|
||
"fetched_at": datetime.now().isoformat(),
|
||
"fetched_ts": time.time(),
|
||
"items": items,
|
||
"errors": errors,
|
||
}
|
||
_list_state["payload"] = payload
|
||
await cache.cache_set(_LIST_KEY, payload, ttl=settings.market_eod_redis_ttl)
|
||
return payload
|
||
|
||
|
||
async def _refresh_list_wrapped() -> None:
|
||
global _list_refresh_error, _list_refreshing
|
||
try:
|
||
await _refresh_list()
|
||
_list_refresh_error = None
|
||
except Exception as e: # noqa: BLE001
|
||
_list_refresh_error = f"国际指数后台刷新: {str(e)[:60]}"
|
||
finally:
|
||
_list_refreshing = False
|
||
|
||
|
||
def _spawn_refresh() -> None:
|
||
global _list_refreshing
|
||
if _list_refreshing:
|
||
return
|
||
_list_refreshing = True
|
||
task = asyncio.create_task(_refresh_list_wrapped())
|
||
_bg_tasks.add(task)
|
||
task.add_done_callback(_bg_tasks.discard)
|
||
|
||
|
||
async def fetch_global_list() -> dict:
|
||
"""国际指数列表：内存新鲜直返 -> Redis 回填 -> 有旧值先返 + SWR 后台刷新 -> 冷启动同步拉。"""
|
||
p = _list_state["payload"]
|
||
if p is not None and time.time() - p["fetched_ts"] < settings.market_eod_fresh_ttl:
|
||
return p
|
||
if p is None:
|
||
cached = await cache.cache_get(_LIST_KEY)
|
||
if cached:
|
||
p = cached
|
||
_list_state["payload"] = p
|
||
if p is not None:
|
||
_spawn_refresh()
|
||
return p
|
||
return await _refresh_list()
|
||
|
||
|
||
async def fetch_index_quote(code: str) -> dict:
|
||
"""单指数最新行情（详情页头部）。国内走 index_daily、国际走 index_global；
|
||
Redis 短缓存 2h（收盘口径一天一变）。"""
|
||
key = f"idxqt:{code}"
|
||
raw = await cache.cache_get(key)
|
||
if isinstance(raw, dict):
|
||
return raw
|
||
pro = await asyncio.to_thread(_get_pro)
|
||
q = await asyncio.to_thread(_fetch_quote_sync, pro, code)
|
||
await cache.cache_set(key, q, ttl=7200)
|
||
return q
|
||
|
||
|
||
# ======================= K 线层：单指数日线全量（两级缓存） =======================
|
||
|
||
def _fetch_global_bars_sync(ts_code: str) -> list[Bar]:
|
||
"""国际指数全量日线（分页）。vol/amount 大部分指数缺失 -> volume 0 / amount None。"""
|
||
pro = _get_pro()
|
||
frames = []
|
||
offset = 0
|
||
while True:
|
||
df = pro.index_global(ts_code=ts_code, offset=offset, limit=_PAGE)
|
||
if df is None or df.empty:
|
||
break
|
||
frames.append(df)
|
||
if len(df) < _PAGE:
|
||
break
|
||
offset += _PAGE
|
||
time.sleep(_CALL_INTERVAL)
|
||
if not frames:
|
||
raise GlobalIndexError(f"Tushare index_global 无数据: {ts_code}")
|
||
|
||
import pandas as pd
|
||
|
||
df = pd.concat(frames).drop_duplicates(subset="trade_date").sort_values("trade_date")
|
||
bars: list[Bar] = []
|
||
for _, r in df.iterrows():
|
||
vol = f_clean(r.get("vol"))
|
||
amt = f_clean(r.get("amount"))
|
||
bars.append(
|
||
Bar(
|
||
ts=datetime.strptime(str(r["trade_date"]), "%Y%m%d"),
|
||
open=float(r["open"]), high=float(r["high"]),
|
||
low=float(r["low"]), close=float(r["close"]),
|
||
volume=vol or 0.0,
|
||
amount=amt,
|
||
)
|
||
)
|
||
return bars
|
||
|
||
|
||
# 进程内缓存（与 index_series 同款）：code -> (bars, 过期时刻)
|
||
_mem: dict[str, tuple[list[Bar], float]] = {}
|
||
_BARS_TTL = 7200
|
||
|
||
|
||
def _bars_to_raw(bars: list[Bar]) -> str:
|
||
return json.dumps(
|
||
[[b.ts.isoformat(), b.open, b.high, b.low, b.close, b.volume, b.amount] for b in bars],
|
||
ensure_ascii=False, separators=(",", ":"),
|
||
)
|
||
|
||
|
||
def _bars_from_raw(raw: str) -> list[Bar]:
|
||
return [
|
||
Bar(ts=datetime.fromisoformat(row[0]), open=row[1], high=row[2], low=row[3],
|
||
close=row[4], volume=row[5], amount=row[6])
|
||
for row in json.loads(raw)
|
||
]
|
||
|
||
|
||
async def get_index_bars(code: str) -> list[Bar]:
|
||
"""单指数全量日线（升序）：国内复用 index_series（上证已有热缓存），国际本模块分页。"""
|
||
if is_cn_index(code):
|
||
from .index_series import get_index_daily
|
||
|
||
return await get_index_daily(code)
|
||
|
||
hit = _mem.get(code)
|
||
if hit and hit[1] > time.monotonic():
|
||
return hit[0]
|
||
key = f"{_BARS_KEY}{code}"
|
||
raw = await cache.cache_get(key)
|
||
if isinstance(raw, str):
|
||
bars = _bars_from_raw(raw)
|
||
_mem[code] = (bars, time.monotonic() + _BARS_TTL)
|
||
return bars
|
||
bars = await asyncio.to_thread(_fetch_global_bars_sync, code)
|
||
_mem[code] = (bars, time.monotonic() + _BARS_TTL)
|
||
await cache.cache_set(key, _bars_to_raw(bars), ttl=_BARS_TTL)
|
||
return bars
|
||
|
||
|
||
# ======================= 元数据：index_basic（国内）/ 静态表（国际） =======================
|
||
|
||
def _fetch_basic_sync(ts_code: str) -> dict:
|
||
df = _get_pro().index_basic(ts_code=ts_code)
|
||
if df is None or df.empty:
|
||
raise GlobalIndexError("index_basic 无数据")
|
||
r = df.iloc[-1] # 同 code 理论唯一，防御性取末行
|
||
return {
|
||
"ts_code": str(r["ts_code"]),
|
||
"name": str(r.get("name") or ""),
|
||
"market": r.get("market"),
|
||
"publisher": r.get("publisher"),
|
||
"category": r.get("category"),
|
||
"base_date": d8_iso(r.get("base_date")),
|
||
"base_point": f_clean(r.get("base_point")),
|
||
"list_date": d8_iso(r.get("list_date")),
|
||
}
|
||
|
||
|
||
async def get_index_basic(code: str) -> dict | None:
|
||
"""指数基本信息：国内 index_basic（24h 缓存，拉不到返 None 不阻塞）；
|
||
国际直接由静态表合成。"""
|
||
if code in GLOBAL_META:
|
||
g = GLOBAL_META[code]
|
||
return {"ts_code": code, "name": g["name"], "market": None, "publisher": None,
|
||
"category": None, "base_date": None, "base_point": None, "list_date": None,
|
||
"country": g["country"], "region": g["region"]}
|
||
key = f"{_BASIC_KEY}{code}"
|
||
raw = await cache.cache_get(key)
|
||
if isinstance(raw, dict):
|
||
return raw
|
||
try:
|
||
basic = await asyncio.to_thread(_fetch_basic_sync, code)
|
||
except Exception: # noqa: BLE001 —— 元数据缺失时详情页行情照常
|
||
return None
|
||
await cache.cache_set(key, basic, ttl=86400)
|
||
return basic
|
||
|
||
|
||
# ======================= 估值：index_dailybasic（仅部分国内指数） =======================
|
||
|
||
def _fetch_valuation_sync(ts_code: str, days: int) -> list[dict]:
|
||
start = (datetime.now() - timedelta(days=days)).strftime("%Y%m%d")
|
||
df = _get_pro().index_dailybasic(ts_code=ts_code, start_date=start)
|
||
if df is None or df.empty:
|
||
return []
|
||
rows = []
|
||
for _, r in df.sort_values("trade_date").iterrows():
|
||
rows.append({
|
||
"trade_date": d8_iso(r["trade_date"]),
|
||
"pe": f_clean(r.get("pe")), "pe_ttm": f_clean(r.get("pe_ttm")), "pb": f_clean(r.get("pb")),
|
||
"turnover_rate": f_clean(r.get("turnover_rate")),
|
||
"total_mv": f_clean(r.get("total_mv")), "float_mv": f_clean(r.get("float_mv")),
|
||
})
|
||
return rows
|
||
|
||
|
||
async def get_index_valuation(code: str, days: int = 400) -> list[dict]:
|
||
"""近 N 日估值序列（升序）。接口只覆盖 DAILYBASIC_CODES 内的指数，其余不调接口直接空。"""
|
||
if code not in DAILYBASIC_CODES:
|
||
return []
|
||
key = f"{_VAL_KEY}{code}:{days}"
|
||
raw = await cache.cache_get(key)
|
||
if isinstance(raw, list):
|
||
return raw
|
||
rows = await asyncio.to_thread(_fetch_valuation_sync, code, days)
|
||
await cache.cache_set(key, rows, ttl=43200)
|
||
return rows
|
||
|
||
|
||
# ======================= 成分权重：index_weight（月度，仅国内指数） =======================
|
||
|
||
def _fetch_weights_sync(ts_code: str) -> dict | None:
|
||
"""index_weight 是月度快照，官方建议按整月窗口查询：本月 -> 上月 -> 前月，取首个有数据的月份。"""
|
||
pro = _get_pro()
|
||
today = date.today()
|
||
for back in range(3):
|
||
first = (today.replace(day=1) - timedelta(days=31 * back)).replace(day=1)
|
||
last_day = (first + timedelta(days=42)).replace(day=1) - timedelta(days=1)
|
||
df = pro.index_weight(
|
||
index_code=ts_code,
|
||
start_date=first.strftime("%Y%m%d"),
|
||
end_date=last_day.strftime("%Y%m%d"),
|
||
)
|
||
if df is None or df.empty:
|
||
continue
|
||
df = df.sort_values(["trade_date", "weight"], ascending=[False, False])
|
||
latest_date = df.iloc[0]["trade_date"]
|
||
rows = df[df["trade_date"] == latest_date]
|
||
return {
|
||
"trade_date": d8_iso(latest_date),
|
||
"total": int(len(rows)),
|
||
"items": [
|
||
{"con_code": str(r["con_code"]), "weight": round(float(r["weight"]), 4)}
|
||
for _, r in rows.sort_values("weight", ascending=False).iterrows()
|
||
],
|
||
}
|
||
return None
|
||
|
||
|
||
async def get_index_weights(code: str) -> dict | None:
|
||
"""最近月度成分权重（全量，按权重降序）。国际指数无此数据，直接 None。"""
|
||
if not is_cn_index(code):
|
||
return None
|
||
key = f"{_W_KEY}{code}"
|
||
raw = await cache.cache_get(key)
|
||
if isinstance(raw, dict):
|
||
return raw
|
||
result = await asyncio.to_thread(_fetch_weights_sync, code)
|
||
if result is None:
|
||
return None
|
||
await cache.cache_set(key, result, ttl=43200)
|
||
return result
|