163 lines
7.0 KiB
Python
163 lines
7.0 KiB
Python
"""全量回补换手率（candles.turnover，单位 %）。
|
||
|
||
用法（在 backend 目录下）：
|
||
uv run python scripts/backfill_turnover.py # 从 2000-01-01（daily_basic 起点）回补到今天
|
||
uv run python scripts/backfill_turnover.py --start 20200101
|
||
uv run python scripts/backfill_turnover.py --force # 已回补的交易日也重拉
|
||
|
||
- 数据源：Tushare daily_basic(trade_date=..., fields='ts_code,turnover_rate')，按日全市场；
|
||
- 幂等可断点续跑：某交易日 candles 已有非空 turnover 即跳过（--force 强制重做）；
|
||
- 交易日取自本地 trade_calendar（缓存覆盖不到起点时自动拉一次宽范围日历）；
|
||
- 每日一条 UPDATE ... FROM unnest(...) 批量写回，仅更新 turnover 列；
|
||
- Tushare 每分钟限频由 _call_retry 自动等待 62s 重试。
|
||
|
||
注意：与 import_tdx_day.py（回填 amount 会整行 upsert）串行运行，避免同表行锁竞争。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import asyncio
|
||
import sys
|
||
import time
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||
|
||
from app.screener.market_sync import _call_retry, _get_pro
|
||
|
||
import asyncpg
|
||
|
||
|
||
def load_db_url() -> str:
|
||
"""与 import_tdx_day.py 相同的 .env -> libpq URL 解析（本地复制避免跨脚本导入）。"""
|
||
env = Path(__file__).resolve().parent.parent / ".env"
|
||
if env.exists():
|
||
for line in env.read_text(encoding="utf-8").splitlines():
|
||
line = line.strip()
|
||
if line.startswith("DATABASE_URL=postgresql+asyncpg://"):
|
||
return "postgresql://" + line.split("://", 1)[1]
|
||
return "postgresql://postgres:postgres@localhost:5432/stock"
|
||
|
||
_DAILY_BASIC_FLOOR = "20000101" # daily_basic 最早覆盖 2000-01-04，更早的交易日无换手数据
|
||
_INTERVAL_MSG = 20
|
||
|
||
|
||
async def _calendar_dates(conn: asyncpg.Connection, start: str, end: str) -> list[str]:
|
||
"""[start, end] 交易日（升序）。本地缓存覆盖不到起点时拉一次宽范围日历并回写。"""
|
||
cached = [r[0] for r in await conn.fetch(
|
||
"SELECT trade_date FROM trade_calendar WHERE trade_date >= $1 AND trade_date <= $2 "
|
||
"ORDER BY trade_date", start, end)]
|
||
if cached and cached[0] <= start:
|
||
return cached
|
||
|
||
pro = _get_pro()
|
||
try:
|
||
cal = await asyncio.to_thread(
|
||
_call_retry, pro.trade_cal, exchange="SSE", start_date=start, end_date=end, is_open="1"
|
||
)
|
||
dates = sorted(cal["cal_date"].tolist())
|
||
except Exception as e: # noqa: BLE001
|
||
if not cached:
|
||
raise
|
||
print(f"交易日历拉取受限（{str(e)[:100]}），沿用本地缓存")
|
||
return cached
|
||
have = set(cached)
|
||
fresh = [d for d in dates if d not in have]
|
||
if fresh:
|
||
await conn.executemany(
|
||
"INSERT INTO trade_calendar (trade_date) VALUES ($1) ON CONFLICT DO NOTHING", [(d,) for d in fresh]
|
||
)
|
||
return dates
|
||
|
||
|
||
async def _day_status(conn: asyncpg.Connection, d: str) -> tuple[int, int]:
|
||
"""(已有换手的行数, 当日总行数)。无行情的日子 total=0 直接跳过。"""
|
||
row = await conn.fetchrow(
|
||
"SELECT count(*) FILTER (WHERE turnover IS NOT NULL) AS done, count(*) AS total "
|
||
"FROM candles WHERE timeframe = '1d' AND ts = $1::timestamp", datetime.strptime(d, "%Y%m%d")
|
||
)
|
||
return row["done"], row["total"]
|
||
|
||
|
||
async def main(start: str, end: str, force: bool) -> None:
|
||
conn = await asyncpg.connect(load_db_url())
|
||
try:
|
||
# 默认起点：daily_basic 覆盖范围与 candles 最早日线的较大者（更早的日期拉了也是空）
|
||
if start is None:
|
||
first = await conn.fetchval(
|
||
"SELECT min(ts) FROM candles WHERE timeframe = '1d' AND symbol <> 'DEMO'")
|
||
start = max(first.strftime("%Y%m%d"), _DAILY_BASIC_FLOOR) if first else _DAILY_BASIC_FLOOR
|
||
if end is None:
|
||
end = datetime.now().strftime("%Y%m%d")
|
||
|
||
dates = await _calendar_dates(conn, start, end)
|
||
todo: list[str] = []
|
||
for d in dates:
|
||
if force:
|
||
done, total = await _day_status(conn, d)
|
||
if total:
|
||
todo.append(d)
|
||
continue
|
||
done, total = await _day_status(conn, d)
|
||
if total and done < total // 2: # 过半缺换手才重做（容忍个别股票无快照）
|
||
todo.append(d)
|
||
print(f"区间 {start}~{end} 共 {len(dates)} 个交易日，待回补 {len(todo)} 个")
|
||
|
||
pro = _get_pro()
|
||
done = 0
|
||
t0 = time.time()
|
||
for d in todo:
|
||
time.sleep(0.15) # 轻微控频；分钟级限频由 _call_retry 自动等待重试
|
||
df = None
|
||
for attempt in range(5): # 网络抖动（超时/断连）也重试，_call_retry 只兜限频
|
||
try:
|
||
df = _call_retry(
|
||
pro.daily_basic, trade_date=d, fields="ts_code,trade_date,turnover_rate"
|
||
)
|
||
break
|
||
except Exception as e: # noqa: BLE001
|
||
wait = min(30 * (attempt + 1), 120)
|
||
print(f" {d} 拉取异常（{str(e)[:80]}），{wait}s 后重试 {attempt + 1}/5")
|
||
time.sleep(wait)
|
||
if df is None:
|
||
print(f" {d} 连续 5 次失败，跳过（断点续跑可补）")
|
||
continue
|
||
if df.empty:
|
||
continue
|
||
|
||
syms: list[str] = []
|
||
vals: list[float] = []
|
||
for _, r in df.iterrows():
|
||
tr = r["turnover_rate"]
|
||
if tr is None or tr != tr: # None / NaN
|
||
continue
|
||
syms.append(str(r["ts_code"]).split(".")[0])
|
||
vals.append(float(tr))
|
||
if not syms:
|
||
continue
|
||
n = await conn.execute(
|
||
"UPDATE candles AS c SET turnover = v.t "
|
||
"FROM unnest($1::text[], $2::float8[]) AS v(sym, t) "
|
||
"WHERE c.symbol = v.sym AND c.timeframe = '1d' AND c.ts = $3::timestamp",
|
||
syms, vals, datetime.strptime(d, "%Y%m%d"),
|
||
)
|
||
done += 1
|
||
if done % _INTERVAL_MSG == 0 or done == len(todo):
|
||
elapsed = time.time() - t0
|
||
eta = elapsed / done * (len(todo) - done) if done else 0
|
||
print(f" 进度 {done}/{len(todo)}（{d}，{len(syms)} 只，{n}），"
|
||
f"{elapsed:.0f}s 已用，预计还需 {eta/60:.0f}m")
|
||
print(f"回补完成：{done} 个交易日")
|
||
finally:
|
||
await conn.close()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
ap = argparse.ArgumentParser(description="全量回补换手率 candles.turnover")
|
||
ap.add_argument("--start", default=None, help="YYYYMMDD，默认 max(candles 最早, 20000101)")
|
||
ap.add_argument("--end", default=None, help="YYYYMMDD，默认今天")
|
||
ap.add_argument("--force", action="store_true", help="已有换手的交易日也重拉")
|
||
a = ap.parse_args()
|
||
asyncio.run(main(a.start, a.end, a.force))
|