cirry/stock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

提交

Browse Source
This commit is contained in:
cirry
2026-09-09 11:35:02 +08:00
parent bc1c72d558
commit d656c05b3d
35 changed files with 711 additions and 3194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
backend/app/screener/market_sync.py
View File

@@ -10,6 +10,7 @@ daily 与 daily_basic 分步独立落库：daily_basic 积分不足时快照仍
from __future__ import annotations
import asyncio
import logging
import time
from datetime import datetime, timedelta
@@ -20,9 +21,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
from .. import cache
from ..config import settings
from ..data.symbols import plain_code
from ..data.sync_utils import call_retry, get_pro_lazy
from ..models import AdjFactor, Candle, DailySnapshot, StockBasic, TradeCalendar
from .llm import ScreenerError
log = logging.getLogger(__name__)
# 进程内单例任务状态（uvicorn --reload 单进程场景够用）
_sync_state: dict = {
"running": False,
@@ -40,32 +44,6 @@ _BATCH = 5000 # executemany 分批行数
# Tushare 积分/权限不足的特征文案（daily_basic 常见门槛）
_PERM_MARKS = ("抱歉，您没有访问该项目权限", "积分", "权限")
# 频率超限特征（等待 62s 重试一次）
_RATE_MARKS = ("频率超限", "每分钟")
def _call_retry(fn, *args, **kwargs):
"""同步调用 tushare 接口；「每分钟」级频率超限等 62s 重试一次（小时级限频直接抛）。"""
try:
return fn(*args, **kwargs)
except Exception as e: # noqa: BLE001
msg = str(e)
if any(m in msg for m in _RATE_MARKS) and "小时" not in msg:
time.sleep(62)
return fn(*args, **kwargs)
raise
def _get_pro():
"""token 检查 + 返回 pro api 客户端（同步对象，调用需 to_thread 包裹）。
经 tushare_provider.get_pro 统一走镜像补丁（15000 积分档 token 只认 quicksync）。
"""
if not settings.tushare_token:
raise ScreenerError("未配置 TUSHARE_TOKEN，无法同步全市场数据（backend/.env）")
from ..data.tushare_provider import get_pro
return get_pro()
def _parse_d(s: str) -> datetime:
@@ -77,7 +55,7 @@ def _fetch_calendar_sync(pro) -> list[str]:
time.sleep(settings.screener_sync_interval)
end = (datetime.now() + timedelta(days=90)).strftime("%Y%m%d")
start = (datetime.now() - timedelta(days=550)).strftime("%Y%m%d")
cal = _call_retry(pro.trade_cal, exchange="SSE", start_date=start, end_date=end, is_open="1")
cal = call_retry(pro.trade_cal, exchange="SSE", start_date=start, end_date=end, is_open="1")
return sorted(cal["cal_date"].tolist())
@@ -112,7 +90,7 @@ async def _recent_trade_dates(session: AsyncSession, pro, days: int) -> list[str
def _fetch_daily(pro, d: str) -> list[dict]:
"""拉取某交易日全市场日线（未复权）。当日数据未生成（盘前/盘中）返回空。"""
time.sleep(settings.screener_sync_interval)
df = _call_retry(pro.daily, trade_date=d)
df = call_retry(pro.daily, trade_date=d)
if df is None or df.empty:
return []
rows = []
@@ -138,7 +116,7 @@ def _fetch_basic(pro, d: str) -> list[dict]:
"""
time.sleep(settings.screener_sync_interval)
try:
df = _call_retry(pro.daily_basic, trade_date=d)
df = call_retry(pro.daily_basic, trade_date=d)
except Exception as e: # noqa: BLE001
msg = str(e)
if any(m in msg for m in _PERM_MARKS):
@@ -169,7 +147,7 @@ def _fetch_basic(pro, d: str) -> list[dict]:
def _fetch_adj_factor(pro, d: str) -> list[dict]:
"""拉取某交易日全市场复权因子（K线 bfq->qfq/hfq 本地换算的底座）。"""
time.sleep(settings.screener_sync_interval)
df = _call_retry(pro.adj_factor, trade_date=d)
df = call_retry(pro.adj_factor, trade_date=d)
if df is None or df.empty:
return []
return [
@@ -181,7 +159,7 @@ def _fetch_adj_factor(pro, d: str) -> list[dict]:
def _sync_stock_list_sync(pro) -> list[dict]:
"""拉取在市股票列表。"""
time.sleep(settings.screener_sync_interval)
df = _call_retry(pro.stock_basic, exchange="", list_status="L",
df = call_retry(pro.stock_basic, exchange="", list_status="L",
fields="ts_code,symbol,name,area,industry,market,exchange,list_status,list_date,delist_date")
rows = []
for _, r in df.iterrows():
@@ -232,6 +210,27 @@ async def _existing_candle_dates(session: AsyncSession) -> set[str]:
return {r[0].strftime("%Y%m%d") for r in res if r[0] is not None}
_UPSERT_CHUNK = 3000 # 单语句行数（asyncpg 参数上限拆批）
async def _recent_day_counts(session: AsyncSession, dates: list[str]) -> dict[str, int]:
"""指定交易日在市股票的 candles 行数（半日数据自愈用）。
单条 GROUP BY 走 ts 索引范围扫，窗口 ≤15 日、代价可忽略。
"""
if not dates:
return {}
lo = _parse_d(min(dates))
hi = _parse_d(max(dates)) + timedelta(days=1)
rows = (await session.execute(
select(func.date(Candle.ts), func.count())
.where(Candle.timeframe == "1d", Candle.ts >= lo, Candle.ts < hi,
Candle.symbol.in_(select(StockBasic.symbol).where(StockBasic.list_status == "L")))
.group_by(func.date(Candle.ts))
)).all()
return {r[0].strftime("%Y%m%d"): int(r[1]) for r in rows if r[0] is not None}
async def _upsert_candle_day(session: AsyncSession, rows: list[dict], listed: set[str], d_str: str) -> None:
"""把某交易日全市场日线 upsert 进 candles（不复权底座，幂等）。
@@ -253,9 +252,10 @@ async def _upsert_candle_day(session: AsyncSession, rows: list[dict], listed: se
if not batch:
return
# on_conflict 语句整批渲染为占位符（非 executemany），asyncpg 单语句参数上限 32766，
# 10 列 x 3000 行 = 30000 参数留出余量
for i in range(0, len(batch), 3000):
stmt = pg_insert(Candle).values(batch[i : i + 3000])
# 10 列 x 3000 行 = 30000 参数留出余量。分批只拆语句，commit 在循环外 ——
# 单日一事务：写一半崩溃整日回滚，该日期语义上「未同步」，下次自然重拉（不留半日数据）
for i in range(0, len(batch), _UPSERT_CHUNK):
stmt = pg_insert(Candle).values(batch[i : i + _UPSERT_CHUNK])
stmt = stmt.on_conflict_do_update(
index_elements=["symbol", "timeframe", "ts"],
set_={
@@ -266,7 +266,7 @@ async def _upsert_candle_day(session: AsyncSession, rows: list[dict], listed: se
},
)
await session.execute(stmt)
await session.commit()
await session.commit()
async def _run_sync(days: int, force: bool) -> None:
@@ -277,7 +277,7 @@ async def _run_sync(days: int, force: bool) -> None:
from ..db import async_session # 延迟导入避免循环
try:
pro = await asyncio.to_thread(_get_pro)
pro = await asyncio.to_thread(get_pro_lazy)
# 1) 股票列表（已有数据则跳过——stock_basic 低积分版限频 1 次/小时）
async with async_session() as session:
@@ -307,6 +307,15 @@ async def _run_sync(days: int, force: bool) -> None:
select(StockBasic.symbol).where(StockBasic.list_status == "L")
)).scalars()
)
if not force:
# 半日数据自愈（修单日原子化之前的历史残留）：写入中途崩溃的日期
# 行数 ≈ 1 批（3000），显著低于完整日（~5300）。只查最近 15 个交易日
# ——实际风险区且窗口内上市数变化 <2%，0.7 阈值安全；更老的日期不查
# （上市数变化会误判，历史缺口本就由 TDX 底座兜底）。
counts = await _recent_day_counts(session, [d for d in dates[:15] if d in have_daily])
if counts:
floor = max(_UPSERT_CHUNK + 1, int(max(counts.values()) * 0.7))
have_daily -= {d for d, n in counts.items() if n < floor}
todo = [d for d in dates if d not in have_daily]
_sync_state["total_days"] = len(todo)
_sync_state["done_days"] = 0
@@ -352,7 +361,7 @@ async def _run_sync(days: int, force: bool) -> None:
try:
await _refresh_stats(await cache.get_version("candles"))
except Exception: # noqa: BLE001 —— 预热失败只影响统计数字的新鲜度
pass
log.warning("统计缓存预热失败（下轮轮询会 SWR 重算）", exc_info=True)
_sync_state["step"] = "同步完成"
except Exception as e: # noqa: BLE001
_sync_state["error"] = f"同步失败：{str(e)[:300]}"
@@ -414,16 +423,9 @@ async def _heavy_stats(session: AsyncSession) -> dict:
async def _store_stats(ver: int, data: dict) -> None:
_status_stats_cache.update(at=time.time(), ver=ver, data=data)
# 写 Redis 后台执行，失败由 cache 层静默降级，不拖慢调用方
tasks = [
asyncio.create_task(cache.cache_set(
f"syncstats:v{ver}", data, ttl=settings.sync_stats_redis_ttl)),
asyncio.create_task(cache.cache_set(
_STATS_LAST_KEY, data, ttl=settings.sync_stats_redis_ttl)),
]
_stats_bg_tasks.update(tasks)
for t in tasks:
t.add_done_callback(_stats_bg_tasks.discard)
# 写 Redis 后台执行（cache.set_bg 挂全局集合防 GC），失败由 cache 层静默降级
cache.set_bg(f"syncstats:v{ver}", data, ttl=settings.sync_stats_redis_ttl)
cache.set_bg(_STATS_LAST_KEY, data, ttl=settings.sync_stats_redis_ttl)
async def _refresh_stats(ver: int) -> None: