提交
This commit is contained in:
1488
backend/app/api.py
1488
backend/app/api.py
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,8 @@
|
||||
"""只登录、不注册的鉴权 API。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections import deque
|
||||
from datetime import timedelta
|
||||
|
||||
from fastapi import APIRouter, Cookie, Depends, HTTPException, Request, Response, status
|
||||
@@ -49,6 +51,33 @@ def clear_session_cookie(response: Response) -> None:
|
||||
)
|
||||
|
||||
|
||||
# ---------- 登录按 IP 限速(进程内滑动窗口,无外部依赖) ----------
|
||||
# 补充账户级锁定(auth_max_failed_logins):本层拦多 IP 分布爆破,也稀释
|
||||
# 「故意输错 5 次锁死他人账户」的滥用面。反代部署时 host 是代理 IP,需改读 X-Forwarded-For。
|
||||
_LOGIN_WINDOW = 60.0
|
||||
_LOGIN_MAX_PER_WINDOW = 15
|
||||
_login_attempts: dict[str, deque[float]] = {}
|
||||
_login_gc_at = 0.0
|
||||
|
||||
|
||||
def _login_rate_limited(ip: str) -> bool:
|
||||
"""超限返回 True;未超限记录本次尝试(成功失败都计)。"""
|
||||
global _login_gc_at
|
||||
now = time.monotonic()
|
||||
q = _login_attempts.setdefault(ip, deque())
|
||||
while q and q[0] <= now - _LOGIN_WINDOW:
|
||||
q.popleft()
|
||||
if len(q) >= _LOGIN_MAX_PER_WINDOW:
|
||||
return True
|
||||
q.append(now)
|
||||
if now - _login_gc_at > 3600: # 顺手回收陈旧 entry,防长跑内存增长
|
||||
_login_gc_at = now
|
||||
stale = now - _LOGIN_WINDOW * 10
|
||||
for k in [k for k, v in _login_attempts.items() if not v or v[-1] <= stale]:
|
||||
del _login_attempts[k]
|
||||
return False
|
||||
|
||||
|
||||
@router.post("/login", response_model=LoginResponse)
|
||||
async def login(
|
||||
payload: LoginRequest,
|
||||
@@ -56,6 +85,10 @@ async def login(
|
||||
response: Response,
|
||||
db: AsyncSession = Depends(get_session),
|
||||
) -> LoginResponse:
|
||||
ip = request.client.host if request.client else "?"
|
||||
if _login_rate_limited(ip):
|
||||
raise HTTPException(status_code=429, detail="登录尝试过于频繁,请稍后再试")
|
||||
|
||||
now = utcnow()
|
||||
username = payload.username.strip()
|
||||
user = (await db.execute(select(User).where(User.username == username))).scalar_one_or_none()
|
||||
|
||||
@@ -11,6 +11,7 @@ match=all(连续满足)/any(曾经满足),多条件之间取 AND。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import date, datetime, timedelta
|
||||
|
||||
import numpy as np
|
||||
@@ -120,6 +121,69 @@ def _stats_block(trades: list[dict]) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _scan_batch(
|
||||
candle_rows: list,
|
||||
code_by_symbol: dict[str, str],
|
||||
f_map: dict,
|
||||
name_map: dict[str, str],
|
||||
spec: EventBacktestSpec,
|
||||
start_ts: datetime,
|
||||
trades_limit: int,
|
||||
) -> list[dict]:
|
||||
"""纯 CPU:一批 bar -> 交易明细(放线程池跑,不阻塞事件循环)。"""
|
||||
trades: list[dict] = []
|
||||
if not candle_rows:
|
||||
return trades
|
||||
bars = pd.DataFrame(
|
||||
candle_rows, columns=["symbol", "ts", "open", "high", "low", "close"]
|
||||
)
|
||||
for symbol, g in bars.groupby("symbol", sort=False):
|
||||
if len(g) < 30:
|
||||
continue
|
||||
g = g.reset_index(drop=True)
|
||||
ts_code_l = code_by_symbol[symbol]
|
||||
cache: dict = {"_families": set()}
|
||||
mask = _signal_mask(g, spec, cache)
|
||||
if not mask.any():
|
||||
continue
|
||||
for sig_i in np.flatnonzero(mask.to_numpy()):
|
||||
ts_sig = g.at[sig_i, "ts"]
|
||||
# 信号必须落在回测窗口内(buffer 区只用于指标配热)
|
||||
if ts_sig < start_ts:
|
||||
continue
|
||||
ie = _entry_exit_indices(int(sig_i), spec, len(g))
|
||||
if ie is None:
|
||||
continue
|
||||
entry_i, exit_i = ie
|
||||
e_row, x_row = g.iloc[entry_i], g.iloc[exit_i]
|
||||
e_price = _price_at(e_row, "open" if spec.entry_timing == "next_open" else "close")
|
||||
x_price = _price_at(x_row, "open" if spec.exit_timing == "open" else "close")
|
||||
if not e_price or not x_price:
|
||||
continue
|
||||
f_in = f_map.get((ts_code_l, e_row["ts"].date()), 1.0)
|
||||
f_out = f_map.get((ts_code_l, x_row["ts"].date()), 1.0)
|
||||
ret_pct = (x_price * f_out) / (e_price * f_in) * 100 - 100
|
||||
trades.append({
|
||||
"ts_code": ts_code_l,
|
||||
"name": name_map.get(ts_code_l),
|
||||
"entry_date": e_row["ts"], "entry_price": round(e_price, 3),
|
||||
"exit_date": x_row["ts"], "exit_price": round(x_price, 3),
|
||||
"ret_pct": round(float(ret_pct), 3),
|
||||
})
|
||||
if len(trades) >= trades_limit:
|
||||
return trades
|
||||
return trades
|
||||
|
||||
|
||||
def _summarize(trades: list[dict]) -> tuple[dict, list[dict]]:
|
||||
"""纯 CPU:汇总统计 + 最好/最差样本(同样下线程池)。"""
|
||||
stats = _stats_block(trades)
|
||||
# 明细样本:最好 100 + 最差 100(其余统计已覆盖)
|
||||
trades_sorted = sorted(trades, key=lambda t: t["ret_pct"], reverse=True)
|
||||
sample = trades_sorted[:100] + (trades_sorted[-100:] if len(trades_sorted) > 100 else [])
|
||||
return stats, sample
|
||||
|
||||
|
||||
async def run_event_backtest(
|
||||
session: AsyncSession,
|
||||
spec: EventBacktestSpec,
|
||||
@@ -195,53 +259,15 @@ async def run_event_backtest(
|
||||
)).all()
|
||||
f_map = {(r[0], r[1].date()): float(r[2]) for r in adj_rows if r[2]}
|
||||
|
||||
bars = pd.DataFrame(
|
||||
candle_rows, columns=["symbol", "ts", "open", "high", "low", "close"]
|
||||
)
|
||||
for symbol, g in bars.groupby("symbol", sort=False):
|
||||
if len(g) < 30:
|
||||
continue
|
||||
g = g.reset_index(drop=True)
|
||||
ts_code_l = code_by_symbol[symbol]
|
||||
cache: dict = {"_families": set()}
|
||||
mask = _signal_mask(g, spec, cache)
|
||||
if not mask.any():
|
||||
continue
|
||||
for sig_i in np.flatnonzero(mask.to_numpy()):
|
||||
ts_sig = g.at[sig_i, "ts"]
|
||||
# 信号必须落在回测窗口内(buffer 区只用于指标配热)
|
||||
if ts_sig < start_ts:
|
||||
continue
|
||||
ie = _entry_exit_indices(int(sig_i), spec, len(g))
|
||||
if ie is None:
|
||||
continue
|
||||
entry_i, exit_i = ie
|
||||
e_row, x_row = g.iloc[entry_i], g.iloc[exit_i]
|
||||
e_price = _price_at(e_row, "open" if spec.entry_timing == "next_open" else "close")
|
||||
x_price = _price_at(x_row, "open" if spec.exit_timing == "open" else "close")
|
||||
if not e_price or not x_price:
|
||||
continue
|
||||
f_in = f_map.get((ts_code_l, e_row["ts"].date()), 1.0)
|
||||
f_out = f_map.get((ts_code_l, x_row["ts"].date()), 1.0)
|
||||
ret_pct = (x_price * f_out) / (e_price * f_in) * 100 - 100
|
||||
trades.append({
|
||||
"ts_code": ts_code_l,
|
||||
"name": name_map.get(ts_code_l),
|
||||
"entry_date": e_row["ts"], "entry_price": round(e_price, 3),
|
||||
"exit_date": x_row["ts"], "exit_price": round(x_price, 3),
|
||||
"ret_pct": round(float(ret_pct), 3),
|
||||
})
|
||||
if len(trades) >= MAX_TRADES:
|
||||
break
|
||||
if len(trades) >= MAX_TRADES:
|
||||
break
|
||||
# pandas 全市场扫描是同步 CPU 重计算,丢线程池跑(await 期间事件循环可服务其他请求)
|
||||
trades.extend(await asyncio.to_thread(
|
||||
_scan_batch, candle_rows, code_by_symbol, f_map, name_map,
|
||||
spec, start_ts, MAX_TRADES - len(trades),
|
||||
))
|
||||
if len(trades) >= MAX_TRADES:
|
||||
break
|
||||
|
||||
stats = _stats_block(trades)
|
||||
# 明细样本:最好 100 + 最差 100(其余统计已覆盖)
|
||||
trades_sorted = sorted(trades, key=lambda t: t["ret_pct"], reverse=True)
|
||||
sample = trades_sorted[:100] + (trades_sorted[-100:] if len(trades_sorted) > 100 else [])
|
||||
stats, sample = await asyncio.to_thread(_summarize, trades)
|
||||
return {
|
||||
"spec": spec,
|
||||
"universe": ts_code or "all",
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
"""Redis 读缓存(可选基础设施)。
|
||||
|
||||
- REDIS_URL 留空、连接失败或超时:所有操作静默退化为「无缓存」,接口照常直查数据库,
|
||||
且本进程内禁用重试(避免每个请求都陪跑一次连接超时)。
|
||||
- REDIS_URL 留空、连接失败或超时:所有操作静默退化为「无缓存」,接口照常直查数据库。
|
||||
故障后熔断 _RECOVERY_SECONDS(期间所有请求直连,不陪跑连接超时),到期自动放行
|
||||
一次探测——成功即完全恢复,仍失败则重新熔断(Redis 属加速件,坏了不能拖慢接口)。
|
||||
- 失效策略:TTL 自然过期 + 版本号(INCR)作废。自选股增删等写操作只 INCR 版本 key,
|
||||
旧缓存 key 里带着旧版本号,无需 SCAN 批量删除。
|
||||
- 只缓存「读多写少、可容忍短暂陈旧」的聚合数据(股票列表、筛选项等);
|
||||
@@ -13,6 +14,7 @@
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
@@ -23,12 +25,13 @@ import redis.asyncio as aioredis
|
||||
from .config import settings
|
||||
|
||||
_pool: aioredis.ConnectionPool | None = None
|
||||
_disabled = False # 一次失败后本进程禁用(Redis 属加速件,坏了不能拖慢接口)
|
||||
_RECOVERY_SECONDS = 60.0 # 熔断时长:期间直连不试 Redis,到期放行一次探测
|
||||
_disabled_until = 0.0 # 熔断截止的 monotonic 时刻;0 = 未熔断
|
||||
|
||||
|
||||
def _client() -> aioredis.Redis | None:
|
||||
global _pool, _disabled
|
||||
if not settings.redis_url or _disabled:
|
||||
global _pool
|
||||
if not settings.redis_url or time.monotonic() < _disabled_until:
|
||||
return None
|
||||
if _pool is None:
|
||||
_pool = aioredis.ConnectionPool.from_url(
|
||||
@@ -43,8 +46,8 @@ def _client() -> aioredis.Redis | None:
|
||||
|
||||
|
||||
def _bail() -> None:
|
||||
global _disabled
|
||||
_disabled = True
|
||||
global _disabled_until
|
||||
_disabled_until = time.monotonic() + _RECOVERY_SECONDS
|
||||
|
||||
|
||||
def digest(*parts: Any) -> str:
|
||||
@@ -84,7 +87,21 @@ def local_set(key: str, raw: str, ttl: int) -> None:
|
||||
_local_store[key] = (time.monotonic() + max(1, min(ttl, 120)), raw)
|
||||
_local_bytes += len(raw) + 64 # 连同 dict/tuple 开销粗略计入
|
||||
while _local_store and (len(_local_store) > _LOCAL_MAX_ENTRIES or _local_bytes > _LOCAL_MAX_BYTES):
|
||||
_local_bytes -= len(_local_store.popitem(last=False)[1][1]) + 64
|
||||
# dict 是插入序:next(iter(...)) 即最旧键(dict.popitem 不支持参数,别写成 OrderedDict 的写法)
|
||||
oldest = next(iter(_local_store))
|
||||
_local_bytes -= len(_local_store.pop(oldest)[1]) + 64
|
||||
|
||||
|
||||
# --- 后台写(fire-and-forget)------------------------------------------------
|
||||
# 读路径拿到响应后异步写 Redis、不阻塞返回。统一入口:任务挂全局集合防 GC,
|
||||
# cache_set 内部自带异常静默(缓存层尽力而为),优雅停机丢最后一次写无害(TTL 兜底)。
|
||||
_bg_tasks: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
def set_bg(key: str, value: Any, ttl: int) -> None:
|
||||
t = asyncio.create_task(cache_set(key, value, ttl))
|
||||
_bg_tasks.add(t)
|
||||
t.add_done_callback(_bg_tasks.discard)
|
||||
|
||||
|
||||
# --- 版本号本地缓存:热请求连 Redis GET ver:xx 都省掉 ------------------------
|
||||
|
||||
@@ -56,6 +56,10 @@ class Settings(BaseSettings):
|
||||
screener_default_limit: int = 200 # 选股结果条数上限
|
||||
screener_sync_interval: float = 0.35 # 全市场批量调用间隔(秒),Tushare 控频
|
||||
|
||||
# ---- 夜间定时任务(收盘后自动同步 + 会话清理;见 app/scheduler.py)----
|
||||
nightly_sync_enabled: bool = True
|
||||
nightly_sync_hour: int = 18 # 本地时间整点,触发在 :05(收盘后日线已生成)
|
||||
|
||||
# A股交易成本(基准日 2026-08)——做成可配置参数,便于将来按生效日期版本化
|
||||
stamp_duty_rate: float = 0.0005 # 印花税 0.05%,单边卖出(2023-08-28 减半)
|
||||
transfer_fee_rate: float = 0.00001 # 过户费 0.001%,沪深双边(2022 调整)
|
||||
|
||||
@@ -9,7 +9,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
@@ -18,12 +18,10 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from ..config import settings
|
||||
from ..db import async_session
|
||||
from ..models import StockCompany
|
||||
from .sync_utils import call_retry, f_clean, fresh, get_pro_lazy, s_clean, utcnow
|
||||
|
||||
_REFRESH_DAYS = 30
|
||||
|
||||
# 频率超限特征(等待 62s 重试一次;与 screener.market_sync / data.etf_sync._call_retry 同款语义)
|
||||
_RATE_MARKS = ("频率超限", "每分钟")
|
||||
|
||||
# 显式列出全部字段:introduction/office/main_business/business_scope 文档标注默认不显示,
|
||||
# 不传 fields 时 tushare 不返回这四列(实测 000001.SZ)
|
||||
_FIELDS = (
|
||||
@@ -32,87 +30,38 @@ _FIELDS = (
|
||||
"employees,main_business,business_scope"
|
||||
)
|
||||
|
||||
_pro = None # 惰性单例(get_pro 每次都 ts.set_token 写文件,没必要重复)
|
||||
|
||||
|
||||
def _get_pro():
|
||||
if not settings.tushare_token:
|
||||
raise RuntimeError("未配置 TUSHARE_TOKEN,无法拉取公司简介(backend/.env)")
|
||||
global _pro
|
||||
if _pro is None:
|
||||
from .tushare_provider import get_pro
|
||||
|
||||
_pro = get_pro()
|
||||
return _pro
|
||||
|
||||
|
||||
def _call_retry(fn, *args, **kwargs):
|
||||
"""同步调用 tushare 接口;「每分钟」级频率超限等 62s 重试一次。"""
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
except Exception as e: # noqa: BLE001
|
||||
msg = str(e)
|
||||
if any(m in msg for m in _RATE_MARKS) and "小时" not in msg:
|
||||
time.sleep(62)
|
||||
return fn(*args, **kwargs)
|
||||
raise
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
|
||||
def _fresh(updated_at: datetime | None) -> bool:
|
||||
return updated_at is not None and updated_at >= _utcnow() - timedelta(days=_REFRESH_DAYS)
|
||||
|
||||
|
||||
def _s(v) -> str | None:
|
||||
"""pandas NaN / 空串 / None -> None,其余 strip。"""
|
||||
if v is None or (isinstance(v, float) and v != v):
|
||||
return None
|
||||
s = str(v).strip()
|
||||
return s or None
|
||||
|
||||
|
||||
def _f(v) -> float | None:
|
||||
try:
|
||||
f = float(v)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return None if f != f else f # NaN -> None
|
||||
|
||||
|
||||
def _i(v) -> int | None:
|
||||
f = _f(v)
|
||||
f = f_clean(v)
|
||||
return None if f is None else int(f)
|
||||
|
||||
|
||||
def fetch_company_sync(ts_code: str) -> dict | None:
|
||||
"""同步拉单只公司简介(需在 to_thread 里跑);返回行 dict,无此股返回 None。"""
|
||||
time.sleep(settings.screener_sync_interval)
|
||||
df = _call_retry(_get_pro().stock_company, ts_code=ts_code, fields=_FIELDS)
|
||||
df = call_retry(get_pro_lazy().stock_company, ts_code=ts_code, fields=_FIELDS)
|
||||
if df is None or df.empty:
|
||||
return None
|
||||
r = df.iloc[0]
|
||||
return {
|
||||
"ts_code": ts_code,
|
||||
"com_name": _s(r.get("com_name")),
|
||||
"com_id": _s(r.get("com_id")),
|
||||
"chairman": _s(r.get("chairman")),
|
||||
"manager": _s(r.get("manager")),
|
||||
"secretary": _s(r.get("secretary")),
|
||||
"reg_capital": _f(r.get("reg_capital")),
|
||||
"setup_date": _s(r.get("setup_date")),
|
||||
"province": _s(r.get("province")),
|
||||
"city": _s(r.get("city")),
|
||||
"introduction": _s(r.get("introduction")),
|
||||
"website": _s(r.get("website")),
|
||||
"email": _s(r.get("email")),
|
||||
"office": _s(r.get("office")),
|
||||
"com_name": s_clean(r.get("com_name")),
|
||||
"com_id": s_clean(r.get("com_id")),
|
||||
"chairman": s_clean(r.get("chairman")),
|
||||
"manager": s_clean(r.get("manager")),
|
||||
"secretary": s_clean(r.get("secretary")),
|
||||
"reg_capital": f_clean(r.get("reg_capital")),
|
||||
"setup_date": s_clean(r.get("setup_date")),
|
||||
"province": s_clean(r.get("province")),
|
||||
"city": s_clean(r.get("city")),
|
||||
"introduction": s_clean(r.get("introduction")),
|
||||
"website": s_clean(r.get("website")),
|
||||
"email": s_clean(r.get("email")),
|
||||
"office": s_clean(r.get("office")),
|
||||
"employees": _i(r.get("employees")),
|
||||
"main_business": _s(r.get("main_business")),
|
||||
"business_scope": _s(r.get("business_scope")),
|
||||
"updated_at": _utcnow(),
|
||||
"main_business": s_clean(r.get("main_business")),
|
||||
"business_scope": s_clean(r.get("business_scope")),
|
||||
"updated_at": utcnow(),
|
||||
}
|
||||
|
||||
|
||||
@@ -151,7 +100,7 @@ async def get_company(session: AsyncSession, ts_code: str) -> dict | None:
|
||||
"""
|
||||
row = (await session.execute(
|
||||
select(StockCompany).where(StockCompany.ts_code == ts_code))).scalar_one_or_none()
|
||||
if row is not None and _fresh(row.updated_at):
|
||||
if row is not None and fresh(row.updated_at, _REFRESH_DAYS):
|
||||
return _row_dict(row) if row.com_name is not None else None # 墓碑 -> None
|
||||
# 释放请求会话持有的连接:后面可能隔着 1-2s 的 tushare 调用,别长占连接池。
|
||||
# 用 close() 而非 rollback():rollback 会把会话身份映射里的实例全部 expire——
|
||||
@@ -165,7 +114,7 @@ async def get_company(session: AsyncSession, ts_code: str) -> dict | None:
|
||||
async with async_session() as s2: # 锁内重读 + 写入走新会话
|
||||
row = (await s2.execute(
|
||||
select(StockCompany).where(StockCompany.ts_code == ts_code))).scalar_one_or_none()
|
||||
if row is not None and _fresh(row.updated_at):
|
||||
if row is not None and fresh(row.updated_at, _REFRESH_DAYS):
|
||||
return _row_dict(row) if row.com_name is not None else None
|
||||
try:
|
||||
fetched = await asyncio.to_thread(fetch_company_sync, ts_code)
|
||||
@@ -174,5 +123,5 @@ async def get_company(session: AsyncSession, ts_code: str) -> dict | None:
|
||||
if row is not None and row.com_name is not None:
|
||||
return _row_dict(row)
|
||||
raise
|
||||
await _upsert(s2, fetched or {"ts_code": ts_code, "updated_at": _utcnow()})
|
||||
await _upsert(s2, fetched or {"ts_code": ts_code, "updated_at": utcnow()})
|
||||
return fetched
|
||||
|
||||
@@ -20,7 +20,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy import delete, func, select, text
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
@@ -29,6 +29,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from .. import cache
|
||||
from ..config import settings
|
||||
from . import etf_provider
|
||||
from .sync_utils import call_retry, get_pro_lazy, utcnow
|
||||
|
||||
# 进程内单例任务状态(uvicorn 单进程场景够用)
|
||||
_state: dict = {
|
||||
@@ -46,34 +47,6 @@ _lock = asyncio.Lock()
|
||||
_BATCH = 3000 # upsert 分批行数(asyncpg 单语句参数上限 32766,10 列/行)
|
||||
# fund_daily 返回全市场基金 ~2100 行,一天一批远小于上限
|
||||
|
||||
# 频率超限特征(等待 62s 重试一次;与 screener.market_sync._call_retry 同款语义)
|
||||
_RATE_MARKS = ("频率超限", "每分钟")
|
||||
|
||||
|
||||
def _call_retry(fn, *args, **kwargs):
|
||||
"""同步调用 tushare 接口;「每分钟」级频率超限等 62s 重试一次。"""
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
except Exception as e: # noqa: BLE001
|
||||
msg = str(e)
|
||||
if any(m in msg for m in _RATE_MARKS) and "小时" not in msg:
|
||||
time.sleep(62)
|
||||
return fn(*args, **kwargs)
|
||||
raise
|
||||
|
||||
|
||||
def _get_pro():
|
||||
"""token 检查 + 返回 pro api 客户端(同步对象,调用需 to_thread 包裹)。"""
|
||||
if not settings.tushare_token:
|
||||
raise RuntimeError("未配置 TUSHARE_TOKEN,无法同步 ETF 日线(backend/.env)")
|
||||
from .tushare_provider import get_pro
|
||||
|
||||
return get_pro()
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
|
||||
def _parse_d(s: str) -> datetime:
|
||||
return datetime.strptime(str(s), "%Y%m%d")
|
||||
@@ -85,7 +58,7 @@ async def _sync_spot(session: AsyncSession) -> int:
|
||||
|
||||
async with etf_provider.new_client() as client:
|
||||
rows = await etf_provider.fetch_etf_spot(client)
|
||||
now = _utcnow()
|
||||
now = utcnow()
|
||||
stmt = pg_insert(EtfBasic).values([{**r, "updated_at": now} for r in rows])
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=["ts_code"],
|
||||
@@ -108,7 +81,7 @@ async def _sync_spot(session: AsyncSession) -> int:
|
||||
def _fetch_day_sync(pro, d: str) -> list[dict]:
|
||||
"""拉某交易日全市场场内基金日线(fund_daily;未生成的日期返回空)。"""
|
||||
time.sleep(settings.screener_sync_interval)
|
||||
df = _call_retry(pro.fund_daily, trade_date=d)
|
||||
df = call_retry(pro.fund_daily, trade_date=d)
|
||||
if df is None or df.empty:
|
||||
return []
|
||||
rows = []
|
||||
@@ -128,7 +101,7 @@ def _fetch_day_sync(pro, d: str) -> list[dict]:
|
||||
def _fetch_symbol_sync(pro, ts_code: str, start: str | None, end: str | None) -> list[dict]:
|
||||
"""按 ts_code 增量/全量拉单只 ETF 日线(start=None 即上市以来全量)。"""
|
||||
time.sleep(settings.screener_sync_interval)
|
||||
df = _call_retry(pro.fund_daily, ts_code=ts_code, start_date=start, end_date=end)
|
||||
df = call_retry(pro.fund_daily, ts_code=ts_code, start_date=start, end_date=end)
|
||||
if df is None or df.empty:
|
||||
return []
|
||||
df = df.sort_values("trade_date")
|
||||
@@ -209,7 +182,7 @@ async def _run_sync(full: bool) -> None:
|
||||
from ..models import Candle, EtfBasic, TradeCalendar
|
||||
|
||||
try:
|
||||
pro = await asyncio.to_thread(_get_pro)
|
||||
pro = await asyncio.to_thread(get_pro_lazy)
|
||||
|
||||
# 1) 快照 -> etf_basic
|
||||
_state["step"] = "正在拉取 ETF 列表"
|
||||
|
||||
@@ -113,6 +113,9 @@ async def sync_symbol(
|
||||
)
|
||||
await session.execute(stmt)
|
||||
await session.commit()
|
||||
# 作废 candles 相关读缓存(preview 等)——不 bump 的话旧版本号的缓存要等 TTL 自然过期
|
||||
from .. import cache
|
||||
await cache.bump_version("candles")
|
||||
return {"symbol": code, "bars": len(bars), "source": used}
|
||||
|
||||
|
||||
|
||||
@@ -13,13 +13,13 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
import time
|
||||
from datetime import date, datetime, timedelta
|
||||
|
||||
from .. import cache
|
||||
from ..config import settings
|
||||
from ..domain import Bar
|
||||
from .sync_utils import d8_iso, f_clean
|
||||
|
||||
# ---- 静态元数据表(tushare index_global 支持的全部 21 个指数,展示顺序即文档顺序)----
|
||||
# region: americas 美洲 / europe 欧洲 / asia 亚太(含港股与富时A50)
|
||||
@@ -82,22 +82,6 @@ class GlobalIndexError(RuntimeError):
|
||||
"""全部国际指数都拉不到(token/网络故障)——接口层转 503。"""
|
||||
|
||||
|
||||
def _f(v) -> float | None:
|
||||
"""pandas 值 -> float;NaN/None -> None。"""
|
||||
if v is None:
|
||||
return None
|
||||
try:
|
||||
f = float(v)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return None if math.isnan(f) else f
|
||||
|
||||
|
||||
def _d(v) -> str | None:
|
||||
"""YYYYMMDD -> 'YYYY-MM-DD'(字符串便于 JSON 缓存)。"""
|
||||
return datetime.strptime(str(v), "%Y%m%d").date().isoformat() if v else None
|
||||
|
||||
|
||||
def is_cn_index(code: str) -> bool:
|
||||
return "." in code
|
||||
|
||||
@@ -128,14 +112,14 @@ def _fetch_quote_sync(pro, ts_code: str) -> dict:
|
||||
tail = df.tail(_SPARK_DAYS)
|
||||
last = df.iloc[-1]
|
||||
return {
|
||||
"close": _f(last["close"]),
|
||||
"change": _f(last.get("change")),
|
||||
"pct_chg": _f(last.get("pct_chg")),
|
||||
"open": _f(last.get("open")),
|
||||
"high": _f(last.get("high")),
|
||||
"low": _f(last.get("low")),
|
||||
"pre_close": _f(last.get("pre_close")),
|
||||
"trade_date": _d(last["trade_date"]),
|
||||
"close": f_clean(last["close"]),
|
||||
"change": f_clean(last.get("change")),
|
||||
"pct_chg": f_clean(last.get("pct_chg")),
|
||||
"open": f_clean(last.get("open")),
|
||||
"high": f_clean(last.get("high")),
|
||||
"low": f_clean(last.get("low")),
|
||||
"pre_close": f_clean(last.get("pre_close")),
|
||||
"trade_date": d8_iso(last["trade_date"]),
|
||||
"spark": [round(float(c), 4) for c in tail["close"]],
|
||||
"spark_dates": [str(d) for d in tail["trade_date"]],
|
||||
}
|
||||
@@ -249,8 +233,8 @@ def _fetch_global_bars_sync(ts_code: str) -> list[Bar]:
|
||||
df = pd.concat(frames).drop_duplicates(subset="trade_date").sort_values("trade_date")
|
||||
bars: list[Bar] = []
|
||||
for _, r in df.iterrows():
|
||||
vol = _f(r.get("vol"))
|
||||
amt = _f(r.get("amount"))
|
||||
vol = f_clean(r.get("vol"))
|
||||
amt = f_clean(r.get("amount"))
|
||||
bars.append(
|
||||
Bar(
|
||||
ts=datetime.strptime(str(r["trade_date"]), "%Y%m%d"),
|
||||
@@ -318,9 +302,9 @@ def _fetch_basic_sync(ts_code: str) -> dict:
|
||||
"market": r.get("market"),
|
||||
"publisher": r.get("publisher"),
|
||||
"category": r.get("category"),
|
||||
"base_date": _d(r.get("base_date")),
|
||||
"base_point": _f(r.get("base_point")),
|
||||
"list_date": _d(r.get("list_date")),
|
||||
"base_date": d8_iso(r.get("base_date")),
|
||||
"base_point": f_clean(r.get("base_point")),
|
||||
"list_date": d8_iso(r.get("list_date")),
|
||||
}
|
||||
|
||||
|
||||
@@ -354,10 +338,10 @@ def _fetch_valuation_sync(ts_code: str, days: int) -> list[dict]:
|
||||
rows = []
|
||||
for _, r in df.sort_values("trade_date").iterrows():
|
||||
rows.append({
|
||||
"trade_date": _d(r["trade_date"]),
|
||||
"pe": _f(r.get("pe")), "pe_ttm": _f(r.get("pe_ttm")), "pb": _f(r.get("pb")),
|
||||
"turnover_rate": _f(r.get("turnover_rate")),
|
||||
"total_mv": _f(r.get("total_mv")), "float_mv": _f(r.get("float_mv")),
|
||||
"trade_date": d8_iso(r["trade_date"]),
|
||||
"pe": f_clean(r.get("pe")), "pe_ttm": f_clean(r.get("pe_ttm")), "pb": f_clean(r.get("pb")),
|
||||
"turnover_rate": f_clean(r.get("turnover_rate")),
|
||||
"total_mv": f_clean(r.get("total_mv")), "float_mv": f_clean(r.get("float_mv")),
|
||||
})
|
||||
return rows
|
||||
|
||||
@@ -395,7 +379,7 @@ def _fetch_weights_sync(ts_code: str) -> dict | None:
|
||||
latest_date = df.iloc[0]["trade_date"]
|
||||
rows = df[df["trade_date"] == latest_date]
|
||||
return {
|
||||
"trade_date": _d(latest_date),
|
||||
"trade_date": d8_iso(latest_date),
|
||||
"total": int(len(rows)),
|
||||
"items": [
|
||||
{"con_code": str(r["con_code"]), "weight": round(float(r["weight"]), 4)}
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
import time
|
||||
from datetime import date, datetime, timedelta
|
||||
|
||||
@@ -26,6 +25,7 @@ import pandas as pd
|
||||
|
||||
from .. import cache
|
||||
from ..config import settings
|
||||
from .sync_utils import d8_iso, f_clean
|
||||
|
||||
# (tushare代码, 名称, 地区, 腾讯符号) —— 展示顺序即列表顺序
|
||||
# 首页聚焦中美(港股/国际指数在 /indexes 国际指数页);标普500 腾讯符号是 s_usINX(不是 s_usSPX)
|
||||
@@ -57,22 +57,6 @@ class MarketOverviewError(RuntimeError):
|
||||
"""所有指数都拉不到(token/网络故障)——接口层转 503。"""
|
||||
|
||||
|
||||
def _f(v) -> float | None:
|
||||
"""pandas 值 -> float;NaN/None -> None(否则 JSON 里会出现 NaN)。"""
|
||||
if v is None:
|
||||
return None
|
||||
try:
|
||||
f = float(v)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return None if math.isnan(f) else f
|
||||
|
||||
|
||||
def _d(v) -> str | None:
|
||||
"""YYYYMMDD -> 'YYYY-MM-DD'(字符串便于 JSON 缓存;pydantic 响应模型自动 coerce)。"""
|
||||
return datetime.strptime(str(v), "%Y%m%d").date().isoformat() if v else None
|
||||
|
||||
|
||||
def _get_pro():
|
||||
if not settings.tushare_token:
|
||||
raise MarketOverviewError("未配置 TUSHARE_TOKEN,无法获取大盘行情(backend/.env)")
|
||||
@@ -165,10 +149,10 @@ def _quote_from_df(df: pd.DataFrame) -> dict | None:
|
||||
tail = df.tail(_SPARK_DAYS)
|
||||
last = df.iloc[-1]
|
||||
return {
|
||||
"close": _f(last["close"]),
|
||||
"change": _f(last.get("change")),
|
||||
"pct_chg": _f(last.get("pct_chg")),
|
||||
"trade_date": _d(last["trade_date"]),
|
||||
"close": f_clean(last["close"]),
|
||||
"change": f_clean(last.get("change")),
|
||||
"pct_chg": f_clean(last.get("pct_chg")),
|
||||
"trade_date": d8_iso(last["trade_date"]),
|
||||
"spark": [round(float(c), 4) for c in tail["close"]],
|
||||
"spark_dates": [str(d) for d in tail["trade_date"]],
|
||||
}
|
||||
@@ -193,10 +177,10 @@ def _fetch_stats_sync(pro) -> dict | None:
|
||||
if sh_m is None or sz_m is None:
|
||||
return None
|
||||
# 两边各自取最新,日期不一致时以较旧一天为准凑齐口径(罕见,通常同日)
|
||||
d = min(_d(sh_m["trade_date"]), _d(sz_m["trade_date"]))
|
||||
d = min(d8_iso(sh_m["trade_date"]), d8_iso(sz_m["trade_date"]))
|
||||
|
||||
def _sum(col: str) -> float | None:
|
||||
a, b = _f(sh_m.get(col)), _f(sz_m.get(col))
|
||||
a, b = f_clean(sh_m.get(col)), f_clean(sz_m.get(col))
|
||||
return None if a is None or b is None else round(a + b, 2)
|
||||
|
||||
return {
|
||||
@@ -204,7 +188,7 @@ def _fetch_stats_sync(pro) -> dict | None:
|
||||
"total_mv": _sum("total_mv"),
|
||||
"float_mv": _sum("float_mv"),
|
||||
"amount": _sum("amount"),
|
||||
"turnover": _f(sh_m.get("tr")), # 换手率仅沪市有,展示口径注明沪市
|
||||
"turnover": f_clean(sh_m.get("tr")), # 换手率仅沪市有,展示口径注明沪市
|
||||
}
|
||||
|
||||
|
||||
@@ -222,7 +206,7 @@ def _fetch_amount_history_sync(pro) -> list[dict]:
|
||||
if len(common) == 0:
|
||||
return []
|
||||
total = (sh_m[common] + sz_m[common]).sort_index()
|
||||
return [{"date": _d(d), "amount": round(float(v), 2)} for d, v in total.tail(_AMOUNT_HIST_BARS).items()]
|
||||
return [{"date": d8_iso(d), "amount": round(float(v), 2)} for d, v in total.tail(_AMOUNT_HIST_BARS).items()]
|
||||
|
||||
|
||||
# ---- EOD 的 SWR(stale-while-revalidate):新鲜期内直返;过期先返旧值后台刷新 ----
|
||||
|
||||
@@ -16,6 +16,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import calendar
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
@@ -29,6 +30,8 @@ from ..db import async_session
|
||||
from ..models import StockReference
|
||||
from .sync_utils import call_retry, f_clean, fresh, get_pro_lazy, read_sync_state, s_clean, upsert_sync_state, utcnow
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_REFRESH_DAYS = 7
|
||||
|
||||
|
||||
@@ -394,12 +397,12 @@ async def _sync_repurchase_locked(only_current: bool) -> dict[str, list[dict]]:
|
||||
|
||||
|
||||
async def _repurchase_backfill() -> None:
|
||||
"""后台全量回填(近 24 个月);失败静默——下次触发重试。"""
|
||||
"""后台全量回填(近 24 个月);失败记录日志——下次触发重试。"""
|
||||
try:
|
||||
async with _repurchase_lock:
|
||||
await _sync_repurchase_locked(only_current=False)
|
||||
except Exception: # noqa: BLE001 后台任务无人接异常
|
||||
pass
|
||||
except Exception: # noqa: BLE001 后台任务无人接异常,至少留痕
|
||||
log.warning("回购数据后台回填失败(下次触发重试)", exc_info=True)
|
||||
|
||||
|
||||
def _spawn_repurchase_backfill() -> None:
|
||||
|
||||
@@ -68,6 +68,11 @@ def f_clean(v) -> float | None:
|
||||
return None if f != f else f # NaN -> None
|
||||
|
||||
|
||||
def d8_iso(v) -> str | None:
|
||||
"""tushare YYYYMMDD -> 'YYYY-MM-DD'(字符串便于 JSON 缓存;pydantic 自动 coerce)。"""
|
||||
return datetime.strptime(str(v), "%Y%m%d").date().isoformat() if v else None
|
||||
|
||||
|
||||
async def read_sync_state(session: AsyncSession, ts_code: str, kind: str) -> StockSyncState | None:
|
||||
return (await session.execute(
|
||||
select(StockSyncState).where(
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
"""FastAPI 入口。数据库结构统一由 Alembic 管理。"""
|
||||
from contextlib import asynccontextmanager
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from sqlalchemy import text
|
||||
|
||||
from . import cache
|
||||
from . import cache, scheduler
|
||||
from .api import router
|
||||
from .auth_api import router as auth_router
|
||||
from .config import settings
|
||||
@@ -16,7 +17,11 @@ from .db import engine
|
||||
async def lifespan(app: FastAPI):
|
||||
async with engine.connect() as conn:
|
||||
await conn.execute(text("SELECT 1"))
|
||||
nightly = asyncio.create_task(scheduler.run_nightly_loop())
|
||||
yield
|
||||
nightly.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await nightly
|
||||
await engine.dispose()
|
||||
await cache.aclose() # 释放 Redis 连接池(未启用时是 no-op)
|
||||
|
||||
|
||||
@@ -4,8 +4,8 @@ Candle 表设计与 TimescaleDB hypertable 完全兼容:将来在目标 PG 库
|
||||
SELECT create_hypertable('candles', 'ts');
|
||||
即可升级为时序表 + Continuous Aggregates 多周期预聚合,无需改表结构。
|
||||
|
||||
智能选股三表(stock_basic / market_daily / daily_snapshot)与回测 candles(qfq)
|
||||
完全隔离:选股用未复权日线按 trade_date 全市场批量落地,避免污染回测复权缓存。
|
||||
智能选股直接读 candles 不复权底座(market_daily 已退役);
|
||||
daily_snapshot 存每日指标快照(估值/市值,选股过滤用)。
|
||||
"""
|
||||
from datetime import date, datetime
|
||||
|
||||
@@ -236,29 +236,8 @@ class StockReference(Base):
|
||||
)
|
||||
|
||||
|
||||
class MarketDaily(Base):
|
||||
"""全市场未复权日线(选股专用,与回测 candles(qfq) 隔离)。
|
||||
|
||||
单位沿用 Tushare 原始:vol 手、amount 千元。
|
||||
"""
|
||||
__tablename__ = "market_daily"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
trade_date: Mapped[datetime] = mapped_column(DateTime, index=True)
|
||||
ts_code: Mapped[str] = mapped_column(String(12), index=True)
|
||||
open: Mapped[float] = mapped_column(Float)
|
||||
high: Mapped[float] = mapped_column(Float)
|
||||
low: Mapped[float] = mapped_column(Float)
|
||||
close: Mapped[float] = mapped_column(Float)
|
||||
pre_close: Mapped[float] = mapped_column(Float)
|
||||
change: Mapped[float | None] = mapped_column(Float)
|
||||
pct_chg: Mapped[float | None] = mapped_column(Float) # 日涨跌幅 %
|
||||
vol: Mapped[float] = mapped_column(Float) # 手
|
||||
amount: Mapped[float] = mapped_column(Float) # 千元
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("ts_code", "trade_date", name="uq_mkt_code_date"),
|
||||
)
|
||||
# market_daily(全市场未复权日线)已于选股改读 candles 后退役:
|
||||
# ORM 模型已删,物理表暂留库中作冷备,确认无用后可手动 DROP TABLE market_daily。
|
||||
|
||||
|
||||
class DailySnapshot(Base):
|
||||
|
||||
@@ -352,6 +352,93 @@ class StockReferenceOut(BaseModel):
|
||||
records: list[dict[str, str | float | None]] = Field(default_factory=list)
|
||||
|
||||
|
||||
# ---------- 打板专题(主页,同花顺口径) ----------
|
||||
class LimitStockOut(BaseModel):
|
||||
"""涨跌停榜单行:三池共用,涨停池字段最全(炸板/跌停池仅价格类字段)。"""
|
||||
ts_code: str
|
||||
name: str | None = None
|
||||
price: float | None = None # 收盘价(元)
|
||||
pct_chg: float | None = None # 涨跌幅 %
|
||||
tag: str | None = None # 涨停标签:首板 / 2天2板(仅涨停池)
|
||||
status: str | None = None # 涨停状态:一字板 / 换手板 / N连板(仅涨停池)
|
||||
lu_desc: str | None = None # 涨停原因(仅涨停池)
|
||||
open_num: float | None = None # 打开次数
|
||||
limit_amount_yi: float | None = None # 封单额(亿元,仅涨停池)
|
||||
turnover_yi: float | None = None # 成交额(亿元,仅涨停池)
|
||||
first_lu_time: str | None = None # 首次涨停时间
|
||||
last_lu_time: str | None = None # 最后涨停时间(仅炸板池)
|
||||
limit_up_suc_rate: float | None = None # 近一年封板率 %(仅涨停池)
|
||||
|
||||
|
||||
class LimitLadderOut(BaseModel):
|
||||
ts_code: str
|
||||
name: str | None = None
|
||||
nums: int # 连板数
|
||||
|
||||
|
||||
class LimitBlockOut(BaseModel):
|
||||
name: str | None = None # 同花顺概念板块名
|
||||
days: float | None = None # 板块连涨天数
|
||||
up_stat: str | None = None # 如「6天3板」
|
||||
cons_nums: float | None = None # 连板家数
|
||||
up_nums: float | None = None # 涨停家数
|
||||
pct_chg: float | None = None # 板块涨跌 %
|
||||
|
||||
|
||||
class LimitSummaryOut(BaseModel):
|
||||
up_count: int = 0
|
||||
broken_count: int = 0
|
||||
down_count: int = 0
|
||||
first_board_count: int = 0
|
||||
max_ladder: LimitLadderOut | None = None
|
||||
ladder_dist: list[dict[str, int]] = Field(default_factory=list) # [{nums, count}] 升序(2板起)
|
||||
|
||||
|
||||
class LimitBoardResponse(BaseModel):
|
||||
trade_date: str # YYYY-MM-DD
|
||||
updated_at: str
|
||||
summary: LimitSummaryOut
|
||||
up: list[LimitStockOut] = Field(default_factory=list) # 涨停池(按封单额降序)
|
||||
broken: list[LimitStockOut] = Field(default_factory=list) # 炸板池
|
||||
down: list[LimitStockOut] = Field(default_factory=list) # 跌停池
|
||||
ladder: list[LimitLadderOut] = Field(default_factory=list) # 连板天梯(连板数降序)
|
||||
blocks: list[LimitBlockOut] = Field(default_factory=list) # 涨停最强板块
|
||||
errors: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
# ---------- 概念板块(THS:ths_index 列表 + ths_daily 行情 + ths_member 成分) ----------
|
||||
class ThsBoardOut(BaseModel):
|
||||
ts_code: str # 885835.TI / 700001.TI
|
||||
name: str | None = None
|
||||
type: str | None = None # N概念 I行业 TH主题 S特色 R地域 BB宽基 ST风格
|
||||
count: float | None = None # 成分个数
|
||||
list_date: str | None = None # YYYYMMDD
|
||||
close: float | None = None # 板块指数收盘(当日快照)
|
||||
pct_change: float | None = None # 涨跌幅 %
|
||||
vol: float | None = None # 成交量(手)
|
||||
turnover_rate: float | None = None # 换手率 %
|
||||
|
||||
|
||||
class ThsBoardListResponse(BaseModel):
|
||||
trade_date: str | None = None
|
||||
updated_at: str | None = None
|
||||
boards: list[ThsBoardOut] = Field(default_factory=list)
|
||||
errors: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ThsMemberOut(BaseModel):
|
||||
con_code: str # 成分股代码 000016.SZ
|
||||
con_name: str | None = None
|
||||
close: float | None = None # 现价(candles 最新,北交所等无底座为空)
|
||||
pct_chg: float | None = None # 涨跌幅 %(最新收盘 / 前收 - 1)
|
||||
|
||||
|
||||
class ThsBoardMembersResponse(BaseModel):
|
||||
code: str
|
||||
name: str | None = None
|
||||
members: list[ThsMemberOut] = Field(default_factory=list)
|
||||
|
||||
|
||||
# ---------- Auth ----------
|
||||
class LoginRequest(BaseModel):
|
||||
username: str = Field(min_length=1, max_length=64)
|
||||
|
||||
@@ -10,6 +10,7 @@ daily 与 daily_basic 分步独立落库:daily_basic 积分不足时快照仍
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
@@ -20,9 +21,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from .. import cache
|
||||
from ..config import settings
|
||||
from ..data.symbols import plain_code
|
||||
from ..data.sync_utils import call_retry, get_pro_lazy
|
||||
from ..models import AdjFactor, Candle, DailySnapshot, StockBasic, TradeCalendar
|
||||
from .llm import ScreenerError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# 进程内单例任务状态(uvicorn --reload 单进程场景够用)
|
||||
_sync_state: dict = {
|
||||
"running": False,
|
||||
@@ -40,32 +44,6 @@ _BATCH = 5000 # executemany 分批行数
|
||||
|
||||
# Tushare 积分/权限不足的特征文案(daily_basic 常见门槛)
|
||||
_PERM_MARKS = ("抱歉,您没有访问该项目权限", "积分", "权限")
|
||||
# 频率超限特征(等待 62s 重试一次)
|
||||
_RATE_MARKS = ("频率超限", "每分钟")
|
||||
|
||||
|
||||
def _call_retry(fn, *args, **kwargs):
|
||||
"""同步调用 tushare 接口;「每分钟」级频率超限等 62s 重试一次(小时级限频直接抛)。"""
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
except Exception as e: # noqa: BLE001
|
||||
msg = str(e)
|
||||
if any(m in msg for m in _RATE_MARKS) and "小时" not in msg:
|
||||
time.sleep(62)
|
||||
return fn(*args, **kwargs)
|
||||
raise
|
||||
|
||||
|
||||
def _get_pro():
|
||||
"""token 检查 + 返回 pro api 客户端(同步对象,调用需 to_thread 包裹)。
|
||||
|
||||
经 tushare_provider.get_pro 统一走镜像补丁(15000 积分档 token 只认 quicksync)。
|
||||
"""
|
||||
if not settings.tushare_token:
|
||||
raise ScreenerError("未配置 TUSHARE_TOKEN,无法同步全市场数据(backend/.env)")
|
||||
from ..data.tushare_provider import get_pro
|
||||
|
||||
return get_pro()
|
||||
|
||||
|
||||
def _parse_d(s: str) -> datetime:
|
||||
@@ -77,7 +55,7 @@ def _fetch_calendar_sync(pro) -> list[str]:
|
||||
time.sleep(settings.screener_sync_interval)
|
||||
end = (datetime.now() + timedelta(days=90)).strftime("%Y%m%d")
|
||||
start = (datetime.now() - timedelta(days=550)).strftime("%Y%m%d")
|
||||
cal = _call_retry(pro.trade_cal, exchange="SSE", start_date=start, end_date=end, is_open="1")
|
||||
cal = call_retry(pro.trade_cal, exchange="SSE", start_date=start, end_date=end, is_open="1")
|
||||
return sorted(cal["cal_date"].tolist())
|
||||
|
||||
|
||||
@@ -112,7 +90,7 @@ async def _recent_trade_dates(session: AsyncSession, pro, days: int) -> list[str
|
||||
def _fetch_daily(pro, d: str) -> list[dict]:
|
||||
"""拉取某交易日全市场日线(未复权)。当日数据未生成(盘前/盘中)返回空。"""
|
||||
time.sleep(settings.screener_sync_interval)
|
||||
df = _call_retry(pro.daily, trade_date=d)
|
||||
df = call_retry(pro.daily, trade_date=d)
|
||||
if df is None or df.empty:
|
||||
return []
|
||||
rows = []
|
||||
@@ -138,7 +116,7 @@ def _fetch_basic(pro, d: str) -> list[dict]:
|
||||
"""
|
||||
time.sleep(settings.screener_sync_interval)
|
||||
try:
|
||||
df = _call_retry(pro.daily_basic, trade_date=d)
|
||||
df = call_retry(pro.daily_basic, trade_date=d)
|
||||
except Exception as e: # noqa: BLE001
|
||||
msg = str(e)
|
||||
if any(m in msg for m in _PERM_MARKS):
|
||||
@@ -169,7 +147,7 @@ def _fetch_basic(pro, d: str) -> list[dict]:
|
||||
def _fetch_adj_factor(pro, d: str) -> list[dict]:
|
||||
"""拉取某交易日全市场复权因子(K线 bfq->qfq/hfq 本地换算的底座)。"""
|
||||
time.sleep(settings.screener_sync_interval)
|
||||
df = _call_retry(pro.adj_factor, trade_date=d)
|
||||
df = call_retry(pro.adj_factor, trade_date=d)
|
||||
if df is None or df.empty:
|
||||
return []
|
||||
return [
|
||||
@@ -181,7 +159,7 @@ def _fetch_adj_factor(pro, d: str) -> list[dict]:
|
||||
def _sync_stock_list_sync(pro) -> list[dict]:
|
||||
"""拉取在市股票列表。"""
|
||||
time.sleep(settings.screener_sync_interval)
|
||||
df = _call_retry(pro.stock_basic, exchange="", list_status="L",
|
||||
df = call_retry(pro.stock_basic, exchange="", list_status="L",
|
||||
fields="ts_code,symbol,name,area,industry,market,exchange,list_status,list_date,delist_date")
|
||||
rows = []
|
||||
for _, r in df.iterrows():
|
||||
@@ -232,6 +210,27 @@ async def _existing_candle_dates(session: AsyncSession) -> set[str]:
|
||||
return {r[0].strftime("%Y%m%d") for r in res if r[0] is not None}
|
||||
|
||||
|
||||
_UPSERT_CHUNK = 3000 # 单语句行数(asyncpg 参数上限拆批)
|
||||
|
||||
|
||||
async def _recent_day_counts(session: AsyncSession, dates: list[str]) -> dict[str, int]:
|
||||
"""指定交易日在市股票的 candles 行数(半日数据自愈用)。
|
||||
|
||||
单条 GROUP BY 走 ts 索引范围扫,窗口 ≤15 日、代价可忽略。
|
||||
"""
|
||||
if not dates:
|
||||
return {}
|
||||
lo = _parse_d(min(dates))
|
||||
hi = _parse_d(max(dates)) + timedelta(days=1)
|
||||
rows = (await session.execute(
|
||||
select(func.date(Candle.ts), func.count())
|
||||
.where(Candle.timeframe == "1d", Candle.ts >= lo, Candle.ts < hi,
|
||||
Candle.symbol.in_(select(StockBasic.symbol).where(StockBasic.list_status == "L")))
|
||||
.group_by(func.date(Candle.ts))
|
||||
)).all()
|
||||
return {r[0].strftime("%Y%m%d"): int(r[1]) for r in rows if r[0] is not None}
|
||||
|
||||
|
||||
async def _upsert_candle_day(session: AsyncSession, rows: list[dict], listed: set[str], d_str: str) -> None:
|
||||
"""把某交易日全市场日线 upsert 进 candles(不复权底座,幂等)。
|
||||
|
||||
@@ -253,9 +252,10 @@ async def _upsert_candle_day(session: AsyncSession, rows: list[dict], listed: se
|
||||
if not batch:
|
||||
return
|
||||
# on_conflict 语句整批渲染为占位符(非 executemany),asyncpg 单语句参数上限 32766,
|
||||
# 10 列 x 3000 行 = 30000 参数留出余量
|
||||
for i in range(0, len(batch), 3000):
|
||||
stmt = pg_insert(Candle).values(batch[i : i + 3000])
|
||||
# 10 列 x 3000 行 = 30000 参数留出余量。分批只拆语句,commit 在循环外 ——
|
||||
# 单日一事务:写一半崩溃整日回滚,该日期语义上「未同步」,下次自然重拉(不留半日数据)
|
||||
for i in range(0, len(batch), _UPSERT_CHUNK):
|
||||
stmt = pg_insert(Candle).values(batch[i : i + _UPSERT_CHUNK])
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=["symbol", "timeframe", "ts"],
|
||||
set_={
|
||||
@@ -266,7 +266,7 @@ async def _upsert_candle_day(session: AsyncSession, rows: list[dict], listed: se
|
||||
},
|
||||
)
|
||||
await session.execute(stmt)
|
||||
await session.commit()
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def _run_sync(days: int, force: bool) -> None:
|
||||
@@ -277,7 +277,7 @@ async def _run_sync(days: int, force: bool) -> None:
|
||||
from ..db import async_session # 延迟导入避免循环
|
||||
|
||||
try:
|
||||
pro = await asyncio.to_thread(_get_pro)
|
||||
pro = await asyncio.to_thread(get_pro_lazy)
|
||||
|
||||
# 1) 股票列表(已有数据则跳过——stock_basic 低积分版限频 1 次/小时)
|
||||
async with async_session() as session:
|
||||
@@ -307,6 +307,15 @@ async def _run_sync(days: int, force: bool) -> None:
|
||||
select(StockBasic.symbol).where(StockBasic.list_status == "L")
|
||||
)).scalars()
|
||||
)
|
||||
if not force:
|
||||
# 半日数据自愈(修单日原子化之前的历史残留):写入中途崩溃的日期
|
||||
# 行数 ≈ 1 批(3000),显著低于完整日(~5300)。只查最近 15 个交易日
|
||||
# ——实际风险区且窗口内上市数变化 <2%,0.7 阈值安全;更老的日期不查
|
||||
# (上市数变化会误判,历史缺口本就由 TDX 底座兜底)。
|
||||
counts = await _recent_day_counts(session, [d for d in dates[:15] if d in have_daily])
|
||||
if counts:
|
||||
floor = max(_UPSERT_CHUNK + 1, int(max(counts.values()) * 0.7))
|
||||
have_daily -= {d for d, n in counts.items() if n < floor}
|
||||
todo = [d for d in dates if d not in have_daily]
|
||||
_sync_state["total_days"] = len(todo)
|
||||
_sync_state["done_days"] = 0
|
||||
@@ -352,7 +361,7 @@ async def _run_sync(days: int, force: bool) -> None:
|
||||
try:
|
||||
await _refresh_stats(await cache.get_version("candles"))
|
||||
except Exception: # noqa: BLE001 —— 预热失败只影响统计数字的新鲜度
|
||||
pass
|
||||
log.warning("统计缓存预热失败(下轮轮询会 SWR 重算)", exc_info=True)
|
||||
_sync_state["step"] = "同步完成"
|
||||
except Exception as e: # noqa: BLE001
|
||||
_sync_state["error"] = f"同步失败:{str(e)[:300]}"
|
||||
@@ -414,16 +423,9 @@ async def _heavy_stats(session: AsyncSession) -> dict:
|
||||
|
||||
async def _store_stats(ver: int, data: dict) -> None:
|
||||
_status_stats_cache.update(at=time.time(), ver=ver, data=data)
|
||||
# 写 Redis 后台执行,失败由 cache 层静默降级,不拖慢调用方
|
||||
tasks = [
|
||||
asyncio.create_task(cache.cache_set(
|
||||
f"syncstats:v{ver}", data, ttl=settings.sync_stats_redis_ttl)),
|
||||
asyncio.create_task(cache.cache_set(
|
||||
_STATS_LAST_KEY, data, ttl=settings.sync_stats_redis_ttl)),
|
||||
]
|
||||
_stats_bg_tasks.update(tasks)
|
||||
for t in tasks:
|
||||
t.add_done_callback(_stats_bg_tasks.discard)
|
||||
# 写 Redis 后台执行(cache.set_bg 挂全局集合防 GC),失败由 cache 层静默降级
|
||||
cache.set_bg(f"syncstats:v{ver}", data, ttl=settings.sync_stats_redis_ttl)
|
||||
cache.set_bg(_STATS_LAST_KEY, data, ttl=settings.sync_stats_redis_ttl)
|
||||
|
||||
|
||||
async def _refresh_stats(ver: int) -> None:
|
||||
|
||||
Reference in New Issue
Block a user