This commit is contained in:
2026-09-09 11:35:02 +08:00
parent bc1c72d558
commit d656c05b3d
35 changed files with 711 additions and 3194 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,8 @@
"""只登录、不注册的鉴权 API。"""
from __future__ import annotations
import time
from collections import deque
from datetime import timedelta
from fastapi import APIRouter, Cookie, Depends, HTTPException, Request, Response, status
@@ -49,6 +51,33 @@ def clear_session_cookie(response: Response) -> None:
)
# ---------- 登录按 IP 限速(进程内滑动窗口,无外部依赖) ----------
# 补充账户级锁定auth_max_failed_logins本层拦多 IP 分布爆破,也稀释
# 「故意输错 5 次锁死他人账户」的滥用面。反代部署时 host 是代理 IP需改读 X-Forwarded-For。
_LOGIN_WINDOW = 60.0
_LOGIN_MAX_PER_WINDOW = 15
_login_attempts: dict[str, deque[float]] = {}
_login_gc_at = 0.0
def _login_rate_limited(ip: str) -> bool:
"""超限返回 True未超限记录本次尝试成功失败都计"""
global _login_gc_at
now = time.monotonic()
q = _login_attempts.setdefault(ip, deque())
while q and q[0] <= now - _LOGIN_WINDOW:
q.popleft()
if len(q) >= _LOGIN_MAX_PER_WINDOW:
return True
q.append(now)
if now - _login_gc_at > 3600: # 顺手回收陈旧 entry防长跑内存增长
_login_gc_at = now
stale = now - _LOGIN_WINDOW * 10
for k in [k for k, v in _login_attempts.items() if not v or v[-1] <= stale]:
del _login_attempts[k]
return False
@router.post("/login", response_model=LoginResponse)
async def login(
payload: LoginRequest,
@@ -56,6 +85,10 @@ async def login(
response: Response,
db: AsyncSession = Depends(get_session),
) -> LoginResponse:
ip = request.client.host if request.client else "?"
if _login_rate_limited(ip):
raise HTTPException(status_code=429, detail="登录尝试过于频繁,请稍后再试")
now = utcnow()
username = payload.username.strip()
user = (await db.execute(select(User).where(User.username == username))).scalar_one_or_none()

View File

@@ -11,6 +11,7 @@ match=all(连续满足)/any(曾经满足),多条件之间取 AND。
"""
from __future__ import annotations
import asyncio
from datetime import date, datetime, timedelta
import numpy as np
@@ -120,6 +121,69 @@ def _stats_block(trades: list[dict]) -> dict:
}
def _scan_batch(
candle_rows: list,
code_by_symbol: dict[str, str],
f_map: dict,
name_map: dict[str, str],
spec: EventBacktestSpec,
start_ts: datetime,
trades_limit: int,
) -> list[dict]:
"""纯 CPU一批 bar -> 交易明细(放线程池跑,不阻塞事件循环)。"""
trades: list[dict] = []
if not candle_rows:
return trades
bars = pd.DataFrame(
candle_rows, columns=["symbol", "ts", "open", "high", "low", "close"]
)
for symbol, g in bars.groupby("symbol", sort=False):
if len(g) < 30:
continue
g = g.reset_index(drop=True)
ts_code_l = code_by_symbol[symbol]
cache: dict = {"_families": set()}
mask = _signal_mask(g, spec, cache)
if not mask.any():
continue
for sig_i in np.flatnonzero(mask.to_numpy()):
ts_sig = g.at[sig_i, "ts"]
# 信号必须落在回测窗口内buffer 区只用于指标配热)
if ts_sig < start_ts:
continue
ie = _entry_exit_indices(int(sig_i), spec, len(g))
if ie is None:
continue
entry_i, exit_i = ie
e_row, x_row = g.iloc[entry_i], g.iloc[exit_i]
e_price = _price_at(e_row, "open" if spec.entry_timing == "next_open" else "close")
x_price = _price_at(x_row, "open" if spec.exit_timing == "open" else "close")
if not e_price or not x_price:
continue
f_in = f_map.get((ts_code_l, e_row["ts"].date()), 1.0)
f_out = f_map.get((ts_code_l, x_row["ts"].date()), 1.0)
ret_pct = (x_price * f_out) / (e_price * f_in) * 100 - 100
trades.append({
"ts_code": ts_code_l,
"name": name_map.get(ts_code_l),
"entry_date": e_row["ts"], "entry_price": round(e_price, 3),
"exit_date": x_row["ts"], "exit_price": round(x_price, 3),
"ret_pct": round(float(ret_pct), 3),
})
if len(trades) >= trades_limit:
return trades
return trades
def _summarize(trades: list[dict]) -> tuple[dict, list[dict]]:
"""纯 CPU汇总统计 + 最好/最差样本(同样下线程池)。"""
stats = _stats_block(trades)
# 明细样本:最好 100 + 最差 100其余统计已覆盖
trades_sorted = sorted(trades, key=lambda t: t["ret_pct"], reverse=True)
sample = trades_sorted[:100] + (trades_sorted[-100:] if len(trades_sorted) > 100 else [])
return stats, sample
async def run_event_backtest(
session: AsyncSession,
spec: EventBacktestSpec,
@@ -195,53 +259,15 @@ async def run_event_backtest(
)).all()
f_map = {(r[0], r[1].date()): float(r[2]) for r in adj_rows if r[2]}
bars = pd.DataFrame(
candle_rows, columns=["symbol", "ts", "open", "high", "low", "close"]
)
for symbol, g in bars.groupby("symbol", sort=False):
if len(g) < 30:
continue
g = g.reset_index(drop=True)
ts_code_l = code_by_symbol[symbol]
cache: dict = {"_families": set()}
mask = _signal_mask(g, spec, cache)
if not mask.any():
continue
for sig_i in np.flatnonzero(mask.to_numpy()):
ts_sig = g.at[sig_i, "ts"]
# 信号必须落在回测窗口内buffer 区只用于指标配热)
if ts_sig < start_ts:
continue
ie = _entry_exit_indices(int(sig_i), spec, len(g))
if ie is None:
continue
entry_i, exit_i = ie
e_row, x_row = g.iloc[entry_i], g.iloc[exit_i]
e_price = _price_at(e_row, "open" if spec.entry_timing == "next_open" else "close")
x_price = _price_at(x_row, "open" if spec.exit_timing == "open" else "close")
if not e_price or not x_price:
continue
f_in = f_map.get((ts_code_l, e_row["ts"].date()), 1.0)
f_out = f_map.get((ts_code_l, x_row["ts"].date()), 1.0)
ret_pct = (x_price * f_out) / (e_price * f_in) * 100 - 100
trades.append({
"ts_code": ts_code_l,
"name": name_map.get(ts_code_l),
"entry_date": e_row["ts"], "entry_price": round(e_price, 3),
"exit_date": x_row["ts"], "exit_price": round(x_price, 3),
"ret_pct": round(float(ret_pct), 3),
})
if len(trades) >= MAX_TRADES:
break
if len(trades) >= MAX_TRADES:
break
# pandas 全市场扫描是同步 CPU 重计算丢线程池跑await 期间事件循环可服务其他请求)
trades.extend(await asyncio.to_thread(
_scan_batch, candle_rows, code_by_symbol, f_map, name_map,
spec, start_ts, MAX_TRADES - len(trades),
))
if len(trades) >= MAX_TRADES:
break
stats = _stats_block(trades)
# 明细样本:最好 100 + 最差 100其余统计已覆盖
trades_sorted = sorted(trades, key=lambda t: t["ret_pct"], reverse=True)
sample = trades_sorted[:100] + (trades_sorted[-100:] if len(trades_sorted) > 100 else [])
stats, sample = await asyncio.to_thread(_summarize, trades)
return {
"spec": spec,
"universe": ts_code or "all",

View File

@@ -1,7 +1,8 @@
"""Redis 读缓存(可选基础设施)。
- REDIS_URL 留空、连接失败或超时:所有操作静默退化为「无缓存」,接口照常直查数据库
且本进程内禁用重试(避免每个请求都陪跑一次连接超时)。
- REDIS_URL 留空、连接失败或超时:所有操作静默退化为「无缓存」,接口照常直查数据库
故障后熔断 _RECOVERY_SECONDS期间所有请求直连不陪跑连接超时到期自动放行
一次探测——成功即完全恢复仍失败则重新熔断Redis 属加速件,坏了不能拖慢接口)。
- 失效策略TTL 自然过期 + 版本号INCR作废。自选股增删等写操作只 INCR 版本 key
旧缓存 key 里带着旧版本号,无需 SCAN 批量删除。
- 只缓存「读多写少、可容忍短暂陈旧」的聚合数据(股票列表、筛选项等);
@@ -13,6 +14,7 @@
"""
from __future__ import annotations
import asyncio
import hashlib
import json
import time
@@ -23,12 +25,13 @@ import redis.asyncio as aioredis
from .config import settings
_pool: aioredis.ConnectionPool | None = None
_disabled = False # 一次失败后本进程禁用Redis 属加速件,坏了不能拖慢接口)
_RECOVERY_SECONDS = 60.0 # 熔断时长:期间直连不试 Redis到期放行一次探测
_disabled_until = 0.0 # 熔断截止的 monotonic 时刻0 = 未熔断
def _client() -> aioredis.Redis | None:
global _pool, _disabled
if not settings.redis_url or _disabled:
global _pool
if not settings.redis_url or time.monotonic() < _disabled_until:
return None
if _pool is None:
_pool = aioredis.ConnectionPool.from_url(
@@ -43,8 +46,8 @@ def _client() -> aioredis.Redis | None:
def _bail() -> None:
global _disabled
_disabled = True
global _disabled_until
_disabled_until = time.monotonic() + _RECOVERY_SECONDS
def digest(*parts: Any) -> str:
@@ -84,7 +87,21 @@ def local_set(key: str, raw: str, ttl: int) -> None:
_local_store[key] = (time.monotonic() + max(1, min(ttl, 120)), raw)
_local_bytes += len(raw) + 64 # 连同 dict/tuple 开销粗略计入
while _local_store and (len(_local_store) > _LOCAL_MAX_ENTRIES or _local_bytes > _LOCAL_MAX_BYTES):
_local_bytes -= len(_local_store.popitem(last=False)[1][1]) + 64
# dict 是插入序next(iter(...)) 即最旧键dict.popitem 不支持参数,别写成 OrderedDict 的写法)
oldest = next(iter(_local_store))
_local_bytes -= len(_local_store.pop(oldest)[1]) + 64
# --- 后台写fire-and-forget------------------------------------------------
# 读路径拿到响应后异步写 Redis、不阻塞返回。统一入口任务挂全局集合防 GC
# cache_set 内部自带异常静默缓存层尽力而为优雅停机丢最后一次写无害TTL 兜底)。
_bg_tasks: set[asyncio.Task] = set()
def set_bg(key: str, value: Any, ttl: int) -> None:
t = asyncio.create_task(cache_set(key, value, ttl))
_bg_tasks.add(t)
t.add_done_callback(_bg_tasks.discard)
# --- 版本号本地缓存:热请求连 Redis GET ver:xx 都省掉 ------------------------

View File

@@ -56,6 +56,10 @@ class Settings(BaseSettings):
screener_default_limit: int = 200 # 选股结果条数上限
screener_sync_interval: float = 0.35 # 全市场批量调用间隔Tushare 控频
# ---- 夜间定时任务(收盘后自动同步 + 会话清理;见 app/scheduler.py----
nightly_sync_enabled: bool = True
nightly_sync_hour: int = 18 # 本地时间整点,触发在 :05收盘后日线已生成
# A股交易成本基准日 2026-08——做成可配置参数便于将来按生效日期版本化
stamp_duty_rate: float = 0.0005 # 印花税 0.05%单边卖出2023-08-28 减半)
transfer_fee_rate: float = 0.00001 # 过户费 0.001%沪深双边2022 调整)

View File

@@ -9,7 +9,7 @@ from __future__ import annotations
import asyncio
import time
from datetime import datetime, timedelta, timezone
from datetime import datetime
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert as pg_insert
@@ -18,12 +18,10 @@ from sqlalchemy.ext.asyncio import AsyncSession
from ..config import settings
from ..db import async_session
from ..models import StockCompany
from .sync_utils import call_retry, f_clean, fresh, get_pro_lazy, s_clean, utcnow
_REFRESH_DAYS = 30
# 频率超限特征(等待 62s 重试一次;与 screener.market_sync / data.etf_sync._call_retry 同款语义)
_RATE_MARKS = ("频率超限", "每分钟")
# 显式列出全部字段introduction/office/main_business/business_scope 文档标注默认不显示,
# 不传 fields 时 tushare 不返回这四列(实测 000001.SZ
_FIELDS = (
@@ -32,87 +30,38 @@ _FIELDS = (
"employees,main_business,business_scope"
)
_pro = None # 惰性单例get_pro 每次都 ts.set_token 写文件,没必要重复)
def _get_pro():
if not settings.tushare_token:
raise RuntimeError("未配置 TUSHARE_TOKEN无法拉取公司简介backend/.env")
global _pro
if _pro is None:
from .tushare_provider import get_pro
_pro = get_pro()
return _pro
def _call_retry(fn, *args, **kwargs):
"""同步调用 tushare 接口;「每分钟」级频率超限等 62s 重试一次。"""
try:
return fn(*args, **kwargs)
except Exception as e: # noqa: BLE001
msg = str(e)
if any(m in msg for m in _RATE_MARKS) and "小时" not in msg:
time.sleep(62)
return fn(*args, **kwargs)
raise
def _utcnow() -> datetime:
return datetime.now(timezone.utc).replace(tzinfo=None)
def _fresh(updated_at: datetime | None) -> bool:
return updated_at is not None and updated_at >= _utcnow() - timedelta(days=_REFRESH_DAYS)
def _s(v) -> str | None:
"""pandas NaN / 空串 / None -> None其余 strip。"""
if v is None or (isinstance(v, float) and v != v):
return None
s = str(v).strip()
return s or None
def _f(v) -> float | None:
try:
f = float(v)
except (TypeError, ValueError):
return None
return None if f != f else f # NaN -> None
def _i(v) -> int | None:
f = _f(v)
f = f_clean(v)
return None if f is None else int(f)
def fetch_company_sync(ts_code: str) -> dict | None:
"""同步拉单只公司简介(需在 to_thread 里跑);返回行 dict无此股返回 None。"""
time.sleep(settings.screener_sync_interval)
df = _call_retry(_get_pro().stock_company, ts_code=ts_code, fields=_FIELDS)
df = call_retry(get_pro_lazy().stock_company, ts_code=ts_code, fields=_FIELDS)
if df is None or df.empty:
return None
r = df.iloc[0]
return {
"ts_code": ts_code,
"com_name": _s(r.get("com_name")),
"com_id": _s(r.get("com_id")),
"chairman": _s(r.get("chairman")),
"manager": _s(r.get("manager")),
"secretary": _s(r.get("secretary")),
"reg_capital": _f(r.get("reg_capital")),
"setup_date": _s(r.get("setup_date")),
"province": _s(r.get("province")),
"city": _s(r.get("city")),
"introduction": _s(r.get("introduction")),
"website": _s(r.get("website")),
"email": _s(r.get("email")),
"office": _s(r.get("office")),
"com_name": s_clean(r.get("com_name")),
"com_id": s_clean(r.get("com_id")),
"chairman": s_clean(r.get("chairman")),
"manager": s_clean(r.get("manager")),
"secretary": s_clean(r.get("secretary")),
"reg_capital": f_clean(r.get("reg_capital")),
"setup_date": s_clean(r.get("setup_date")),
"province": s_clean(r.get("province")),
"city": s_clean(r.get("city")),
"introduction": s_clean(r.get("introduction")),
"website": s_clean(r.get("website")),
"email": s_clean(r.get("email")),
"office": s_clean(r.get("office")),
"employees": _i(r.get("employees")),
"main_business": _s(r.get("main_business")),
"business_scope": _s(r.get("business_scope")),
"updated_at": _utcnow(),
"main_business": s_clean(r.get("main_business")),
"business_scope": s_clean(r.get("business_scope")),
"updated_at": utcnow(),
}
@@ -151,7 +100,7 @@ async def get_company(session: AsyncSession, ts_code: str) -> dict | None:
"""
row = (await session.execute(
select(StockCompany).where(StockCompany.ts_code == ts_code))).scalar_one_or_none()
if row is not None and _fresh(row.updated_at):
if row is not None and fresh(row.updated_at, _REFRESH_DAYS):
return _row_dict(row) if row.com_name is not None else None # 墓碑 -> None
# 释放请求会话持有的连接:后面可能隔着 1-2s 的 tushare 调用,别长占连接池。
# 用 close() 而非 rollback()rollback 会把会话身份映射里的实例全部 expire——
@@ -165,7 +114,7 @@ async def get_company(session: AsyncSession, ts_code: str) -> dict | None:
async with async_session() as s2: # 锁内重读 + 写入走新会话
row = (await s2.execute(
select(StockCompany).where(StockCompany.ts_code == ts_code))).scalar_one_or_none()
if row is not None and _fresh(row.updated_at):
if row is not None and fresh(row.updated_at, _REFRESH_DAYS):
return _row_dict(row) if row.com_name is not None else None
try:
fetched = await asyncio.to_thread(fetch_company_sync, ts_code)
@@ -174,5 +123,5 @@ async def get_company(session: AsyncSession, ts_code: str) -> dict | None:
if row is not None and row.com_name is not None:
return _row_dict(row)
raise
await _upsert(s2, fetched or {"ts_code": ts_code, "updated_at": _utcnow()})
await _upsert(s2, fetched or {"ts_code": ts_code, "updated_at": utcnow()})
return fetched

View File

@@ -20,7 +20,7 @@ from __future__ import annotations
import asyncio
import time
from datetime import datetime, timedelta, timezone
from datetime import datetime, timedelta
from sqlalchemy import delete, func, select, text
from sqlalchemy.dialects.postgresql import insert as pg_insert
@@ -29,6 +29,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from .. import cache
from ..config import settings
from . import etf_provider
from .sync_utils import call_retry, get_pro_lazy, utcnow
# 进程内单例任务状态uvicorn 单进程场景够用)
_state: dict = {
@@ -46,34 +47,6 @@ _lock = asyncio.Lock()
_BATCH = 3000 # upsert 分批行数asyncpg 单语句参数上限 3276610 列/行)
# fund_daily 返回全市场基金 ~2100 行,一天一批远小于上限
# 频率超限特征(等待 62s 重试一次;与 screener.market_sync._call_retry 同款语义)
_RATE_MARKS = ("频率超限", "每分钟")
def _call_retry(fn, *args, **kwargs):
"""同步调用 tushare 接口;「每分钟」级频率超限等 62s 重试一次。"""
try:
return fn(*args, **kwargs)
except Exception as e: # noqa: BLE001
msg = str(e)
if any(m in msg for m in _RATE_MARKS) and "小时" not in msg:
time.sleep(62)
return fn(*args, **kwargs)
raise
def _get_pro():
"""token 检查 + 返回 pro api 客户端(同步对象,调用需 to_thread 包裹)。"""
if not settings.tushare_token:
raise RuntimeError("未配置 TUSHARE_TOKEN无法同步 ETF 日线backend/.env")
from .tushare_provider import get_pro
return get_pro()
def _utcnow() -> datetime:
return datetime.now(timezone.utc).replace(tzinfo=None)
def _parse_d(s: str) -> datetime:
return datetime.strptime(str(s), "%Y%m%d")
@@ -85,7 +58,7 @@ async def _sync_spot(session: AsyncSession) -> int:
async with etf_provider.new_client() as client:
rows = await etf_provider.fetch_etf_spot(client)
now = _utcnow()
now = utcnow()
stmt = pg_insert(EtfBasic).values([{**r, "updated_at": now} for r in rows])
stmt = stmt.on_conflict_do_update(
index_elements=["ts_code"],
@@ -108,7 +81,7 @@ async def _sync_spot(session: AsyncSession) -> int:
def _fetch_day_sync(pro, d: str) -> list[dict]:
"""拉某交易日全市场场内基金日线fund_daily未生成的日期返回空"""
time.sleep(settings.screener_sync_interval)
df = _call_retry(pro.fund_daily, trade_date=d)
df = call_retry(pro.fund_daily, trade_date=d)
if df is None or df.empty:
return []
rows = []
@@ -128,7 +101,7 @@ def _fetch_day_sync(pro, d: str) -> list[dict]:
def _fetch_symbol_sync(pro, ts_code: str, start: str | None, end: str | None) -> list[dict]:
"""按 ts_code 增量/全量拉单只 ETF 日线start=None 即上市以来全量)。"""
time.sleep(settings.screener_sync_interval)
df = _call_retry(pro.fund_daily, ts_code=ts_code, start_date=start, end_date=end)
df = call_retry(pro.fund_daily, ts_code=ts_code, start_date=start, end_date=end)
if df is None or df.empty:
return []
df = df.sort_values("trade_date")
@@ -209,7 +182,7 @@ async def _run_sync(full: bool) -> None:
from ..models import Candle, EtfBasic, TradeCalendar
try:
pro = await asyncio.to_thread(_get_pro)
pro = await asyncio.to_thread(get_pro_lazy)
# 1) 快照 -> etf_basic
_state["step"] = "正在拉取 ETF 列表"

View File

@@ -113,6 +113,9 @@ async def sync_symbol(
)
await session.execute(stmt)
await session.commit()
# 作废 candles 相关读缓存preview 等)——不 bump 的话旧版本号的缓存要等 TTL 自然过期
from .. import cache
await cache.bump_version("candles")
return {"symbol": code, "bars": len(bars), "source": used}

View File

@@ -13,13 +13,13 @@ from __future__ import annotations
import asyncio
import json
import math
import time
from datetime import date, datetime, timedelta
from .. import cache
from ..config import settings
from ..domain import Bar
from .sync_utils import d8_iso, f_clean
# ---- 静态元数据表tushare index_global 支持的全部 21 个指数,展示顺序即文档顺序)----
# region: americas 美洲 / europe 欧洲 / asia 亚太含港股与富时A50
@@ -82,22 +82,6 @@ class GlobalIndexError(RuntimeError):
"""全部国际指数都拉不到token/网络故障)——接口层转 503。"""
def _f(v) -> float | None:
"""pandas 值 -> floatNaN/None -> None。"""
if v is None:
return None
try:
f = float(v)
except (TypeError, ValueError):
return None
return None if math.isnan(f) else f
def _d(v) -> str | None:
"""YYYYMMDD -> 'YYYY-MM-DD'(字符串便于 JSON 缓存)。"""
return datetime.strptime(str(v), "%Y%m%d").date().isoformat() if v else None
def is_cn_index(code: str) -> bool:
return "." in code
@@ -128,14 +112,14 @@ def _fetch_quote_sync(pro, ts_code: str) -> dict:
tail = df.tail(_SPARK_DAYS)
last = df.iloc[-1]
return {
"close": _f(last["close"]),
"change": _f(last.get("change")),
"pct_chg": _f(last.get("pct_chg")),
"open": _f(last.get("open")),
"high": _f(last.get("high")),
"low": _f(last.get("low")),
"pre_close": _f(last.get("pre_close")),
"trade_date": _d(last["trade_date"]),
"close": f_clean(last["close"]),
"change": f_clean(last.get("change")),
"pct_chg": f_clean(last.get("pct_chg")),
"open": f_clean(last.get("open")),
"high": f_clean(last.get("high")),
"low": f_clean(last.get("low")),
"pre_close": f_clean(last.get("pre_close")),
"trade_date": d8_iso(last["trade_date"]),
"spark": [round(float(c), 4) for c in tail["close"]],
"spark_dates": [str(d) for d in tail["trade_date"]],
}
@@ -249,8 +233,8 @@ def _fetch_global_bars_sync(ts_code: str) -> list[Bar]:
df = pd.concat(frames).drop_duplicates(subset="trade_date").sort_values("trade_date")
bars: list[Bar] = []
for _, r in df.iterrows():
vol = _f(r.get("vol"))
amt = _f(r.get("amount"))
vol = f_clean(r.get("vol"))
amt = f_clean(r.get("amount"))
bars.append(
Bar(
ts=datetime.strptime(str(r["trade_date"]), "%Y%m%d"),
@@ -318,9 +302,9 @@ def _fetch_basic_sync(ts_code: str) -> dict:
"market": r.get("market"),
"publisher": r.get("publisher"),
"category": r.get("category"),
"base_date": _d(r.get("base_date")),
"base_point": _f(r.get("base_point")),
"list_date": _d(r.get("list_date")),
"base_date": d8_iso(r.get("base_date")),
"base_point": f_clean(r.get("base_point")),
"list_date": d8_iso(r.get("list_date")),
}
@@ -354,10 +338,10 @@ def _fetch_valuation_sync(ts_code: str, days: int) -> list[dict]:
rows = []
for _, r in df.sort_values("trade_date").iterrows():
rows.append({
"trade_date": _d(r["trade_date"]),
"pe": _f(r.get("pe")), "pe_ttm": _f(r.get("pe_ttm")), "pb": _f(r.get("pb")),
"turnover_rate": _f(r.get("turnover_rate")),
"total_mv": _f(r.get("total_mv")), "float_mv": _f(r.get("float_mv")),
"trade_date": d8_iso(r["trade_date"]),
"pe": f_clean(r.get("pe")), "pe_ttm": f_clean(r.get("pe_ttm")), "pb": f_clean(r.get("pb")),
"turnover_rate": f_clean(r.get("turnover_rate")),
"total_mv": f_clean(r.get("total_mv")), "float_mv": f_clean(r.get("float_mv")),
})
return rows
@@ -395,7 +379,7 @@ def _fetch_weights_sync(ts_code: str) -> dict | None:
latest_date = df.iloc[0]["trade_date"]
rows = df[df["trade_date"] == latest_date]
return {
"trade_date": _d(latest_date),
"trade_date": d8_iso(latest_date),
"total": int(len(rows)),
"items": [
{"con_code": str(r["con_code"]), "weight": round(float(r["weight"]), 4)}

View File

@@ -17,7 +17,6 @@
from __future__ import annotations
import asyncio
import math
import time
from datetime import date, datetime, timedelta
@@ -26,6 +25,7 @@ import pandas as pd
from .. import cache
from ..config import settings
from .sync_utils import d8_iso, f_clean
# (tushare代码, 名称, 地区, 腾讯符号) —— 展示顺序即列表顺序
# 首页聚焦中美(港股/国际指数在 /indexes 国际指数页标普500 腾讯符号是 s_usINX不是 s_usSPX
@@ -57,22 +57,6 @@ class MarketOverviewError(RuntimeError):
"""所有指数都拉不到token/网络故障)——接口层转 503。"""
def _f(v) -> float | None:
"""pandas 值 -> floatNaN/None -> None否则 JSON 里会出现 NaN"""
if v is None:
return None
try:
f = float(v)
except (TypeError, ValueError):
return None
return None if math.isnan(f) else f
def _d(v) -> str | None:
"""YYYYMMDD -> 'YYYY-MM-DD'(字符串便于 JSON 缓存pydantic 响应模型自动 coerce"""
return datetime.strptime(str(v), "%Y%m%d").date().isoformat() if v else None
def _get_pro():
if not settings.tushare_token:
raise MarketOverviewError("未配置 TUSHARE_TOKEN无法获取大盘行情backend/.env")
@@ -165,10 +149,10 @@ def _quote_from_df(df: pd.DataFrame) -> dict | None:
tail = df.tail(_SPARK_DAYS)
last = df.iloc[-1]
return {
"close": _f(last["close"]),
"change": _f(last.get("change")),
"pct_chg": _f(last.get("pct_chg")),
"trade_date": _d(last["trade_date"]),
"close": f_clean(last["close"]),
"change": f_clean(last.get("change")),
"pct_chg": f_clean(last.get("pct_chg")),
"trade_date": d8_iso(last["trade_date"]),
"spark": [round(float(c), 4) for c in tail["close"]],
"spark_dates": [str(d) for d in tail["trade_date"]],
}
@@ -193,10 +177,10 @@ def _fetch_stats_sync(pro) -> dict | None:
if sh_m is None or sz_m is None:
return None
# 两边各自取最新,日期不一致时以较旧一天为准凑齐口径(罕见,通常同日)
d = min(_d(sh_m["trade_date"]), _d(sz_m["trade_date"]))
d = min(d8_iso(sh_m["trade_date"]), d8_iso(sz_m["trade_date"]))
def _sum(col: str) -> float | None:
a, b = _f(sh_m.get(col)), _f(sz_m.get(col))
a, b = f_clean(sh_m.get(col)), f_clean(sz_m.get(col))
return None if a is None or b is None else round(a + b, 2)
return {
@@ -204,7 +188,7 @@ def _fetch_stats_sync(pro) -> dict | None:
"total_mv": _sum("total_mv"),
"float_mv": _sum("float_mv"),
"amount": _sum("amount"),
"turnover": _f(sh_m.get("tr")), # 换手率仅沪市有,展示口径注明沪市
"turnover": f_clean(sh_m.get("tr")), # 换手率仅沪市有,展示口径注明沪市
}
@@ -222,7 +206,7 @@ def _fetch_amount_history_sync(pro) -> list[dict]:
if len(common) == 0:
return []
total = (sh_m[common] + sz_m[common]).sort_index()
return [{"date": _d(d), "amount": round(float(v), 2)} for d, v in total.tail(_AMOUNT_HIST_BARS).items()]
return [{"date": d8_iso(d), "amount": round(float(v), 2)} for d, v in total.tail(_AMOUNT_HIST_BARS).items()]
# ---- EOD 的 SWRstale-while-revalidate新鲜期内直返过期先返旧值后台刷新 ----

View File

@@ -16,6 +16,7 @@ from __future__ import annotations
import asyncio
import calendar
import json
import logging
import time
from dataclasses import dataclass
from datetime import date
@@ -29,6 +30,8 @@ from ..db import async_session
from ..models import StockReference
from .sync_utils import call_retry, f_clean, fresh, get_pro_lazy, read_sync_state, s_clean, upsert_sync_state, utcnow
log = logging.getLogger(__name__)
_REFRESH_DAYS = 7
@@ -394,12 +397,12 @@ async def _sync_repurchase_locked(only_current: bool) -> dict[str, list[dict]]:
async def _repurchase_backfill() -> None:
"""后台全量回填(近 24 个月);失败静默——下次触发重试。"""
"""后台全量回填(近 24 个月);失败记录日志——下次触发重试。"""
try:
async with _repurchase_lock:
await _sync_repurchase_locked(only_current=False)
except Exception: # noqa: BLE001 后台任务无人接异常
pass
except Exception: # noqa: BLE001 后台任务无人接异常,至少留痕
log.warning("回购数据后台回填失败(下次触发重试)", exc_info=True)
def _spawn_repurchase_backfill() -> None:

View File

@@ -68,6 +68,11 @@ def f_clean(v) -> float | None:
return None if f != f else f # NaN -> None
def d8_iso(v) -> str | None:
"""tushare YYYYMMDD -> 'YYYY-MM-DD'(字符串便于 JSON 缓存pydantic 自动 coerce"""
return datetime.strptime(str(v), "%Y%m%d").date().isoformat() if v else None
async def read_sync_state(session: AsyncSession, ts_code: str, kind: str) -> StockSyncState | None:
return (await session.execute(
select(StockSyncState).where(

View File

@@ -1,11 +1,12 @@
"""FastAPI 入口。数据库结构统一由 Alembic 管理。"""
from contextlib import asynccontextmanager
import asyncio
from contextlib import asynccontextmanager, suppress
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from sqlalchemy import text
from . import cache
from . import cache, scheduler
from .api import router
from .auth_api import router as auth_router
from .config import settings
@@ -16,7 +17,11 @@ from .db import engine
async def lifespan(app: FastAPI):
async with engine.connect() as conn:
await conn.execute(text("SELECT 1"))
nightly = asyncio.create_task(scheduler.run_nightly_loop())
yield
nightly.cancel()
with suppress(asyncio.CancelledError):
await nightly
await engine.dispose()
await cache.aclose() # 释放 Redis 连接池(未启用时是 no-op

View File

@@ -4,8 +4,8 @@ Candle 表设计与 TimescaleDB hypertable 完全兼容:将来在目标 PG 库
SELECT create_hypertable('candles', 'ts');
即可升级为时序表 + Continuous Aggregates 多周期预聚合,无需改表结构。
智能选股三表stock_basic / market_daily / daily_snapshot与回测 candles(qfq)
完全隔离:选股用未复权日线按 trade_date 全市场批量落地,避免污染回测复权缓存
智能选股直接读 candles 不复权底座market_daily 已退役);
daily_snapshot 存每日指标快照(估值/市值,选股过滤用)
"""
from datetime import date, datetime
@@ -236,29 +236,8 @@ class StockReference(Base):
)
class MarketDaily(Base):
"""全市场未复权日线(选股专用,与回测 candles(qfq) 隔离)。
单位沿用 Tushare 原始vol 手、amount 千元。
"""
__tablename__ = "market_daily"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
trade_date: Mapped[datetime] = mapped_column(DateTime, index=True)
ts_code: Mapped[str] = mapped_column(String(12), index=True)
open: Mapped[float] = mapped_column(Float)
high: Mapped[float] = mapped_column(Float)
low: Mapped[float] = mapped_column(Float)
close: Mapped[float] = mapped_column(Float)
pre_close: Mapped[float] = mapped_column(Float)
change: Mapped[float | None] = mapped_column(Float)
pct_chg: Mapped[float | None] = mapped_column(Float) # 日涨跌幅 %
vol: Mapped[float] = mapped_column(Float) # 手
amount: Mapped[float] = mapped_column(Float) # 千元
__table_args__ = (
UniqueConstraint("ts_code", "trade_date", name="uq_mkt_code_date"),
)
# market_daily(全市场未复权日线)已于选股改读 candles 后退役:
# ORM 模型已删,物理表暂留库中作冷备,确认无用后可手动 DROP TABLE market_daily。
class DailySnapshot(Base):

View File

@@ -352,6 +352,93 @@ class StockReferenceOut(BaseModel):
records: list[dict[str, str | float | None]] = Field(default_factory=list)
# ---------- 打板专题(主页,同花顺口径) ----------
class LimitStockOut(BaseModel):
"""涨跌停榜单行:三池共用,涨停池字段最全(炸板/跌停池仅价格类字段)。"""
ts_code: str
name: str | None = None
price: float | None = None # 收盘价(元)
pct_chg: float | None = None # 涨跌幅 %
tag: str | None = None # 涨停标签:首板 / 2天2板仅涨停池
status: str | None = None # 涨停状态:一字板 / 换手板 / N连板仅涨停池
lu_desc: str | None = None # 涨停原因(仅涨停池)
open_num: float | None = None # 打开次数
limit_amount_yi: float | None = None # 封单额(亿元,仅涨停池)
turnover_yi: float | None = None # 成交额(亿元,仅涨停池)
first_lu_time: str | None = None # 首次涨停时间
last_lu_time: str | None = None # 最后涨停时间(仅炸板池)
limit_up_suc_rate: float | None = None # 近一年封板率 %(仅涨停池)
class LimitLadderOut(BaseModel):
ts_code: str
name: str | None = None
nums: int # 连板数
class LimitBlockOut(BaseModel):
name: str | None = None # 同花顺概念板块名
days: float | None = None # 板块连涨天数
up_stat: str | None = None # 如「6天3板」
cons_nums: float | None = None # 连板家数
up_nums: float | None = None # 涨停家数
pct_chg: float | None = None # 板块涨跌 %
class LimitSummaryOut(BaseModel):
up_count: int = 0
broken_count: int = 0
down_count: int = 0
first_board_count: int = 0
max_ladder: LimitLadderOut | None = None
ladder_dist: list[dict[str, int]] = Field(default_factory=list) # [{nums, count}] 升序2板起
class LimitBoardResponse(BaseModel):
trade_date: str # YYYY-MM-DD
updated_at: str
summary: LimitSummaryOut
up: list[LimitStockOut] = Field(default_factory=list) # 涨停池(按封单额降序)
broken: list[LimitStockOut] = Field(default_factory=list) # 炸板池
down: list[LimitStockOut] = Field(default_factory=list) # 跌停池
ladder: list[LimitLadderOut] = Field(default_factory=list) # 连板天梯(连板数降序)
blocks: list[LimitBlockOut] = Field(default_factory=list) # 涨停最强板块
errors: list[str] = Field(default_factory=list)
# ---------- 概念板块THSths_index 列表 + ths_daily 行情 + ths_member 成分) ----------
class ThsBoardOut(BaseModel):
ts_code: str # 885835.TI / 700001.TI
name: str | None = None
type: str | None = None # N概念 I行业 TH主题 S特色 R地域 BB宽基 ST风格
count: float | None = None # 成分个数
list_date: str | None = None # YYYYMMDD
close: float | None = None # 板块指数收盘(当日快照)
pct_change: float | None = None # 涨跌幅 %
vol: float | None = None # 成交量(手)
turnover_rate: float | None = None # 换手率 %
class ThsBoardListResponse(BaseModel):
trade_date: str | None = None
updated_at: str | None = None
boards: list[ThsBoardOut] = Field(default_factory=list)
errors: list[str] = Field(default_factory=list)
class ThsMemberOut(BaseModel):
con_code: str # 成分股代码 000016.SZ
con_name: str | None = None
close: float | None = None # 现价candles 最新,北交所等无底座为空)
pct_chg: float | None = None # 涨跌幅 %(最新收盘 / 前收 - 1
class ThsBoardMembersResponse(BaseModel):
code: str
name: str | None = None
members: list[ThsMemberOut] = Field(default_factory=list)
# ---------- Auth ----------
class LoginRequest(BaseModel):
username: str = Field(min_length=1, max_length=64)

View File

@@ -10,6 +10,7 @@ daily 与 daily_basic 分步独立落库daily_basic 积分不足时快照仍
from __future__ import annotations
import asyncio
import logging
import time
from datetime import datetime, timedelta
@@ -20,9 +21,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
from .. import cache
from ..config import settings
from ..data.symbols import plain_code
from ..data.sync_utils import call_retry, get_pro_lazy
from ..models import AdjFactor, Candle, DailySnapshot, StockBasic, TradeCalendar
from .llm import ScreenerError
log = logging.getLogger(__name__)
# 进程内单例任务状态uvicorn --reload 单进程场景够用)
_sync_state: dict = {
"running": False,
@@ -40,32 +44,6 @@ _BATCH = 5000 # executemany 分批行数
# Tushare 积分/权限不足的特征文案daily_basic 常见门槛)
_PERM_MARKS = ("抱歉,您没有访问该项目权限", "积分", "权限")
# 频率超限特征(等待 62s 重试一次)
_RATE_MARKS = ("频率超限", "每分钟")
def _call_retry(fn, *args, **kwargs):
"""同步调用 tushare 接口;「每分钟」级频率超限等 62s 重试一次(小时级限频直接抛)。"""
try:
return fn(*args, **kwargs)
except Exception as e: # noqa: BLE001
msg = str(e)
if any(m in msg for m in _RATE_MARKS) and "小时" not in msg:
time.sleep(62)
return fn(*args, **kwargs)
raise
def _get_pro():
"""token 检查 + 返回 pro api 客户端(同步对象,调用需 to_thread 包裹)。
经 tushare_provider.get_pro 统一走镜像补丁15000 积分档 token 只认 quicksync
"""
if not settings.tushare_token:
raise ScreenerError("未配置 TUSHARE_TOKEN无法同步全市场数据backend/.env")
from ..data.tushare_provider import get_pro
return get_pro()
def _parse_d(s: str) -> datetime:
@@ -77,7 +55,7 @@ def _fetch_calendar_sync(pro) -> list[str]:
time.sleep(settings.screener_sync_interval)
end = (datetime.now() + timedelta(days=90)).strftime("%Y%m%d")
start = (datetime.now() - timedelta(days=550)).strftime("%Y%m%d")
cal = _call_retry(pro.trade_cal, exchange="SSE", start_date=start, end_date=end, is_open="1")
cal = call_retry(pro.trade_cal, exchange="SSE", start_date=start, end_date=end, is_open="1")
return sorted(cal["cal_date"].tolist())
@@ -112,7 +90,7 @@ async def _recent_trade_dates(session: AsyncSession, pro, days: int) -> list[str
def _fetch_daily(pro, d: str) -> list[dict]:
"""拉取某交易日全市场日线(未复权)。当日数据未生成(盘前/盘中)返回空。"""
time.sleep(settings.screener_sync_interval)
df = _call_retry(pro.daily, trade_date=d)
df = call_retry(pro.daily, trade_date=d)
if df is None or df.empty:
return []
rows = []
@@ -138,7 +116,7 @@ def _fetch_basic(pro, d: str) -> list[dict]:
"""
time.sleep(settings.screener_sync_interval)
try:
df = _call_retry(pro.daily_basic, trade_date=d)
df = call_retry(pro.daily_basic, trade_date=d)
except Exception as e: # noqa: BLE001
msg = str(e)
if any(m in msg for m in _PERM_MARKS):
@@ -169,7 +147,7 @@ def _fetch_basic(pro, d: str) -> list[dict]:
def _fetch_adj_factor(pro, d: str) -> list[dict]:
"""拉取某交易日全市场复权因子K线 bfq->qfq/hfq 本地换算的底座)。"""
time.sleep(settings.screener_sync_interval)
df = _call_retry(pro.adj_factor, trade_date=d)
df = call_retry(pro.adj_factor, trade_date=d)
if df is None or df.empty:
return []
return [
@@ -181,7 +159,7 @@ def _fetch_adj_factor(pro, d: str) -> list[dict]:
def _sync_stock_list_sync(pro) -> list[dict]:
"""拉取在市股票列表。"""
time.sleep(settings.screener_sync_interval)
df = _call_retry(pro.stock_basic, exchange="", list_status="L",
df = call_retry(pro.stock_basic, exchange="", list_status="L",
fields="ts_code,symbol,name,area,industry,market,exchange,list_status,list_date,delist_date")
rows = []
for _, r in df.iterrows():
@@ -232,6 +210,27 @@ async def _existing_candle_dates(session: AsyncSession) -> set[str]:
return {r[0].strftime("%Y%m%d") for r in res if r[0] is not None}
_UPSERT_CHUNK = 3000 # 单语句行数asyncpg 参数上限拆批)
async def _recent_day_counts(session: AsyncSession, dates: list[str]) -> dict[str, int]:
"""指定交易日在市股票的 candles 行数(半日数据自愈用)。
单条 GROUP BY 走 ts 索引范围扫,窗口 ≤15 日、代价可忽略。
"""
if not dates:
return {}
lo = _parse_d(min(dates))
hi = _parse_d(max(dates)) + timedelta(days=1)
rows = (await session.execute(
select(func.date(Candle.ts), func.count())
.where(Candle.timeframe == "1d", Candle.ts >= lo, Candle.ts < hi,
Candle.symbol.in_(select(StockBasic.symbol).where(StockBasic.list_status == "L")))
.group_by(func.date(Candle.ts))
)).all()
return {r[0].strftime("%Y%m%d"): int(r[1]) for r in rows if r[0] is not None}
async def _upsert_candle_day(session: AsyncSession, rows: list[dict], listed: set[str], d_str: str) -> None:
"""把某交易日全市场日线 upsert 进 candles不复权底座幂等
@@ -253,9 +252,10 @@ async def _upsert_candle_day(session: AsyncSession, rows: list[dict], listed: se
if not batch:
return
# on_conflict 语句整批渲染为占位符(非 executemanyasyncpg 单语句参数上限 32766
# 10 列 x 3000 行 = 30000 参数留出余量
for i in range(0, len(batch), 3000):
stmt = pg_insert(Candle).values(batch[i : i + 3000])
# 10 列 x 3000 行 = 30000 参数留出余量。分批只拆语句commit 在循环外 ——
# 单日一事务:写一半崩溃整日回滚,该日期语义上「未同步」,下次自然重拉(不留半日数据)
for i in range(0, len(batch), _UPSERT_CHUNK):
stmt = pg_insert(Candle).values(batch[i : i + _UPSERT_CHUNK])
stmt = stmt.on_conflict_do_update(
index_elements=["symbol", "timeframe", "ts"],
set_={
@@ -266,7 +266,7 @@ async def _upsert_candle_day(session: AsyncSession, rows: list[dict], listed: se
},
)
await session.execute(stmt)
await session.commit()
await session.commit()
async def _run_sync(days: int, force: bool) -> None:
@@ -277,7 +277,7 @@ async def _run_sync(days: int, force: bool) -> None:
from ..db import async_session # 延迟导入避免循环
try:
pro = await asyncio.to_thread(_get_pro)
pro = await asyncio.to_thread(get_pro_lazy)
# 1) 股票列表已有数据则跳过——stock_basic 低积分版限频 1 次/小时)
async with async_session() as session:
@@ -307,6 +307,15 @@ async def _run_sync(days: int, force: bool) -> None:
select(StockBasic.symbol).where(StockBasic.list_status == "L")
)).scalars()
)
if not force:
# 半日数据自愈(修单日原子化之前的历史残留):写入中途崩溃的日期
# 行数 ≈ 1 批3000显著低于完整日~5300。只查最近 15 个交易日
# ——实际风险区且窗口内上市数变化 <2%0.7 阈值安全;更老的日期不查
# (上市数变化会误判,历史缺口本就由 TDX 底座兜底)。
counts = await _recent_day_counts(session, [d for d in dates[:15] if d in have_daily])
if counts:
floor = max(_UPSERT_CHUNK + 1, int(max(counts.values()) * 0.7))
have_daily -= {d for d, n in counts.items() if n < floor}
todo = [d for d in dates if d not in have_daily]
_sync_state["total_days"] = len(todo)
_sync_state["done_days"] = 0
@@ -352,7 +361,7 @@ async def _run_sync(days: int, force: bool) -> None:
try:
await _refresh_stats(await cache.get_version("candles"))
except Exception: # noqa: BLE001 —— 预热失败只影响统计数字的新鲜度
pass
log.warning("统计缓存预热失败(下轮轮询会 SWR 重算)", exc_info=True)
_sync_state["step"] = "同步完成"
except Exception as e: # noqa: BLE001
_sync_state["error"] = f"同步失败:{str(e)[:300]}"
@@ -414,16 +423,9 @@ async def _heavy_stats(session: AsyncSession) -> dict:
async def _store_stats(ver: int, data: dict) -> None:
_status_stats_cache.update(at=time.time(), ver=ver, data=data)
# 写 Redis 后台执行,失败由 cache 层静默降级,不拖慢调用方
tasks = [
asyncio.create_task(cache.cache_set(
f"syncstats:v{ver}", data, ttl=settings.sync_stats_redis_ttl)),
asyncio.create_task(cache.cache_set(
_STATS_LAST_KEY, data, ttl=settings.sync_stats_redis_ttl)),
]
_stats_bg_tasks.update(tasks)
for t in tasks:
t.add_done_callback(_stats_bg_tasks.discard)
# 写 Redis 后台执行cache.set_bg 挂全局集合防 GC,失败由 cache 层静默降级
cache.set_bg(f"syncstats:v{ver}", data, ttl=settings.sync_stats_redis_ttl)
cache.set_bg(_STATS_LAST_KEY, data, ttl=settings.sync_stats_redis_ttl)
async def _refresh_stats(ver: int) -> None: