cirry/stock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

看股功能更新

Browse Source
This commit is contained in:
cirry
2026-08-16 20:20:59 +08:00
parent 63f8ed8b69
commit 65d2f54d1b
9 changed files with 296 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
backend/app/screener/engine.py
View File

@@ -1,6 +1,6 @@
"""选股执行引擎：SQL 快照预筛缩小范围 -> 逐股指标计算过滤。
性能：预筛在 SQLite 索引上完成（毫秒级）；指标阶段候选集通常数百~数千只 × ~90 根 bar，
性能：预筛在数据库索引上完成（毫秒级）；指标阶段候选集通常数百~数千只 × ~90 根 bar，
pandas 逐股计算（复用 app/indicators，指标按 (族, 参数) 去重计算），秒级完成。
"""
from __future__ import annotations
@@ -14,7 +14,8 @@ from sqlalchemy import and_, func, not_, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from .. import indicators as ind
from ..models import DailySnapshot, MarketDaily, StockBasic
from ..data.symbols import plain_code
from ..models import Candle, DailySnapshot, StockBasic
from ..schemas import IndicatorCondition, ScreenConditions, SnapshotCondition
# 快照字段 -> (DB 列, 中文标签, LLM 值 -> DB 值的换算乘数)
@@ -25,7 +26,7 @@ SNAPSHOT_FIELDS: dict[str, tuple[str, str, float]] = {
"pe_ttm": ("pe_ttm", "市盈率TTM", 1.0),
"pb": ("pb", "市净率", 1.0),
"turnover_rate": ("turnover_rate", "换手率%", 1.0),
"close": ("close", "最新价", 1.0), # 实际取 market_daily.close，无需快照表
"close": ("close", "最新价", 1.0), # 实际取 candles 最新 bar，无需快照表
}
@@ -198,7 +199,7 @@ def _snapshot_clause(cond: SnapshotCondition):
if cond.field not in SNAPSHOT_FIELDS:
raise ValueError(f"未知快照字段: {cond.field}")
col_name, _, scale = SNAPSHOT_FIELDS[cond.field]
col = MarketDaily.close if cond.field == "close" else getattr(DailySnapshot, col_name)
col = Candle.close if cond.field == "close" else getattr(DailySnapshot, col_name)
lo = cond.value * scale
hi = (cond.value2 * scale) if cond.op == "between" and cond.value2 is not None else None
if cond.op == "gt":
@@ -219,17 +220,22 @@ _CAND_COLS = ["ts_code", "name", "close", "pct_chg",
async def _prefilter(session: AsyncSession, conds: ScreenConditions, target_date: datetime) -> pd.DataFrame:
"""最新交易日截面预筛：快照条件 + 排除项 + 名称/收盘价。返回候选 DataFrame。"""
"""最新交易日截面预筛：candles(全量不复权底座) + stock_basic + daily_snapshot。
只取 target_date 当日有交易的股票（停牌股无当日 bar，自然排除，与原先 market_daily
的 trade_date == target_date 行为一致）。pct_chg 用前一交易日收盘价计算。
"""
snap_date = await session.scalar(select(func.max(DailySnapshot.trade_date))) or target_date
stmt = (
select(
MarketDaily.ts_code, StockBasic.name, MarketDaily.close, MarketDaily.pct_chg,
Candle.symbol, StockBasic.ts_code, StockBasic.name, Candle.close,
DailySnapshot.total_mv, DailySnapshot.circ_mv,
DailySnapshot.pe_ttm, DailySnapshot.pb, DailySnapshot.turnover_rate,
)
.join(StockBasic, StockBasic.ts_code == MarketDaily.ts_code)
.outerjoin(DailySnapshot, and_(DailySnapshot.ts_code == MarketDaily.ts_code,
DailySnapshot.trade_date == target_date))
.where(MarketDaily.trade_date == target_date)
.join(StockBasic, StockBasic.symbol == Candle.symbol)
.outerjoin(DailySnapshot, and_(DailySnapshot.ts_code == StockBasic.ts_code,
DailySnapshot.trade_date == snap_date))
.where(Candle.timeframe == "1d", Candle.ts == target_date)
)
if conds.exclude_delisted:
@@ -237,13 +243,39 @@ async def _prefilter(session: AsyncSession, conds: ScreenConditions, target_date
if conds.exclude_st:
stmt = stmt.where(not_(or_(StockBasic.name.like("%ST%"), StockBasic.name.like("%退%"))))
if conds.exclude_bj:
stmt = stmt.where(not_(MarketDaily.ts_code.like("%.BJ")))
stmt = stmt.where(not_(StockBasic.ts_code.like("%.BJ")))
for c in conds.snapshot:
stmt = stmt.where(_snapshot_clause(c))
rows = (await session.execute(stmt)).all()
return pd.DataFrame(rows, columns=_CAND_COLS)
df = pd.DataFrame(rows, columns=["symbol", "ts_code", "name", "close",
"total_mv", "circ_mv", "pe_ttm", "pb", "turnover_rate"])
if df.empty:
return df[_CAND_COLS]
# pct_chg：candles 无现成涨跌幅列，用前一交易日的收盘价计算
prev_dt = await session.scalar(
select(func.max(Candle.ts)).where(Candle.timeframe == "1d", Candle.ts < target_date)
)
prev_map: dict[str, float] = {}
if prev_dt is not None:
pr = await session.execute(
select(Candle.symbol, Candle.close).where(
Candle.timeframe == "1d", Candle.ts == prev_dt,
Candle.symbol.in_(df["symbol"].tolist()),
)
)
prev_map = {r.symbol: r.close for r in pr}
prev = df["symbol"].map(prev_map)
def _pct(c, p) -> float | None:
if p is None or p != p or float(p) == 0:
return None
return (float(c) / float(p) - 1) * 100
df["pct_chg"] = [_pct(c, p) for c, p in zip(df["close"], prev)]
return df[_CAND_COLS]
# ---------- 主流程 ----------
@@ -260,18 +292,30 @@ def _max_needed_bars(conds: ScreenConditions) -> int:
async def _load_bars(session: AsyncSession, ts_codes: list[str],
target_date: datetime, min_date: datetime) -> pd.DataFrame:
"""载入候选股的 K 线窗口。候选 <= 2000 用 IN 精确圈定；否则拉全窗口再 pandas 过滤。"""
"""载入候选股的 K 线窗口（candles 全量不复权底座）。
候选 <= 2000 用 IN 精确圈定；否则拉全窗口再 pandas 过滤。
ts_code 按候选集映射回 symbol 查询，pct_chg 用每股收盘价环比计算。
"""
symbols = [plain_code(t) for t in ts_codes]
stmt = select(
MarketDaily.ts_code, MarketDaily.trade_date, MarketDaily.open, MarketDaily.high,
MarketDaily.low, MarketDaily.close, MarketDaily.pct_chg,
).where(MarketDaily.trade_date >= min_date, MarketDaily.trade_date <= target_date)
if len(ts_codes) <= 2000:
stmt = stmt.where(MarketDaily.ts_code.in_(set(ts_codes)))
Candle.symbol, Candle.ts, Candle.open, Candle.high, Candle.low, Candle.close,
).where(Candle.timeframe == "1d", Candle.ts >= min_date, Candle.ts <= target_date)
if len(symbols) <= 2000:
stmt = stmt.where(Candle.symbol.in_(set(symbols)))
rows = (await session.execute(stmt)).all()
df = pd.DataFrame(rows, columns=["ts_code", "trade_date", "open", "high", "low", "close", "pct_chg"])
if not df.empty and len(ts_codes) > 2000:
df = df[df["ts_code"].isin(set(ts_codes))]
return df.sort_values(["ts_code", "trade_date"]).reset_index(drop=True)
df = pd.DataFrame(rows, columns=["symbol", "trade_date", "open", "high", "low", "close"])
if not df.empty and len(symbols) > 2000:
df = df[df["symbol"].isin(set(symbols))]
df = df.sort_values(["symbol", "trade_date"]).reset_index(drop=True)
if df.empty:
df["ts_code"] = pd.Series(dtype=object)
df["pct_chg"] = pd.Series(dtype=object)
else:
df["pct_chg"] = df.groupby("symbol")["close"].pct_change() * 100
ts_map = {plain_code(t): t for t in ts_codes}
df["ts_code"] = df["symbol"].map(ts_map)
return df[["ts_code", "trade_date", "open", "high", "low", "close", "pct_chg"]]
def _f(v) -> float | None:
@@ -303,7 +347,9 @@ async def run_screen(session: AsyncSession, conds: ScreenConditions, limit: int)
if not conds.indicator and not conds.snapshot:
raise ValueError("筛选条件为空")
target_date = await session.scalar(select(func.max(MarketDaily.trade_date)))
target_date = await session.scalar(
select(func.max(Candle.ts)).where(Candle.timeframe == "1d")
)
if target_date is None:
raise DataNotReadyError("全市场数据未同步：请先在选股页点击「同步市场数据」")
@@ -330,10 +376,11 @@ async def run_screen(session: AsyncSession, conds: ScreenConditions, limit: int)
# 纯快照条件：预筛结果即命中
items = [_item_from_row(row) | {"indicators": {}} for _, row in cand.iterrows()]
else:
# 圈定 K 线窗口：按已同步交易日序列回溯 needed 根
# 圈定 K 线窗口：按 candles 全量交易日序列回溯 needed 根（不再受同步窗口限制）
need = _max_needed_bars(conds)
dates_res = await session.execute(
select(MarketDaily.trade_date).distinct().order_by(MarketDaily.trade_date.desc()).limit(need)
select(Candle.ts).where(Candle.timeframe == "1d").distinct()
.order_by(Candle.ts.desc()).limit(need)
)
min_date = min(r[0] for r in dates_res)
bars = await _load_bars(session, cand["ts_code"].tolist(), target_date, min_date)

115
backend/app/screener/market_sync.py
View File

@@ -1,10 +1,11 @@
"""全市场数据同步（选股专用，未复权；与回测 candles 表隔离）。
"""全市场数据同步（未复权，写入 candles 全量底座）。
设计：trade_cal 取近 N 个交易日 -> 逐日 pro.daily(trade_date=...) / pro.daily_basic(trade_date=...)
一次返回全市场当日数据 -> 按 trade_date 删旧插新批量入库（幂等）。
设计：trade_cal 取近 N 个交易日 -> 逐日 pro.daily(trade_date=...) 一次返回全市场当日数据
-> upsert 进 candles（不复权底座，ON CONFLICT 幂等）；daily_basic 仅同步最新交易日到
DailySnapshot（市值/PE/PB/换手率等截面字段）。
同步为进程内后台任务（MVP 不引入任务队列），前端轮询 /api/screener/sync/status。
daily 与 daily_basic 分步独立落库：daily_basic 积分不足时日线仍可用，错误写入状态不中断任务。
daily 与 daily_basic 分步独立落库：daily_basic 积分不足时快照仍可用，错误写入状态不中断任务。
"""
from __future__ import annotations
@@ -13,10 +14,13 @@ import time
from datetime import datetime, timedelta
from sqlalchemy import delete, func, insert, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import AsyncSession
from .. import cache
from ..config import settings
from ..models import AdjFactor, DailySnapshot, MarketDaily, StockBasic, TradeCalendar
from ..data.symbols import plain_code
from ..models import AdjFactor, Candle, DailySnapshot, StockBasic, TradeCalendar
from .llm import ScreenerError
# 进程内单例任务状态（uvicorn --reload 单进程场景够用）
@@ -211,6 +215,48 @@ async def _replace_day(session: AsyncSession, model, rows: list[dict], d_str: st
await session.commit()
async def _existing_candle_dates(session: AsyncSession) -> set[str]:
"""candles 表已落库的交易日集合（YYYYMMDD 字符串，便于比对）。"""
res = await session.execute(
select(func.distinct(func.date(Candle.ts))).where(Candle.timeframe == "1d")
)
return {r[0].strftime("%Y%m%d") for r in res if r[0] is not None}
async def _upsert_candle_day(session: AsyncSession, rows: list[dict], listed: set[str], d_str: str) -> None:
"""把某交易日全市场日线 upsert 进 candles（不复权底座，幂等）。
rows 来自 _fetch_daily（ts_code/vol手/amount千元）；只写 stock_basic 在市股票，
与 TDX 底座口径一致；amount 已有（TDX 回补）时保留旧值。
"""
batch = [
{
"symbol": plain_code(r["ts_code"]), "timeframe": "1d",
"ts": _parse_d(d_str),
"open": r["open"], "high": r["high"], "low": r["low"], "close": r["close"],
"volume": r["vol"] * 100.0, # 手 -> 股
"amount": (r["amount"] * 1000.0) if r["amount"] is not None else None, # 千元 -> 元
"turnover": None, # 换手率另由 daily_basic 快照维护
}
for r in rows
if plain_code(r["ts_code"]) in listed
]
if not batch:
return
stmt = pg_insert(Candle).values(batch)
stmt = stmt.on_conflict_do_update(
index_elements=["symbol", "timeframe", "ts"],
set_={
"open": stmt.excluded.open, "high": stmt.excluded.high,
"low": stmt.excluded.low, "close": stmt.excluded.close,
"volume": stmt.excluded.volume,
"amount": func.coalesce(Candle.amount, stmt.excluded.amount),
},
)
await session.execute(stmt)
await session.commit()
async def _run_sync(days: int, force: bool) -> None:
"""后台任务主体：stock_basic -> 逐日日线 -> 最新交易日快照。异常写状态。
@@ -239,10 +285,16 @@ async def _run_sync(days: int, force: bool) -> None:
else:
raise
# 2) 逐交易日日线（增量；当日未生成则跳过）
# 2) 逐交易日全市场日线 -> candles（增量；当日未生成则跳过）
async with async_session() as session:
dates = await _recent_trade_dates(session, pro, days)
have_daily = set() if force else await _existing_dates(session, MarketDaily)
have_daily = set() if force else await _existing_candle_dates(session)
# 在市股票集合，限定写入范围（与 TDX 底座口径一致）
listed = set(
(await session.execute(
select(StockBasic.symbol).where(StockBasic.list_status == "L")
)).scalars()
)
todo = [d for d in dates if d not in have_daily]
_sync_state["total_days"] = len(todo)
_sync_state["done_days"] = 0
@@ -252,7 +304,7 @@ async def _run_sync(days: int, force: bool) -> None:
daily_rows = await asyncio.to_thread(_fetch_daily, pro, d)
if daily_rows: # 盘前/盘中等未生成数据的日期直接跳过
async with async_session() as session:
await _replace_day(session, MarketDaily, daily_rows, d)
await _upsert_candle_day(session, daily_rows, listed, d)
_sync_state["done_days"] += 1
# 2.5) 复权因子（与日线同窗口增量；历史全量由 scripts/backfill_adj_factor.py 回补）
@@ -266,9 +318,9 @@ async def _run_sync(days: int, force: bool) -> None:
await _replace_day(session, AdjFactor, adj_rows, d)
# 3) 最新「有数据」交易日的快照（daily_basic，仅 1 次调用）
# 用 market_daily 实际最大交易日（今天的数据收盘后才生成，日历最新日会拉到空）
# 用 candles 实际最大交易日（今天的数据收盘后才生成，日历最新日会拉到空）
async with async_session() as session:
latest_dt = await session.scalar(select(func.max(MarketDaily.trade_date)))
latest_dt = await session.scalar(select(func.max(Candle.ts)))
latest = latest_dt.strftime("%Y%m%d") if latest_dt else None
if latest:
async with async_session() as session:
@@ -280,6 +332,8 @@ async def _run_sync(days: int, force: bool) -> None:
async with async_session() as session:
await _replace_day(session, DailySnapshot, basic_rows, latest)
# candles/复权因子已更新：作废旧 K 线预览缓存（键含版本号，自增即全体失效）
await cache.bump_version("candles")
_sync_state["step"] = "同步完成"
except Exception as e: # noqa: BLE001
_sync_state["error"] = f"同步失败：{str(e)[:300]}"
@@ -303,19 +357,42 @@ async def start_sync(session: AsyncSession, days: int, force: bool) -> dict:
return dict(_sync_state)
# candles 是千万行表，count 较重；前端每 2s 轮询状态，需 TTL 缓存降载
_status_stats_cache: dict = {"at": 0.0, "data": None}
_STATS_TTL = 30.0
async def _db_stats(session: AsyncSession) -> dict:
"""candles/快照/股票列表实况（30s TTL 缓存）。"""
now = time.time()
if _status_stats_cache["data"] is not None and now - _status_stats_cache["at"] < _STATS_TTL:
return _status_stats_cache["data"]
stocks = int(await session.scalar(select(func.count()).select_from(StockBasic)) or 0)
daily_rows = int(await session.scalar(select(func.count()).select_from(Candle)) or 0)
snap_rows = int(await session.scalar(select(func.count()).select_from(DailySnapshot)) or 0)
last_daily = await session.scalar(
select(func.max(Candle.ts)).where(Candle.timeframe == "1d")
)
n_dates = int(await session.scalar(
select(func.count(func.distinct(func.date(Candle.ts)))).where(Candle.timeframe == "1d")
) or 0)
data = {
"stocks": stocks, "daily_rows": daily_rows, "snapshot_rows": snap_rows,
"last_daily": last_daily, "dates": n_dates,
}
_status_stats_cache.update(at=now, data=data)
return data
async def get_sync_status(session: AsyncSession) -> dict:
"""合并任务状态 + DB 实况（最新交易日/行数/ready 标志），与 ScreenerSyncStatus DTO 对齐。"""
stocks = int(await session.scalar(select(func.count()).select_from(StockBasic)) or 0)
daily_rows = int(await session.scalar(select(func.count()).select_from(MarketDaily)) or 0)
snap_rows = int(await session.scalar(select(func.count()).select_from(DailySnapshot)) or 0)
last_daily = await session.scalar(select(func.max(MarketDaily.trade_date)))
n_dates = int(await session.scalar(select(func.count(func.distinct(MarketDaily.trade_date)))) or 0)
stats = await _db_stats(session)
status = dict(_sync_state)
status.update({
"stats": {"stocks": stocks, "daily_rows": daily_rows, "snapshot_rows": snap_rows, "dates": n_dates},
"last_trade_date": last_daily,
"stats": {"stocks": stats["stocks"], "daily_rows": stats["daily_rows"],
"snapshot_rows": stats["snapshot_rows"], "dates": stats["dates"]},
"last_trade_date": stats["last_daily"],
"last_synced_at": _sync_state.get("finished_at") or _sync_state.get("started_at"),
"ready": daily_rows > 0,
"ready": stats["daily_rows"] > 0,
})
return status