cirry/stock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

提交

Browse Source
This commit is contained in:
cirry
2026-09-09 11:35:02 +08:00
parent bc1c72d558
commit d656c05b3d
35 changed files with 711 additions and 3194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
backend/app/data/company.py
View File

@@ -9,7 +9,7 @@ from __future__ import annotations
import asyncio
import time
from datetime import datetime, timedelta, timezone
from datetime import datetime
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert as pg_insert
@@ -18,12 +18,10 @@ from sqlalchemy.ext.asyncio import AsyncSession
from ..config import settings
from ..db import async_session
from ..models import StockCompany
from .sync_utils import call_retry, f_clean, fresh, get_pro_lazy, s_clean, utcnow
_REFRESH_DAYS = 30
# 频率超限特征（等待 62s 重试一次；与 screener.market_sync / data.etf_sync._call_retry 同款语义）
_RATE_MARKS = ("频率超限", "每分钟")
# 显式列出全部字段：introduction/office/main_business/business_scope 文档标注默认不显示，
# 不传 fields 时 tushare 不返回这四列（实测 000001.SZ）
_FIELDS = (
@@ -32,87 +30,38 @@ _FIELDS = (
"employees,main_business,business_scope"
)
_pro = None # 惰性单例（get_pro 每次都 ts.set_token 写文件，没必要重复）
def _get_pro():
if not settings.tushare_token:
raise RuntimeError("未配置 TUSHARE_TOKEN，无法拉取公司简介（backend/.env）")
global _pro
if _pro is None:
from .tushare_provider import get_pro
_pro = get_pro()
return _pro
def _call_retry(fn, *args, **kwargs):
"""同步调用 tushare 接口；「每分钟」级频率超限等 62s 重试一次。"""
try:
return fn(*args, **kwargs)
except Exception as e: # noqa: BLE001
msg = str(e)
if any(m in msg for m in _RATE_MARKS) and "小时" not in msg:
time.sleep(62)
return fn(*args, **kwargs)
raise
def _utcnow() -> datetime:
return datetime.now(timezone.utc).replace(tzinfo=None)
def _fresh(updated_at: datetime | None) -> bool:
return updated_at is not None and updated_at >= _utcnow() - timedelta(days=_REFRESH_DAYS)
def _s(v) -> str | None:
"""pandas NaN / 空串 / None -> None，其余 strip。"""
if v is None or (isinstance(v, float) and v != v):
return None
s = str(v).strip()
return s or None
def _f(v) -> float | None:
try:
f = float(v)
except (TypeError, ValueError):
return None
return None if f != f else f # NaN -> None
def _i(v) -> int | None:
f = _f(v)
f = f_clean(v)
return None if f is None else int(f)
def fetch_company_sync(ts_code: str) -> dict | None:
"""同步拉单只公司简介（需在 to_thread 里跑）；返回行 dict，无此股返回 None。"""
time.sleep(settings.screener_sync_interval)
df = _call_retry(_get_pro().stock_company, ts_code=ts_code, fields=_FIELDS)
df = call_retry(get_pro_lazy().stock_company, ts_code=ts_code, fields=_FIELDS)
if df is None or df.empty:
return None
r = df.iloc[0]
return {
"ts_code": ts_code,
"com_name": _s(r.get("com_name")),
"com_id": _s(r.get("com_id")),
"chairman": _s(r.get("chairman")),
"manager": _s(r.get("manager")),
"secretary": _s(r.get("secretary")),
"reg_capital": _f(r.get("reg_capital")),
"setup_date": _s(r.get("setup_date")),
"province": _s(r.get("province")),
"city": _s(r.get("city")),
"introduction": _s(r.get("introduction")),
"website": _s(r.get("website")),
"email": _s(r.get("email")),
"office": _s(r.get("office")),
"com_name": s_clean(r.get("com_name")),
"com_id": s_clean(r.get("com_id")),
"chairman": s_clean(r.get("chairman")),
"manager": s_clean(r.get("manager")),
"secretary": s_clean(r.get("secretary")),
"reg_capital": f_clean(r.get("reg_capital")),
"setup_date": s_clean(r.get("setup_date")),
"province": s_clean(r.get("province")),
"city": s_clean(r.get("city")),
"introduction": s_clean(r.get("introduction")),
"website": s_clean(r.get("website")),
"email": s_clean(r.get("email")),
"office": s_clean(r.get("office")),
"employees": _i(r.get("employees")),
"main_business": _s(r.get("main_business")),
"business_scope": _s(r.get("business_scope")),
"updated_at": _utcnow(),
"main_business": s_clean(r.get("main_business")),
"business_scope": s_clean(r.get("business_scope")),
"updated_at": utcnow(),
}
@@ -151,7 +100,7 @@ async def get_company(session: AsyncSession, ts_code: str) -> dict | None:
"""
row = (await session.execute(
select(StockCompany).where(StockCompany.ts_code == ts_code))).scalar_one_or_none()
if row is not None and _fresh(row.updated_at):
if row is not None and fresh(row.updated_at, _REFRESH_DAYS):
return _row_dict(row) if row.com_name is not None else None # 墓碑 -> None
# 释放请求会话持有的连接：后面可能隔着 1-2s 的 tushare 调用，别长占连接池。
# 用 close() 而非 rollback()：rollback 会把会话身份映射里的实例全部 expire——
@@ -165,7 +114,7 @@ async def get_company(session: AsyncSession, ts_code: str) -> dict | None:
async with async_session() as s2: # 锁内重读 + 写入走新会话
row = (await s2.execute(
select(StockCompany).where(StockCompany.ts_code == ts_code))).scalar_one_or_none()
if row is not None and _fresh(row.updated_at):
if row is not None and fresh(row.updated_at, _REFRESH_DAYS):
return _row_dict(row) if row.com_name is not None else None
try:
fetched = await asyncio.to_thread(fetch_company_sync, ts_code)
@@ -174,5 +123,5 @@ async def get_company(session: AsyncSession, ts_code: str) -> dict | None:
if row is not None and row.com_name is not None:
return _row_dict(row)
raise
await _upsert(s2, fetched or {"ts_code": ts_code, "updated_at": _utcnow()})
await _upsert(s2, fetched or {"ts_code": ts_code, "updated_at": utcnow()})
return fetched

39
backend/app/data/etf_sync.py
View File

@@ -20,7 +20,7 @@ from __future__ import annotations
import asyncio
import time
from datetime import datetime, timedelta, timezone
from datetime import datetime, timedelta
from sqlalchemy import delete, func, select, text
from sqlalchemy.dialects.postgresql import insert as pg_insert
@@ -29,6 +29,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from .. import cache
from ..config import settings
from . import etf_provider
from .sync_utils import call_retry, get_pro_lazy, utcnow
# 进程内单例任务状态（uvicorn 单进程场景够用）
_state: dict = {
@@ -46,34 +47,6 @@ _lock = asyncio.Lock()
_BATCH = 3000 # upsert 分批行数（asyncpg 单语句参数上限 32766，10 列/行）
# fund_daily 返回全市场基金 ~2100 行，一天一批远小于上限
# 频率超限特征（等待 62s 重试一次；与 screener.market_sync._call_retry 同款语义）
_RATE_MARKS = ("频率超限", "每分钟")
def _call_retry(fn, *args, **kwargs):
"""同步调用 tushare 接口；「每分钟」级频率超限等 62s 重试一次。"""
try:
return fn(*args, **kwargs)
except Exception as e: # noqa: BLE001
msg = str(e)
if any(m in msg for m in _RATE_MARKS) and "小时" not in msg:
time.sleep(62)
return fn(*args, **kwargs)
raise
def _get_pro():
"""token 检查 + 返回 pro api 客户端（同步对象，调用需 to_thread 包裹）。"""
if not settings.tushare_token:
raise RuntimeError("未配置 TUSHARE_TOKEN，无法同步 ETF 日线（backend/.env）")
from .tushare_provider import get_pro
return get_pro()
def _utcnow() -> datetime:
return datetime.now(timezone.utc).replace(tzinfo=None)
def _parse_d(s: str) -> datetime:
return datetime.strptime(str(s), "%Y%m%d")
@@ -85,7 +58,7 @@ async def _sync_spot(session: AsyncSession) -> int:
async with etf_provider.new_client() as client:
rows = await etf_provider.fetch_etf_spot(client)
now = _utcnow()
now = utcnow()
stmt = pg_insert(EtfBasic).values([{**r, "updated_at": now} for r in rows])
stmt = stmt.on_conflict_do_update(
index_elements=["ts_code"],
@@ -108,7 +81,7 @@ async def _sync_spot(session: AsyncSession) -> int:
def _fetch_day_sync(pro, d: str) -> list[dict]:
"""拉某交易日全市场场内基金日线（fund_daily；未生成的日期返回空）。"""
time.sleep(settings.screener_sync_interval)
df = _call_retry(pro.fund_daily, trade_date=d)
df = call_retry(pro.fund_daily, trade_date=d)
if df is None or df.empty:
return []
rows = []
@@ -128,7 +101,7 @@ def _fetch_day_sync(pro, d: str) -> list[dict]:
def _fetch_symbol_sync(pro, ts_code: str, start: str | None, end: str | None) -> list[dict]:
"""按 ts_code 增量/全量拉单只 ETF 日线（start=None 即上市以来全量）。"""
time.sleep(settings.screener_sync_interval)
df = _call_retry(pro.fund_daily, ts_code=ts_code, start_date=start, end_date=end)
df = call_retry(pro.fund_daily, ts_code=ts_code, start_date=start, end_date=end)
if df is None or df.empty:
return []
df = df.sort_values("trade_date")
@@ -209,7 +182,7 @@ async def _run_sync(full: bool) -> None:
from ..models import Candle, EtfBasic, TradeCalendar
try:
pro = await asyncio.to_thread(_get_pro)
pro = await asyncio.to_thread(get_pro_lazy)
# 1) 快照 -> etf_basic
_state["step"] = "正在拉取 ETF 列表"

3
backend/app/data/fetcher.py
View File

@@ -113,6 +113,9 @@ async def sync_symbol(
)
await session.execute(stmt)
await session.commit()
# 作废 candles 相关读缓存（preview 等）——不 bump 的话旧版本号的缓存要等 TTL 自然过期
from .. import cache
await cache.bump_version("candles")
return {"symbol": code, "bars": len(bars), "source": used}

54
backend/app/data/index_global.py
View File

@@ -13,13 +13,13 @@ from __future__ import annotations
import asyncio
import json
import math
import time
from datetime import date, datetime, timedelta
from .. import cache
from ..config import settings
from ..domain import Bar
from .sync_utils import d8_iso, f_clean
# ---- 静态元数据表（tushare index_global 支持的全部 21 个指数，展示顺序即文档顺序）----
# region: americas 美洲 / europe 欧洲 / asia 亚太（含港股与富时A50）
@@ -82,22 +82,6 @@ class GlobalIndexError(RuntimeError):
"""全部国际指数都拉不到（token/网络故障）——接口层转 503。"""
def _f(v) -> float | None:
"""pandas 值 -> float；NaN/None -> None。"""
if v is None:
return None
try:
f = float(v)
except (TypeError, ValueError):
return None
return None if math.isnan(f) else f
def _d(v) -> str | None:
"""YYYYMMDD -> 'YYYY-MM-DD'（字符串便于 JSON 缓存）。"""
return datetime.strptime(str(v), "%Y%m%d").date().isoformat() if v else None
def is_cn_index(code: str) -> bool:
return "." in code
@@ -128,14 +112,14 @@ def _fetch_quote_sync(pro, ts_code: str) -> dict:
tail = df.tail(_SPARK_DAYS)
last = df.iloc[-1]
return {
"close": _f(last["close"]),
"change": _f(last.get("change")),
"pct_chg": _f(last.get("pct_chg")),
"open": _f(last.get("open")),
"high": _f(last.get("high")),
"low": _f(last.get("low")),
"pre_close": _f(last.get("pre_close")),
"trade_date": _d(last["trade_date"]),
"close": f_clean(last["close"]),
"change": f_clean(last.get("change")),
"pct_chg": f_clean(last.get("pct_chg")),
"open": f_clean(last.get("open")),
"high": f_clean(last.get("high")),
"low": f_clean(last.get("low")),
"pre_close": f_clean(last.get("pre_close")),
"trade_date": d8_iso(last["trade_date"]),
"spark": [round(float(c), 4) for c in tail["close"]],
"spark_dates": [str(d) for d in tail["trade_date"]],
}
@@ -249,8 +233,8 @@ def _fetch_global_bars_sync(ts_code: str) -> list[Bar]:
df = pd.concat(frames).drop_duplicates(subset="trade_date").sort_values("trade_date")
bars: list[Bar] = []
for _, r in df.iterrows():
vol = _f(r.get("vol"))
amt = _f(r.get("amount"))
vol = f_clean(r.get("vol"))
amt = f_clean(r.get("amount"))
bars.append(
Bar(
ts=datetime.strptime(str(r["trade_date"]), "%Y%m%d"),
@@ -318,9 +302,9 @@ def _fetch_basic_sync(ts_code: str) -> dict:
"market": r.get("market"),
"publisher": r.get("publisher"),
"category": r.get("category"),
"base_date": _d(r.get("base_date")),
"base_point": _f(r.get("base_point")),
"list_date": _d(r.get("list_date")),
"base_date": d8_iso(r.get("base_date")),
"base_point": f_clean(r.get("base_point")),
"list_date": d8_iso(r.get("list_date")),
}
@@ -354,10 +338,10 @@ def _fetch_valuation_sync(ts_code: str, days: int) -> list[dict]:
rows = []
for _, r in df.sort_values("trade_date").iterrows():
rows.append({
"trade_date": _d(r["trade_date"]),
"pe": _f(r.get("pe")), "pe_ttm": _f(r.get("pe_ttm")), "pb": _f(r.get("pb")),
"turnover_rate": _f(r.get("turnover_rate")),
"total_mv": _f(r.get("total_mv")), "float_mv": _f(r.get("float_mv")),
"trade_date": d8_iso(r["trade_date"]),
"pe": f_clean(r.get("pe")), "pe_ttm": f_clean(r.get("pe_ttm")), "pb": f_clean(r.get("pb")),
"turnover_rate": f_clean(r.get("turnover_rate")),
"total_mv": f_clean(r.get("total_mv")), "float_mv": f_clean(r.get("float_mv")),
})
return rows
@@ -395,7 +379,7 @@ def _fetch_weights_sync(ts_code: str) -> dict | None:
latest_date = df.iloc[0]["trade_date"]
rows = df[df["trade_date"] == latest_date]
return {
"trade_date": _d(latest_date),
"trade_date": d8_iso(latest_date),
"total": int(len(rows)),
"items": [
{"con_code": str(r["con_code"]), "weight": round(float(r["weight"]), 4)}

34
backend/app/data/market_overview.py
View File

@@ -17,7 +17,6 @@
from __future__ import annotations
import asyncio
import math
import time
from datetime import date, datetime, timedelta
@@ -26,6 +25,7 @@ import pandas as pd
from .. import cache
from ..config import settings
from .sync_utils import d8_iso, f_clean
# (tushare代码, 名称, 地区, 腾讯符号) —— 展示顺序即列表顺序
# 首页聚焦中美（港股/国际指数在 /indexes 国际指数页）；标普500 腾讯符号是 s_usINX（不是 s_usSPX）
@@ -57,22 +57,6 @@ class MarketOverviewError(RuntimeError):
"""所有指数都拉不到（token/网络故障）——接口层转 503。"""
def _f(v) -> float | None:
"""pandas 值 -> float；NaN/None -> None（否则 JSON 里会出现 NaN）。"""
if v is None:
return None
try:
f = float(v)
except (TypeError, ValueError):
return None
return None if math.isnan(f) else f
def _d(v) -> str | None:
"""YYYYMMDD -> 'YYYY-MM-DD'（字符串便于 JSON 缓存；pydantic 响应模型自动 coerce）。"""
return datetime.strptime(str(v), "%Y%m%d").date().isoformat() if v else None
def _get_pro():
if not settings.tushare_token:
raise MarketOverviewError("未配置 TUSHARE_TOKEN，无法获取大盘行情（backend/.env）")
@@ -165,10 +149,10 @@ def _quote_from_df(df: pd.DataFrame) -> dict | None:
tail = df.tail(_SPARK_DAYS)
last = df.iloc[-1]
return {
"close": _f(last["close"]),
"change": _f(last.get("change")),
"pct_chg": _f(last.get("pct_chg")),
"trade_date": _d(last["trade_date"]),
"close": f_clean(last["close"]),
"change": f_clean(last.get("change")),
"pct_chg": f_clean(last.get("pct_chg")),
"trade_date": d8_iso(last["trade_date"]),
"spark": [round(float(c), 4) for c in tail["close"]],
"spark_dates": [str(d) for d in tail["trade_date"]],
}
@@ -193,10 +177,10 @@ def _fetch_stats_sync(pro) -> dict | None:
if sh_m is None or sz_m is None:
return None
# 两边各自取最新，日期不一致时以较旧一天为准凑齐口径（罕见，通常同日）
d = min(_d(sh_m["trade_date"]), _d(sz_m["trade_date"]))
d = min(d8_iso(sh_m["trade_date"]), d8_iso(sz_m["trade_date"]))
def _sum(col: str) -> float | None:
a, b = _f(sh_m.get(col)), _f(sz_m.get(col))
a, b = f_clean(sh_m.get(col)), f_clean(sz_m.get(col))
return None if a is None or b is None else round(a + b, 2)
return {
@@ -204,7 +188,7 @@ def _fetch_stats_sync(pro) -> dict | None:
"total_mv": _sum("total_mv"),
"float_mv": _sum("float_mv"),
"amount": _sum("amount"),
"turnover": _f(sh_m.get("tr")), # 换手率仅沪市有，展示口径注明沪市
"turnover": f_clean(sh_m.get("tr")), # 换手率仅沪市有，展示口径注明沪市
}
@@ -222,7 +206,7 @@ def _fetch_amount_history_sync(pro) -> list[dict]:
if len(common) == 0:
return []
total = (sh_m[common] + sz_m[common]).sort_index()
return [{"date": _d(d), "amount": round(float(v), 2)} for d, v in total.tail(_AMOUNT_HIST_BARS).items()]
return [{"date": d8_iso(d), "amount": round(float(v), 2)} for d, v in total.tail(_AMOUNT_HIST_BARS).items()]
# ---- EOD 的 SWR（stale-while-revalidate）：新鲜期内直返；过期先返旧值后台刷新 ----

9
backend/app/data/reference.py
View File

@@ -16,6 +16,7 @@ from __future__ import annotations
import asyncio
import calendar
import json
import logging
import time
from dataclasses import dataclass
from datetime import date
@@ -29,6 +30,8 @@ from ..db import async_session
from ..models import StockReference
from .sync_utils import call_retry, f_clean, fresh, get_pro_lazy, read_sync_state, s_clean, upsert_sync_state, utcnow
log = logging.getLogger(__name__)
_REFRESH_DAYS = 7
@@ -394,12 +397,12 @@ async def _sync_repurchase_locked(only_current: bool) -> dict[str, list[dict]]:
async def _repurchase_backfill() -> None:
"""后台全量回填（近 24 个月）；失败静默——下次触发重试。"""
"""后台全量回填（近 24 个月）；失败记录日志——下次触发重试。"""
try:
async with _repurchase_lock:
await _sync_repurchase_locked(only_current=False)
except Exception: # noqa: BLE001 后台任务无人接异常
pass
except Exception: # noqa: BLE001 后台任务无人接异常，至少留痕
log.warning("回购数据后台回填失败（下次触发重试）", exc_info=True)
def _spawn_repurchase_backfill() -> None:

5
backend/app/data/sync_utils.py
View File

@@ -68,6 +68,11 @@ def f_clean(v) -> float | None:
return None if f != f else f # NaN -> None
def d8_iso(v) -> str | None:
"""tushare YYYYMMDD -> 'YYYY-MM-DD'（字符串便于 JSON 缓存；pydantic 自动 coerce）。"""
return datetime.strptime(str(v), "%Y%m%d").date().isoformat() if v else None
async def read_sync_state(session: AsyncSession, ts_code: str, kind: str) -> StockSyncState | None:
return (await session.execute(
select(StockSyncState).where(