提交
This commit is contained in:
152
backend/scripts/active_mv_calib.py
Normal file
152
backend/scripts/active_mv_calib.py
Normal file
@@ -0,0 +1,152 @@
|
||||
"""0AMV 校验 v2：对照指南针 app EOD 收盘读数，寻找能否精确一致。
|
||||
|
||||
指南针目标（亿元，app 收盘读数）：
|
||||
2026-09-11: 172,364.3
|
||||
2026-09-14: 169,908.5（开 171,979.5 高 175,413.6 低 169,908.5）
|
||||
2026-09-15: 166,791.5（开 169,782.7 高 173,060.4 低 166,658.0）
|
||||
|
||||
模型：0AMV = Σ 自由流通市值 × active20，active20 = 1-Π(1-流通换手) 滚动20日。
|
||||
自由流通市值 = volume/(turnover_rate_f%) × close（daily_snapshot 近期才有 tff）。
|
||||
检验变体：全市场 / 剔北交所 / 再剔科创板 / 剔次新（上市<90自然日），
|
||||
若均差一个常数因子 → 指南针自由流通股本口径私有，无法精确复刻。
|
||||
|
||||
用法（backend 目录）：
|
||||
env -u SSLKEYLOGFILE PYTHONIOENCODING=utf-8 uv run python scripts/active_mv_calib.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import async_session
|
||||
|
||||
N_WIN = 20
|
||||
TARGETS = {"2026-09-11": 172364.3e8, "2026-09-14": 169908.5e8, "2026-09-15": 166791.5e8}
|
||||
|
||||
|
||||
async def _load() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]:
|
||||
async with async_session() as session:
|
||||
hist = (await session.execute(text("""
|
||||
SELECT ts, symbol, close, volume, amount, turnover
|
||||
FROM candles
|
||||
WHERE timeframe = '1d'
|
||||
AND turnover IS NOT NULL AND turnover > 0
|
||||
AND volume > 0
|
||||
AND ts >= now() - interval '500 days'
|
||||
"""))).fetchall()
|
||||
pend = (await session.execute(text("""
|
||||
SELECT ts, symbol, close, volume, amount
|
||||
FROM candles
|
||||
WHERE timeframe = '1d' AND volume > 0
|
||||
AND ts::date = (SELECT max(ts)::date FROM candles
|
||||
WHERE timeframe='1d' AND volume > 0)
|
||||
"""))).fetchall()
|
||||
snap = (await session.execute(text("""
|
||||
SELECT ts_code, trade_date::date AS d, turnover_rate, turnover_rate_f
|
||||
FROM daily_snapshot
|
||||
WHERE trade_date >= now() - interval '20 days'
|
||||
AND turnover_rate IS NOT NULL AND turnover_rate_f > 0
|
||||
"""))).fetchall()
|
||||
basic = (await session.execute(text("""
|
||||
SELECT symbol, market, exchange, list_date FROM stock_basic
|
||||
"""))).fetchall()
|
||||
return (
|
||||
pd.DataFrame(hist, columns=["ts", "symbol", "close", "volume", "amount", "turnover"]),
|
||||
pd.DataFrame(pend, columns=["ts", "symbol", "close", "volume", "amount"]),
|
||||
pd.DataFrame(snap, columns=["ts_code", "d", "turnover", "turnover_rate_f"]),
|
||||
pd.DataFrame(basic, columns=["symbol", "market", "exchange", "list_date"]),
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
hist, pend, snap, basic = asyncio.run(_load())
|
||||
snap["symbol"] = snap["ts_code"].str.split(".").str[0]
|
||||
|
||||
# 最新 candles 交易日若缺换手率（夜间 3.5 步未跑），用 snapshot 补
|
||||
if not pend.empty and not snap.empty:
|
||||
d_pend = pend["ts"].dt.date.max()
|
||||
sp = snap[snap["d"] == d_pend][["symbol", "turnover"]]
|
||||
pend = pend[pend["ts"].dt.date == d_pend].merge(sp, on="symbol", how="inner")
|
||||
pend = pend[pend["turnover"] > 0]
|
||||
print(f"最新交易日 {d_pend}：{len(pend):,} 行用 snapshot 换手率补齐")
|
||||
|
||||
df = pd.concat([hist, pend[hist.columns]], ignore_index=True)
|
||||
df["date"] = df["ts"].dt.date
|
||||
df = df.sort_values(["symbol", "date"], kind="stable").reset_index(drop=True)
|
||||
df = df.merge(basic, on="symbol", how="left")
|
||||
print(f"合计 {len(df):,} 行，{df['symbol'].nunique():,} 只，{df['date'].min()} ~ {df['date'].max()}")
|
||||
print("板块分布：", df.drop_duplicates("symbol")["market"].value_counts().to_dict())
|
||||
|
||||
t = (df["turnover"] / 100.0).clip(upper=0.9999)
|
||||
df["log_inactive"] = np.log1p(-t)
|
||||
decay = df.groupby("symbol")["log_inactive"].transform(
|
||||
lambda s: s.rolling(N_WIN, min_periods=1).sum()
|
||||
)
|
||||
df["active20"] = 1.0 - np.exp(decay.to_numpy())
|
||||
|
||||
# ---- 每个有 tff 的日期：个股级 FF20 明细 ----
|
||||
ms: dict = {}
|
||||
for d in sorted(snap["d"].unique())[-12:]:
|
||||
day = df[df["date"] == d]
|
||||
if day.empty:
|
||||
continue
|
||||
m = day.merge(snap.loc[snap["d"] == d, ["symbol", "turnover_rate_f"]],
|
||||
on="symbol", how="inner")
|
||||
m = m[m["turnover_rate_f"] > 0]
|
||||
tff = (m["turnover_rate_f"] / 100.0).clip(upper=0.9999)
|
||||
m["free_mv"] = m["volume"] / tff * m["close"]
|
||||
m["ff"] = m["free_mv"] * m["active20"]
|
||||
listed = pd.to_datetime(m["list_date"], format="%Y%m%d", errors="coerce")
|
||||
m["age_days"] = (pd.Timestamp(d) - listed).dt.days.fillna(10**6)
|
||||
ms[d] = m
|
||||
|
||||
# ---- 参数扫描：剔除次新窗口 × 是否剔北交所 ----
|
||||
print("\n== 参数扫描（对指南针三日的比值；理想=1.00000 稳定）==")
|
||||
print(f"{'剔北交所':<6}{'剔次新(自然日)':>12}{'09-11':>10}{'09-14':>10}{'09-15':>10}{'均值':>10}{'极差':>9}")
|
||||
results = []
|
||||
for bse_ex in (False, True):
|
||||
for w in (0, 60, 90, 120, 150, 180, 270, 365, 550):
|
||||
ratios = []
|
||||
for tgt_d, tgt in TARGETS.items():
|
||||
d = pd.Timestamp(tgt_d).date()
|
||||
if d not in ms:
|
||||
break
|
||||
m = ms[d]
|
||||
mask = m["age_days"] >= w
|
||||
if bse_ex:
|
||||
mask &= m["market"] != "北交所"
|
||||
ratios.append(m.loc[mask, "ff"].sum() / tgt)
|
||||
if len(ratios) < 3:
|
||||
continue
|
||||
mean_r = float(np.mean(ratios))
|
||||
spread = max(ratios) - min(ratios)
|
||||
results.append((abs(mean_r - 1) + spread, bse_ex, w, ratios, mean_r, spread))
|
||||
print(f"{'是' if bse_ex else '否':<6}{w:>12}{ratios[0]:>10.5f}{ratios[1]:>10.5f}"
|
||||
f"{ratios[2]:>10.5f}{mean_r:>10.5f}{spread:>9.5f}")
|
||||
|
||||
results.sort()
|
||||
_, best_bse, best_w, best_ratios, best_mean, best_spread = results[0]
|
||||
print(f"\n最优配置：剔北交所={'是' if best_bse else '否'}，剔上市<{best_w}自然日")
|
||||
print(f" 三日比值 {['%.5f' % r for r in best_ratios]}，极差 {best_spread:.5f}")
|
||||
|
||||
# 最优配置下的每日序列
|
||||
print("\n== 最优配置近 12 日 FF20（亿）==")
|
||||
for d in sorted(ms):
|
||||
m = ms[d]
|
||||
mask = m["age_days"] >= best_w
|
||||
if best_bse:
|
||||
mask &= m["market"] != "北交所"
|
||||
v = m.loc[mask, "ff"].sum() / 1e8
|
||||
mark = f" ←指南针 {TARGETS[str(d)]/1e8:,.1f}" if str(d) in TARGETS else ""
|
||||
print(f"{d} {v:>10,.0f} n={int(mask.sum())}{mark}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
116
backend/scripts/active_mv_probe.py
Normal file
116
backend/scripts/active_mv_probe.py
Normal file
@@ -0,0 +1,116 @@
|
||||
"""复刻指南针 0AMV（活跃市值）——可行性验证原型。
|
||||
|
||||
模型：
|
||||
活跃筹码比例(t) = 1 - Π_{近N个交易日} (1 - 换手率) （换手衰减）
|
||||
0AMV(t) = Σ_个股 流通市值(t) × 活跃筹码比例(t)
|
||||
流通市值不额外拉接口：成交量(股) / (换手%/100) 反推流通股本，× close 即得，
|
||||
且随解禁/增发每日自适应。N 为指南针未公开的窗口参数，输出多组候选供对照 app 校准。
|
||||
|
||||
用法（backend 目录）：
|
||||
env -u SSLKEYLOGFILE uv run python scripts/active_mv_probe.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import async_session
|
||||
|
||||
WINDOWS = (20, 60, 120) # 活跃窗口候选（交易日）
|
||||
LOOKBACK_DAYS = 500 # 日历日回看，足够 120 交易日窗口
|
||||
SHOW_DAYS = 20 # 打印最近 N 个交易日
|
||||
|
||||
|
||||
async def _load() -> tuple[pd.DataFrame, pd.DataFrame]:
|
||||
async with async_session() as session:
|
||||
rows = (await session.execute(text("""
|
||||
SELECT ts, symbol, close, volume, amount, turnover
|
||||
FROM candles
|
||||
WHERE timeframe = '1d'
|
||||
AND turnover IS NOT NULL AND turnover > 0
|
||||
AND volume > 0
|
||||
AND ts >= now() - interval '500 days'
|
||||
"""))).fetchall()
|
||||
snap = (await session.execute(text("""
|
||||
SELECT ts_code, circ_mv, trade_date
|
||||
FROM daily_snapshot
|
||||
WHERE trade_date >= (SELECT max(trade_date) - interval '10 days' FROM daily_snapshot)
|
||||
AND circ_mv IS NOT NULL
|
||||
"""))).fetchall()
|
||||
df = pd.DataFrame(rows, columns=["ts", "symbol", "close", "volume", "amount", "turnover"])
|
||||
snap = pd.DataFrame(snap, columns=["ts_code", "circ_mv_wan", "snap_date"])
|
||||
return df, snap
|
||||
|
||||
|
||||
def _validate(df: pd.DataFrame, snap: pd.DataFrame) -> None:
|
||||
"""反推流通市值 vs daily_snapshot（万元→元）交叉验证。"""
|
||||
if snap.empty:
|
||||
print("校验：daily_snapshot 为空")
|
||||
return
|
||||
snap["symbol"] = snap["ts_code"].str.split(".").str[0]
|
||||
snap["d"] = snap["snap_date"].dt.date
|
||||
common = set(snap["d"]) & set(df["date"])
|
||||
if not common:
|
||||
print("校验：daily_snapshot 与 candles 无可对比日期")
|
||||
return
|
||||
snap_date = max(common)
|
||||
day = df[df["date"] == snap_date]
|
||||
m = day[["symbol", "circ_mv"]].merge(
|
||||
snap.loc[snap["d"] == snap_date, ["symbol", "circ_mv_wan"]],
|
||||
on="symbol", how="inner",
|
||||
)
|
||||
if m.empty:
|
||||
print("校验：无可对比个股")
|
||||
return
|
||||
ratio = m["circ_mv"] / (m["circ_mv_wan"] * 1e4)
|
||||
print(f"校验（{snap_date}，{len(m)} 只）：反推/官方流通市值中位数 = {ratio.median():.4f}，"
|
||||
f"P10={ratio.quantile(0.1):.3f}，P90={ratio.quantile(0.9):.3f}")
|
||||
big = m.nlargest(5, "circ_mv")
|
||||
for _, r in big.iterrows():
|
||||
print(f" {r['symbol']}: 反推 {r['circ_mv']/1e8:,.0f}亿 vs 官方 {r['circ_mv_wan']/1e4:,.0f}亿"
|
||||
f"（比值 {r['circ_mv']/(r['circ_mv_wan']*1e4):.3f}）")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
df, snap = asyncio.run(_load())
|
||||
df["date"] = df["ts"].dt.date
|
||||
# 关键：分组滚动前必须按 (symbol, date) 排序，SQL 返回顺序不保证
|
||||
df = df.sort_values(["symbol", "date"], kind="stable").reset_index(drop=True)
|
||||
print(f"candles 载入 {len(df):,} 行，{df['symbol'].nunique():,} 只，"
|
||||
f"{df['date'].min()} ~ {df['date'].max()}")
|
||||
|
||||
t = (df["turnover"] / 100.0).clip(upper=0.9999)
|
||||
df["float_shares"] = df["volume"] / t # 流通股本（股）；volume 单位=股
|
||||
df["circ_mv"] = df["float_shares"] * df["close"] # 流通市值（元）
|
||||
|
||||
_validate(df, snap)
|
||||
|
||||
df["log_inactive"] = np.log1p(-t)
|
||||
agg_cols = {"circ_mv": "sum", "amount": "sum"}
|
||||
for n in WINDOWS:
|
||||
decay = df.groupby("symbol")["log_inactive"].transform(
|
||||
lambda s: s.rolling(n, min_periods=1).sum()
|
||||
)
|
||||
df[f"amv_{n}"] = df["circ_mv"] * (1.0 - np.exp(decay.to_numpy()))
|
||||
agg_cols[f"amv_{n}"] = "sum"
|
||||
|
||||
daily = df.groupby("date").agg(agg_cols).sort_index().tail(SHOW_DAYS)
|
||||
print("\n日期 全市场流通市值(万亿) 日成交额(万亿) " +
|
||||
" ".join(f"0AMV_{n}(万亿) 活跃占比_{n}" for n in WINDOWS))
|
||||
for d, r in daily.iterrows():
|
||||
print(f"{d} {r['circ_mv']/1e12:8.3f} {r['amount']/1e12:6.3f} " +
|
||||
" ".join(
|
||||
f"{r[f'amv_{n}']/1e12:7.3f} {r[f'amv_{n}']/r['circ_mv']*100:5.1f}%"
|
||||
for n in WINDOWS
|
||||
))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,162 +0,0 @@
|
||||
"""全量回补换手率（candles.turnover，单位 %）。
|
||||
|
||||
用法（在 backend 目录下）：
|
||||
uv run python scripts/backfill_turnover.py # 从 2000-01-01（daily_basic 起点）回补到今天
|
||||
uv run python scripts/backfill_turnover.py --start 20200101
|
||||
uv run python scripts/backfill_turnover.py --force # 已回补的交易日也重拉
|
||||
|
||||
- 数据源：Tushare daily_basic(trade_date=..., fields='ts_code,turnover_rate')，按日全市场；
|
||||
- 幂等可断点续跑：某交易日 candles 已有非空 turnover 即跳过（--force 强制重做）；
|
||||
- 交易日取自本地 trade_calendar（缓存覆盖不到起点时自动拉一次宽范围日历）；
|
||||
- 每日一条 UPDATE ... FROM unnest(...) 批量写回，仅更新 turnover 列；
|
||||
- Tushare 每分钟限频由 _call_retry 自动等待 62s 重试。
|
||||
|
||||
注意：与 import_tdx_day.py（回填 amount 会整行 upsert）串行运行，避免同表行锁竞争。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.screener.market_sync import _call_retry, _get_pro
|
||||
|
||||
import asyncpg
|
||||
|
||||
|
||||
def load_db_url() -> str:
|
||||
"""与 import_tdx_day.py 相同的 .env -> libpq URL 解析（本地复制避免跨脚本导入）。"""
|
||||
env = Path(__file__).resolve().parent.parent / ".env"
|
||||
if env.exists():
|
||||
for line in env.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("DATABASE_URL=postgresql+asyncpg://"):
|
||||
return "postgresql://" + line.split("://", 1)[1]
|
||||
return "postgresql://postgres:postgres@localhost:5432/stock"
|
||||
|
||||
_DAILY_BASIC_FLOOR = "20000101" # daily_basic 最早覆盖 2000-01-04，更早的交易日无换手数据
|
||||
_INTERVAL_MSG = 20
|
||||
|
||||
|
||||
async def _calendar_dates(conn: asyncpg.Connection, start: str, end: str) -> list[str]:
|
||||
"""[start, end] 交易日（升序）。本地缓存覆盖不到起点时拉一次宽范围日历并回写。"""
|
||||
cached = [r[0] for r in await conn.fetch(
|
||||
"SELECT trade_date FROM trade_calendar WHERE trade_date >= $1 AND trade_date <= $2 "
|
||||
"ORDER BY trade_date", start, end)]
|
||||
if cached and cached[0] <= start:
|
||||
return cached
|
||||
|
||||
pro = _get_pro()
|
||||
try:
|
||||
cal = await asyncio.to_thread(
|
||||
_call_retry, pro.trade_cal, exchange="SSE", start_date=start, end_date=end, is_open="1"
|
||||
)
|
||||
dates = sorted(cal["cal_date"].tolist())
|
||||
except Exception as e: # noqa: BLE001
|
||||
if not cached:
|
||||
raise
|
||||
print(f"交易日历拉取受限（{str(e)[:100]}），沿用本地缓存")
|
||||
return cached
|
||||
have = set(cached)
|
||||
fresh = [d for d in dates if d not in have]
|
||||
if fresh:
|
||||
await conn.executemany(
|
||||
"INSERT INTO trade_calendar (trade_date) VALUES ($1) ON CONFLICT DO NOTHING", [(d,) for d in fresh]
|
||||
)
|
||||
return dates
|
||||
|
||||
|
||||
async def _day_status(conn: asyncpg.Connection, d: str) -> tuple[int, int]:
|
||||
"""(已有换手的行数, 当日总行数)。无行情的日子 total=0 直接跳过。"""
|
||||
row = await conn.fetchrow(
|
||||
"SELECT count(*) FILTER (WHERE turnover IS NOT NULL) AS done, count(*) AS total "
|
||||
"FROM candles WHERE timeframe = '1d' AND ts = $1::timestamp", datetime.strptime(d, "%Y%m%d")
|
||||
)
|
||||
return row["done"], row["total"]
|
||||
|
||||
|
||||
async def main(start: str, end: str, force: bool) -> None:
|
||||
conn = await asyncpg.connect(load_db_url())
|
||||
try:
|
||||
# 默认起点：daily_basic 覆盖范围与 candles 最早日线的较大者（更早的日期拉了也是空）
|
||||
if start is None:
|
||||
first = await conn.fetchval(
|
||||
"SELECT min(ts) FROM candles WHERE timeframe = '1d' AND symbol <> 'DEMO'")
|
||||
start = max(first.strftime("%Y%m%d"), _DAILY_BASIC_FLOOR) if first else _DAILY_BASIC_FLOOR
|
||||
if end is None:
|
||||
end = datetime.now().strftime("%Y%m%d")
|
||||
|
||||
dates = await _calendar_dates(conn, start, end)
|
||||
todo: list[str] = []
|
||||
for d in dates:
|
||||
if force:
|
||||
done, total = await _day_status(conn, d)
|
||||
if total:
|
||||
todo.append(d)
|
||||
continue
|
||||
done, total = await _day_status(conn, d)
|
||||
if total and done < total // 2: # 过半缺换手才重做（容忍个别股票无快照）
|
||||
todo.append(d)
|
||||
print(f"区间 {start}~{end} 共 {len(dates)} 个交易日，待回补 {len(todo)} 个")
|
||||
|
||||
pro = _get_pro()
|
||||
done = 0
|
||||
t0 = time.time()
|
||||
for d in todo:
|
||||
time.sleep(0.15) # 轻微控频；分钟级限频由 _call_retry 自动等待重试
|
||||
df = None
|
||||
for attempt in range(5): # 网络抖动（超时/断连）也重试，_call_retry 只兜限频
|
||||
try:
|
||||
df = _call_retry(
|
||||
pro.daily_basic, trade_date=d, fields="ts_code,trade_date,turnover_rate"
|
||||
)
|
||||
break
|
||||
except Exception as e: # noqa: BLE001
|
||||
wait = min(30 * (attempt + 1), 120)
|
||||
print(f" {d} 拉取异常（{str(e)[:80]}），{wait}s 后重试 {attempt + 1}/5")
|
||||
time.sleep(wait)
|
||||
if df is None:
|
||||
print(f" {d} 连续 5 次失败，跳过（断点续跑可补）")
|
||||
continue
|
||||
if df.empty:
|
||||
continue
|
||||
|
||||
syms: list[str] = []
|
||||
vals: list[float] = []
|
||||
for _, r in df.iterrows():
|
||||
tr = r["turnover_rate"]
|
||||
if tr is None or tr != tr: # None / NaN
|
||||
continue
|
||||
syms.append(str(r["ts_code"]).split(".")[0])
|
||||
vals.append(float(tr))
|
||||
if not syms:
|
||||
continue
|
||||
n = await conn.execute(
|
||||
"UPDATE candles AS c SET turnover = v.t "
|
||||
"FROM unnest($1::text[], $2::float8[]) AS v(sym, t) "
|
||||
"WHERE c.symbol = v.sym AND c.timeframe = '1d' AND c.ts = $3::timestamp",
|
||||
syms, vals, datetime.strptime(d, "%Y%m%d"),
|
||||
)
|
||||
done += 1
|
||||
if done % _INTERVAL_MSG == 0 or done == len(todo):
|
||||
elapsed = time.time() - t0
|
||||
eta = elapsed / done * (len(todo) - done) if done else 0
|
||||
print(f" 进度 {done}/{len(todo)}（{d}，{len(syms)} 只，{n}），"
|
||||
f"{elapsed:.0f}s 已用，预计还需 {eta/60:.0f}m")
|
||||
print(f"回补完成：{done} 个交易日")
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ap = argparse.ArgumentParser(description="全量回补换手率 candles.turnover")
|
||||
ap.add_argument("--start", default=None, help="YYYYMMDD，默认 max(candles 最早, 20000101)")
|
||||
ap.add_argument("--end", default=None, help="YYYYMMDD，默认今天")
|
||||
ap.add_argument("--force", action="store_true", help="已有换手的交易日也重拉")
|
||||
a = ap.parse_args()
|
||||
asyncio.run(main(a.start, a.end, a.force))
|
||||
Reference in New Issue
Block a user