日K加载提速:adj_factor 覆盖索引 + 冷路径并发 + 热路径进程内缓存直返
- adj_factor 覆盖索引 (ts_code,trade_date) INCLUDE (adj_factor):根治堆碎片化 (单股 6516 行散 6516 块,满载时位图堆扫 1.5s+),Index Only Scan ~2ms; 因子查询全部改 2 列投影,lag() 窗口只取变点(6516→32 行) - preview 冷路径 2 会话 2 波并发,信息卡合并为 LEFT JOIN LATERAL 一条 - 鉴权会话 60s 进程内缓存(登出/全端登出即时失效),全站请求省 ~80ms - pvj/chipsj/stocksj/facetsj 存 model_dump_json 原串直返(与 response_model 字节一致),热路径 230-2190ms → 1-2ms;get_version 本地缓存+bump 即时可见 - 连接池 10+20;smoke_test 适配鉴权缓存
This commit is contained in:
@@ -6,11 +6,16 @@
|
||||
旧缓存 key 里带着旧版本号,无需 SCAN 批量删除。
|
||||
- 只缓存「读多写少、可容忍短暂陈旧」的聚合数据(股票列表、筛选项等);
|
||||
K线/回测等口径敏感数据不走这里。
|
||||
- Redis 之前还有一层进程内本地缓存(local_get/local_set,0 RTT):只存已序列化好的
|
||||
JSON 字符串,命中后接口直接 Response(content=raw) 原样返回,跳过 json.loads +
|
||||
pydantic 校验/序列化(大响应这两步合计可达数百 ms)。本地 TTL 恒 ≤ Redis TTL,
|
||||
多进程部署时本地条目最多比 Redis 多陈旧 120s;版本号 bump 在同进程立即生效。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import redis.asyncio as aioredis
|
||||
@@ -48,6 +53,46 @@ def digest(*parts: Any) -> str:
|
||||
return hashlib.md5(raw.encode()).hexdigest() # noqa: S324
|
||||
|
||||
|
||||
# --- 进程内本地缓存层(Redis 之前,0 RTT)-----------------------------------
|
||||
# asyncio 单线程,dict 读写无锁安全;按插入序淘汰近似 LRU(命中不续期,足够用)。
|
||||
_LOCAL_MAX_BYTES = 64 * 1024 * 1024 # 总字节预算(preview 响应 ~250KB,留足 1y 大窗口)
|
||||
_LOCAL_MAX_ENTRIES = 128
|
||||
_local_store: dict[str, tuple[float, str]] = {} # key -> (expires_monotonic, raw_json)
|
||||
_local_bytes = 0
|
||||
|
||||
|
||||
def local_get(key: str) -> str | None:
|
||||
"""命中返回已序列化的 JSON 字符串(调用方直接 Response 原样返回)。"""
|
||||
global _local_bytes
|
||||
ent = _local_store.get(key)
|
||||
if ent is None:
|
||||
return None
|
||||
expires, raw = ent
|
||||
if expires < time.monotonic():
|
||||
_local_store.pop(key, None)
|
||||
_local_bytes -= len(raw) + 64
|
||||
return None
|
||||
return raw
|
||||
|
||||
|
||||
def local_set(key: str, raw: str, ttl: int) -> None:
|
||||
"""ttl 为秒;调用方应传 min(本地默认, Redis TTL),保证本地不比 Redis 活得久。"""
|
||||
global _local_bytes
|
||||
old = _local_store.pop(key, None)
|
||||
if old is not None:
|
||||
_local_bytes -= len(old[1])
|
||||
_local_store[key] = (time.monotonic() + max(1, min(ttl, 120)), raw)
|
||||
_local_bytes += len(raw) + 64 # 连同 dict/tuple 开销粗略计入
|
||||
while _local_store and (len(_local_store) > _LOCAL_MAX_ENTRIES or _local_bytes > _LOCAL_MAX_BYTES):
|
||||
_local_bytes -= len(_local_store.popitem(last=False)[1][1]) + 64
|
||||
|
||||
|
||||
# --- 版本号本地缓存:热请求连 Redis GET ver:xx 都省掉 ------------------------
|
||||
# 同进程 bump_version 立即刷新本地;其他进程 bump 后本地最多陈旧 _LOCAL_VERSION_TTL。
|
||||
_LOCAL_VERSION_TTL = 60.0
|
||||
_local_versions: dict[str, tuple[int, float]] = {} # name -> (version, fetched_monotonic)
|
||||
|
||||
|
||||
async def cache_get(key: str) -> Any | None:
|
||||
c = _client()
|
||||
if c is None:
|
||||
@@ -72,13 +117,20 @@ async def cache_set(key: str, value: Any, ttl: int) -> None:
|
||||
|
||||
|
||||
async def get_version(name: str) -> int:
|
||||
"""读版本号(缺省 0)。版本号参与缓存 key:INCR 后旧 key 全部失效。"""
|
||||
"""读版本号(缺省 0)。版本号参与缓存 key:INCR 后旧 key 全部失效。
|
||||
先查本地(60s),热请求 0 RTT。"""
|
||||
ent = _local_versions.get(name)
|
||||
now = time.monotonic()
|
||||
if ent is not None and ent[1] > now:
|
||||
return ent[0]
|
||||
c = _client()
|
||||
if c is None:
|
||||
return 0
|
||||
try:
|
||||
v = await c.get(f"ver:{name}")
|
||||
return int(v) if v is not None else 0
|
||||
val = int(v) if v is not None else 0
|
||||
_local_versions[name] = (val, now + _LOCAL_VERSION_TTL)
|
||||
return val
|
||||
except Exception: # noqa: BLE001
|
||||
_bail()
|
||||
return 0
|
||||
@@ -89,7 +141,9 @@ async def bump_version(name: str) -> None:
|
||||
if c is None:
|
||||
return
|
||||
try:
|
||||
await c.incr(f"ver:{name}")
|
||||
v = await c.incr(f"ver:{name}")
|
||||
# 同进程 bump 立即可见(夜同步跑在本进程时响应缓存零陈旧窗口)
|
||||
_local_versions[name] = (int(v), time.monotonic() + _LOCAL_VERSION_TTL)
|
||||
except Exception: # noqa: BLE001
|
||||
_bail()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user