mirror of
https://github.com/shy3130/tick-stock-panel.git
synced 2026-11-02 10:26:13 +00:00
feat(factors): P2b 内容寻址因子列缓存 — +10 因子热路径增量 0.1ms, P2 两项验收达成
自定义 (DSL) 因子此前每次求值都全量重算 frame_transform (132 万行帧上 每因子 ~36ms, 随因子数线性增长 — 回测/守护复验/实验室批量的共同热路径)。 - app/factors/materialize_cache.py: 键 = sha256(定义 + 依赖列内容指纹 + 行数) — 内容指纹即失效传播 (公式变/输入任何值变 → 键变), 无需数据版本 号。两层: 进程内 LRU (字节上限) + 磁盘 parquet (跨进程, 文件数上限, 目录可整删)。fail-open: 任何故障回退现算。列指纹按帧记忆化 (weakref finalize 回收), 同帧 K 个因子每列只哈希一次 (~8ms/百万行)。 - materialize_scoring_columns 两段式: 命中列一次批量附着 (成本与因子数 无关); 未命中逐个现算 (依赖顺序)。DSL 分阶段临时列不再泄漏进下游帧 (与命中路径列集合完全一致)。 - 顺带修复存量 bug: winsorize(x) 省略 k 时 DSL 编译器直接 IndexError (该写法此前完全不可用), 补回归测试。 - 基准 (132 万行, K3→K13): 热路径 0.0ms→0.1ms (+10 因子绝对增量 0.1ms, 基线 +400ms 线性); 冷启动 480.7ms vs 无缓存 478ms (开销 ~0.5%)。 端到端优化器复测 trials 4.23×/含 prepare 3.16×, stats 完全一致。 - 回归: 552 个相关测试全过 (含 6 个缓存契约测试: 命中等价/输入与定义 变更失效/损坏回退/小帧旁路/临时列不泄漏); ruff 新增文件全清。 Signed-off-by: shy3130 <shy3130@qq.com>
This commit is contained in:
@@ -320,7 +320,12 @@ def _check_call(node: dict, errors: list[DslError]) -> dict[str, float]:
|
||||
return {}
|
||||
constants: dict[str, float] = {}
|
||||
for index, const_name in enumerate(const_names):
|
||||
arg = args[n_expr + index]
|
||||
arg_index = n_expr + index
|
||||
if arg_index >= len(args):
|
||||
# 可选常量参数被省略 (winsorize 的 k, 默认 3) —
|
||||
# 此前这里直接 IndexError, winsorize(x) 完全不可用。
|
||||
break
|
||||
arg = args[arg_index]
|
||||
value = _const_value(arg)
|
||||
if value is None:
|
||||
errors.append(DslError(
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
"""L2 内容寻址因子列缓存 (P2b) — 自定义 (DSL) 因子物化结果的复用。
|
||||
|
||||
键 = sha256(因子定义指纹 + 全部依赖列内容指纹 + 行数)。内容指纹即失效
|
||||
传播: 公式变 / 依赖列任何值变 → 键变 → 自然失效, 不需要数据版本号或
|
||||
mtime 签名 — 键本身覆盖「同一因子作用在同一输入上」的全部语义。
|
||||
|
||||
fail-open: 缓存读写任何故障 (目录不可写 / parquet 损坏 / 超限) 都回退
|
||||
现算。缓存是纯优化, 永不影响正确性; 目录可整删 (内容寻址, 删除安全)。
|
||||
|
||||
两层:
|
||||
- 进程内 LRU (键 → 列, 字节上限): 同帧/同数据的重复求值近零成本
|
||||
(守护复验、优化器、实验室批量回测);
|
||||
- 磁盘 parquet (data/cache/factor_columns/<key>.parquet, 文件数上限):
|
||||
跨进程复用 (回测 worker 每任务一个进程)。
|
||||
|
||||
列内容指纹按 (frame, 列名) 记忆 — 同一帧上 K 个因子共享依赖列, 每列只
|
||||
哈希一次 (hash+sha256 ≈ 8ms/百万行, 远低于因子现算)。frame 指纹表用
|
||||
weakref 弱持有, 帧被回收后自动失效, 不阻止内存释放。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import logging
|
||||
import threading
|
||||
import weakref
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 小帧不缓存: 哈希+IO 比现算还贵 (筛选页预览、单标的明细等)。
|
||||
_MIN_ROWS = 50_000
|
||||
_LRU_MAX_BYTES = 256 * 1024 * 1024
|
||||
_DISK_MAX_FILES = 1000
|
||||
_DISK_SUBDIR = Path("cache") / "factor_columns"
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Entry:
|
||||
series: pl.Series
|
||||
nbytes: int
|
||||
|
||||
|
||||
_lru: OrderedDict[str, _Entry] = OrderedDict()
|
||||
_lru_bytes = 0
|
||||
# (id(frame) → {列名: 指纹}); finalize 在帧被 GC 时清除条目 — 不强持帧引用,
|
||||
# id 也不会复用到陈旧条目 (条目随帧消亡)。
|
||||
_frame_digests: dict[int, dict[str, str]] = {}
|
||||
_stats = {"lru_hit": 0, "disk_hit": 0, "miss": 0, "store": 0, "bypass_small": 0, "error": 0}
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def stats() -> dict[str, int]:
|
||||
"""缓存命中计数 (观测/测试用)。"""
|
||||
return dict(_stats)
|
||||
|
||||
|
||||
def invalidate(*, disk: bool = False) -> None:
|
||||
"""清空进程内 LRU 并重置命中计数; disk=True 时连磁盘缓存目录一起清
|
||||
(内容寻址, 整删安全)。"""
|
||||
global _lru_bytes
|
||||
with _lock:
|
||||
_lru.clear()
|
||||
_lru_bytes = 0
|
||||
_frame_digests.clear()
|
||||
_stats.update({"lru_hit": 0, "disk_hit": 0, "miss": 0, "store": 0, "bypass_small": 0, "error": 0})
|
||||
if disk:
|
||||
import shutil
|
||||
|
||||
root = Path(settings.data_dir) / _DISK_SUBDIR
|
||||
if root.exists():
|
||||
shutil.rmtree(root, ignore_errors=True)
|
||||
|
||||
|
||||
def _column_digest(frame: pl.DataFrame, column: str) -> str:
|
||||
"""单列内容指纹 (同帧记忆化): dtype + null 数 + 逐行 murmur 哈希的 sha256。
|
||||
|
||||
Series.hash 跨进程确定 (同 polars 版本, 固定种子), 行序敏感 — 顺序不同
|
||||
视为不同输入 (物化按行位对齐, 顺序本来就是语义的一部分)。"""
|
||||
with _lock:
|
||||
frame_key = id(frame)
|
||||
memo = _frame_digests.get(frame_key)
|
||||
if memo is None:
|
||||
memo = {}
|
||||
_frame_digests[frame_key] = memo
|
||||
weakref.finalize(frame, _frame_digests.pop, frame_key, None)
|
||||
elif column in memo:
|
||||
return memo[column]
|
||||
series = frame.get_column(column)
|
||||
digest = hashlib.sha256()
|
||||
digest.update(column.encode())
|
||||
digest.update(str(series.dtype).encode())
|
||||
digest.update(f"nulls={series.null_count()};rows={series.len()};".encode())
|
||||
digest.update(series.hash().to_numpy().tobytes())
|
||||
value = digest.hexdigest()
|
||||
with _lock:
|
||||
memo[column] = value
|
||||
return value
|
||||
|
||||
|
||||
def cache_key(
|
||||
factor_id: str,
|
||||
formula_text: str,
|
||||
version: int,
|
||||
frame: pl.DataFrame,
|
||||
dependency_columns,
|
||||
) -> str | None:
|
||||
"""定义 + 输入内容 → 缓存键; None = 不缓存 (小帧 / 依赖列缺一)。"""
|
||||
if frame.height < _MIN_ROWS:
|
||||
_stats["bypass_small"] += 1
|
||||
return None
|
||||
columns = sorted(set(dependency_columns))
|
||||
digests: list[str] = []
|
||||
for column in columns:
|
||||
if column not in frame.columns:
|
||||
return None # 缺列时 frame_transform 本就会 E013 拒绝, 无缓存意义
|
||||
digests.append(_column_digest(frame, column))
|
||||
digest = hashlib.sha256()
|
||||
digest.update(f"{factor_id}|v{version}|{formula_text}|rows={frame.height};".encode())
|
||||
digest.update(";".join(digests).encode())
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _disk_path(key: str) -> Path:
|
||||
return Path(settings.data_dir) / _DISK_SUBDIR / f"{key}.parquet"
|
||||
|
||||
|
||||
def get_cached(key: str, expected_rows: int) -> pl.Series | None:
|
||||
"""取缓存的因子列; LRU 未命中读磁盘并晋升; 任何异常按未命中处理。"""
|
||||
global _lru_bytes
|
||||
with _lock:
|
||||
entry = _lru.get(key)
|
||||
if entry is not None:
|
||||
_lru.move_to_end(key)
|
||||
if entry.series.len() == expected_rows:
|
||||
_stats["lru_hit"] += 1
|
||||
return entry.series
|
||||
_lru.pop(key)
|
||||
_lru_bytes -= entry.nbytes
|
||||
path = _disk_path(key)
|
||||
if not path.exists():
|
||||
_stats["miss"] += 1
|
||||
return None
|
||||
try:
|
||||
series = pl.read_parquet(path).to_series(0)
|
||||
if series.len() != expected_rows:
|
||||
raise ValueError(f"缓存行数不符: {series.len()} != {expected_rows}")
|
||||
except Exception as e: # noqa: BLE001 — 损坏缓存删除并回退现算
|
||||
_stats["error"] += 1
|
||||
logger.debug("factor cache read failed %s: %s", key[:12], e)
|
||||
with contextlib.suppress(OSError):
|
||||
path.unlink(missing_ok=True)
|
||||
_stats["miss"] += 1
|
||||
return None
|
||||
_stats["disk_hit"] += 1
|
||||
put_cached(key, series)
|
||||
return series
|
||||
|
||||
|
||||
def put_cached(key: str, series: pl.Series) -> None:
|
||||
"""写入两层缓存; 超限/故障静默跳过 (fail-open)。"""
|
||||
global _lru_bytes
|
||||
nbytes = series.len() * 8 # 物化因子列均为 float64
|
||||
with _lock:
|
||||
if key not in _lru:
|
||||
_lru[key] = _Entry(series=series, nbytes=nbytes)
|
||||
_lru_bytes += nbytes
|
||||
while _lru_bytes > _LRU_MAX_BYTES and len(_lru) > 1:
|
||||
_, evicted = _lru.popitem(last=False)
|
||||
_lru_bytes -= evicted.nbytes
|
||||
_stats["store"] += 1
|
||||
try:
|
||||
root = _disk_path(key).parent
|
||||
if sum(1 for _ in root.glob("*.parquet")) >= _DISK_MAX_FILES:
|
||||
return # 文件数上限: 跳写不淘汰 (目录可整删, 见模块 docstring)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
tmp = root / f"{key}.parquet.tmp"
|
||||
series.to_frame().write_parquet(tmp)
|
||||
tmp.replace(root / f"{key}.parquet")
|
||||
except OSError as e:
|
||||
_stats["error"] += 1
|
||||
logger.debug("factor cache write failed %s: %s", key[:12], e)
|
||||
@@ -290,8 +290,15 @@ def materialize_scoring_columns(
|
||||
) -> pl.DataFrame:
|
||||
# custom (DSL) 因子先物化: frame_transform 可能需要多阶段临时列 (嵌套窗口规避),
|
||||
# 与单表达式路径不同, 必须整体走帧变换 —— 与检验/试算共用同一条计算路径 (P3)。
|
||||
# P2b: 物化结果走内容寻址缓存 (键 = 定义 + 依赖列内容), 重复求值零成本;
|
||||
# 缓存故障一律回退现算, 不影响正确性。
|
||||
from app.factors.dsl import FACTOR_COLUMN, compile_formula_cached
|
||||
from app.factors.materialize_cache import cache_key, get_cached, put_cached
|
||||
|
||||
# 两段式: 命中列一次批量附着 (成本与因子数无关 — 每列一次 with_columns
|
||||
# 会随 K 线性拷贝帧); 未命中的逐个现算 (帧变换有依赖顺序)。
|
||||
pending: list[tuple[str, object, str | None]] = []
|
||||
hits: dict[str, pl.Series] = {}
|
||||
for name in names:
|
||||
spec = _registry_get_factor(str(name))
|
||||
if spec is None or spec.kind != "custom" or name in frame.columns:
|
||||
@@ -299,10 +306,34 @@ def materialize_scoring_columns(
|
||||
compiled = compile_formula_cached(spec.formula_text)
|
||||
if compiled.frame_transform is None:
|
||||
continue
|
||||
key = cache_key(
|
||||
str(name), spec.formula_text, spec.version, frame,
|
||||
set(spec.dependencies) | set(compiled.referenced_factors),
|
||||
)
|
||||
if key is not None:
|
||||
cached = get_cached(key, frame.height)
|
||||
if cached is not None:
|
||||
hits[str(name)] = cached
|
||||
continue
|
||||
pending.append((str(name), compiled, key))
|
||||
if hits:
|
||||
frame = frame.with_columns([series.alias(name) for name, series in hits.items()])
|
||||
for name, compiled, key in pending:
|
||||
if name in frame.columns:
|
||||
continue
|
||||
transformed = compiled.frame_transform(frame)
|
||||
if transformed is None:
|
||||
continue
|
||||
frame = transformed.with_columns(pl.col(FACTOR_COLUMN).alias(str(name))).drop(FACTOR_COLUMN)
|
||||
column = transformed.get_column(FACTOR_COLUMN).alias(name)
|
||||
if key is not None:
|
||||
put_cached(key, column)
|
||||
# 丢弃 DSL 分阶段临时列 — 与缓存命中路径的列集合完全一致
|
||||
# (临时列本就是变换内部工件, 不应泄漏进下游帧)。
|
||||
staged_temps = [
|
||||
c for c in transformed.columns
|
||||
if c not in frame.columns and c != FACTOR_COLUMN
|
||||
]
|
||||
frame = transformed.drop([FACTOR_COLUMN, *staged_temps]).with_columns(column)
|
||||
expressions = [
|
||||
expression.alias(name)
|
||||
for name in names
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
"""P2b 基准: 自定义因子物化的热路径成本随因子数量的增长。
|
||||
|
||||
场景: 同一帧上反复物化 K 个自定义 (DSL) 因子 — 回测/筛选/评分的每次运行
|
||||
都走 materialize_scoring_columns。验收 (docs/ext-architecture-plan.md P2):
|
||||
新增 10 个自定义指标后热路径耗时不变 (内容寻址缓存命中时应近似平坦)。
|
||||
|
||||
用法: cd backend && uv run python scripts/bench_factor_materialize.py [--json PATH]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import logging
|
||||
|
||||
logging.getLogger("app").setLevel(logging.WARNING)
|
||||
|
||||
import polars as pl # noqa: E402
|
||||
|
||||
from app.config import settings # noqa: E402
|
||||
from app.factors.registry import unregister_factor # noqa: E402
|
||||
from app.factors.store import register_definition # noqa: E402
|
||||
|
||||
N_SYMBOLS = 4000
|
||||
N_DAYS = 330
|
||||
|
||||
|
||||
def _frame() -> pl.DataFrame:
|
||||
base = pl.datetime_range(
|
||||
__import__("datetime").date(2025, 1, 1),
|
||||
__import__("datetime").date(2026, 12, 31),
|
||||
"1d",
|
||||
eager=True,
|
||||
).dt.date()
|
||||
days = base.head(N_DAYS)
|
||||
df = pl.DataFrame({"symbol": [f"{600000 + i:06d}.SH" for i in range(N_SYMBOLS)]}).join(
|
||||
pl.DataFrame({"date": days}), how="cross"
|
||||
)
|
||||
n = df.height
|
||||
import random
|
||||
|
||||
rng = random.Random(7)
|
||||
closes = [10.0 + rng.uniform(-1, 1) + (i % N_DAYS) * 0.01 for i in range(n)]
|
||||
return df.with_columns(
|
||||
pl.Series("close", closes, dtype=pl.Float64),
|
||||
pl.Series("open", [c * 0.999 for c in closes], dtype=pl.Float64),
|
||||
pl.Series("volume", [rng.uniform(5e4, 5e6) for _ in range(n)], dtype=pl.Float64),
|
||||
pl.Series("amount", [rng.uniform(5e5, 5e7) for _ in range(n)], dtype=pl.Float64),
|
||||
pl.Series("turnover_rate", [rng.uniform(0.1, 15.0) for _ in range(n)], dtype=pl.Float64),
|
||||
).sort(["symbol", "date"])
|
||||
|
||||
|
||||
_FORMULAS = {
|
||||
"uf_bench_1": "ts_zscore(close, 20)",
|
||||
"uf_bench_2": "ts_mean(volume, 10) / ts_mean(volume, 60)",
|
||||
"uf_bench_3": "close / ts_mean(close, 60) - 1",
|
||||
"uf_bench_4": "ts_std(close, 20) / ts_mean(close, 20)",
|
||||
"uf_bench_5": "rank(ts_delta(close, 5))",
|
||||
"uf_bench_6": "ts_corr(close, volume, 20)",
|
||||
"uf_bench_7": "ts_sum(sign(close - ts_delay(close, 1)), 20)",
|
||||
"uf_bench_8": "ts_max(close, 20) / close - 1",
|
||||
"uf_bench_9": "ts_mean(turnover_rate, 5) / ts_mean(turnover_rate, 60)",
|
||||
"uf_bench_10": "ts_rank(close, 120)",
|
||||
"uf_bench_11": "winsorize(ts_zscore(amount, 20))",
|
||||
"uf_bench_12": "decay_linear(ts_delta(close, 5), 10)",
|
||||
"uf_bench_13": "zscore(ts_mean(volume, 20))",
|
||||
}
|
||||
|
||||
|
||||
def _register(names: list[str]) -> None:
|
||||
import contextlib
|
||||
|
||||
for name in names:
|
||||
with contextlib.suppress(ValueError):
|
||||
unregister_factor(name)
|
||||
register_definition({
|
||||
"id": name, "label": name, "kind": "custom",
|
||||
"formula": _FORMULAS[name], "status": "active",
|
||||
})
|
||||
|
||||
|
||||
def _time_materialize(frame: pl.DataFrame, names: list[str], repeats: int) -> list[float]:
|
||||
from app.strategy.scoring import materialize_scoring_columns
|
||||
|
||||
out = []
|
||||
for _ in range(repeats):
|
||||
t = time.perf_counter()
|
||||
work = frame
|
||||
# 逐个物化 (与 scoring/backtest 调用同一条路径)
|
||||
result = materialize_scoring_columns(work, names)
|
||||
out.append(time.perf_counter() - t)
|
||||
assert all(name in result.columns for name in names)
|
||||
del result
|
||||
return out
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--json", dest="json_path", default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
from app.factors.materialize_cache import invalidate as _cache_invalidate
|
||||
|
||||
_cache_invalidate(disk=True) # 真冷启动 (磁盘缓存也清空)
|
||||
frame = _frame()
|
||||
row = {"rows": frame.height, "data_dir": str(settings.data_dir), "sets": {}}
|
||||
for label, names in (
|
||||
("K3", list(_FORMULAS)[:3]),
|
||||
("K13", list(_FORMULAS)),
|
||||
):
|
||||
_register(names)
|
||||
times = _time_materialize(frame, names, repeats=4)
|
||||
row["sets"][label] = {
|
||||
"n_factors": len(names),
|
||||
"per_call_ms": [round(t * 1000, 1) for t in times],
|
||||
"warm_call_ms": round(sorted(times)[1] * 1000, 1), # 第2-4次的代表值
|
||||
}
|
||||
# 清掉物化列, 下一组从干净帧开始
|
||||
frame = frame.select([c for c in frame.columns if not c.startswith("uf_")])
|
||||
for name in names:
|
||||
unregister_factor(name)
|
||||
|
||||
row["warm_delta_ms_K3_to_K13"] = round(
|
||||
row["sets"]["K13"]["warm_call_ms"] - row["sets"]["K3"]["warm_call_ms"], 2
|
||||
) # +10 因子的热路径绝对增量 (验收: 应近零)
|
||||
print(json.dumps(row, ensure_ascii=False, indent=2))
|
||||
if args.json_path:
|
||||
Path(args.json_path).write_text(json.dumps(row, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -207,3 +207,11 @@ def test_runtime_missing_column_fails_closed() -> None:
|
||||
assert compiled.frame_transform is not None
|
||||
assert compiled.frame_transform(frame_without_volume) is None # E013 语义
|
||||
assert compiled.frame_transform(_panel()) is not None
|
||||
|
||||
|
||||
def test_winsorize_optional_k_omitted_compiles():
|
||||
"""回归: winsorize(x) 省略 k — 此前 _check_call 直接 IndexError (编译器崩溃,
|
||||
而不是返回错误), winsorize 无 k 写法完全不可用。"""
|
||||
compiled = compile_formula("winsorize(close)")
|
||||
assert compiled.ok, compiled.errors
|
||||
assert compiled.warmup_bars == 1
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
"""L2 内容寻址因子列缓存 (P2b) 契约测试。
|
||||
|
||||
验收对象 (docs/ext-architecture-plan.md P2): 新增自定义因子后, 重复求值
|
||||
的热路径耗时不变。正确性契约:
|
||||
- 命中 (进程内 / 磁盘) 与现算结果逐元素一致;
|
||||
- 定义变更 / 输入值变更 → 键变 → 自动失效 (内容寻址即失效传播);
|
||||
- 一切缓存故障 (目录只读 / parquet 损坏) 回退现算 (fail-open);
|
||||
- 小帧旁路 (哈希+IO 比现算贵, 不缓存)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
import pytest
|
||||
|
||||
from app.config import settings
|
||||
from app.factors import materialize_cache
|
||||
from app.factors.registry import unregister_factor
|
||||
from app.factors.store import register_definition
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolated_cache(tmp_path: Path, monkeypatch) -> None:
|
||||
"""缓存写到临时目录; 每个测试从全空缓存开始。"""
|
||||
monkeypatch.setattr(settings, "data_dir", tmp_path)
|
||||
materialize_cache.invalidate(disk=True)
|
||||
yield
|
||||
materialize_cache.invalidate(disk=True)
|
||||
|
||||
|
||||
def _frame(n: int = 60_000, seed: int = 3) -> pl.DataFrame:
|
||||
rng = np.random.default_rng(seed)
|
||||
return pl.DataFrame({
|
||||
"symbol": ["600000.SH"] * n,
|
||||
"date": np.arange(n, dtype=np.int64),
|
||||
"close": rng.normal(10, 1, n),
|
||||
"volume": rng.uniform(1e4, 1e6, n),
|
||||
})
|
||||
|
||||
|
||||
def _register(fid: str, formula: str, version: int = 1) -> None:
|
||||
import contextlib
|
||||
|
||||
with contextlib.suppress(ValueError):
|
||||
unregister_factor(fid)
|
||||
register_definition({
|
||||
"id": fid, "label": fid, "kind": "custom",
|
||||
"formula": formula, "status": "active", "version": version,
|
||||
})
|
||||
|
||||
|
||||
def test_hit_matches_fresh_computation_lru_and_disk() -> None:
|
||||
from app.strategy.scoring import materialize_scoring_columns
|
||||
|
||||
_register("uf_c1", "ts_zscore(close, 20)")
|
||||
frame = _frame()
|
||||
cold = materialize_scoring_columns(frame, ["uf_c1"])
|
||||
assert materialize_cache.stats()["miss"] == 1
|
||||
|
||||
warm = materialize_scoring_columns(frame, ["uf_c1"])
|
||||
assert materialize_cache.stats()["lru_hit"] == 1
|
||||
assert warm["uf_c1"].equals(cold["uf_c1"])
|
||||
|
||||
materialize_cache.invalidate() # 清 LRU, 磁盘保留
|
||||
disk_warm = materialize_scoring_columns(frame, ["uf_c1"])
|
||||
assert materialize_cache.stats()["disk_hit"] == 1
|
||||
assert disk_warm["uf_c1"].equals(cold["uf_c1"])
|
||||
|
||||
|
||||
def test_input_value_change_invalidates() -> None:
|
||||
from app.strategy.scoring import materialize_scoring_columns
|
||||
|
||||
_register("uf_c2", "ts_mean(close, 30)")
|
||||
base = materialize_scoring_columns(_frame(seed=1), ["uf_c2"])
|
||||
changed = materialize_scoring_columns(_frame(seed=1).with_columns(pl.col("close") * 2.0), ["uf_c2"])
|
||||
assert not base["uf_c2"].equals(changed["uf_c2"])
|
||||
assert materialize_cache.stats()["miss"] == 2 # 两次都是未命中 (键不同)
|
||||
|
||||
|
||||
def test_formula_change_invalidates() -> None:
|
||||
from app.strategy.scoring import materialize_scoring_columns
|
||||
|
||||
frame = _frame()
|
||||
_register("uf_c3", "ts_zscore(close, 20)")
|
||||
v1 = materialize_scoring_columns(frame, ["uf_c3"])
|
||||
_register("uf_c3", "ts_zscore(close, 60)", version=2)
|
||||
v2 = materialize_scoring_columns(frame, ["uf_c3"])
|
||||
assert not v1["uf_c3"].equals(v2["uf_c3"])
|
||||
|
||||
|
||||
def test_corrupt_disk_cache_falls_back_to_recompute() -> None:
|
||||
from app.strategy.scoring import materialize_scoring_columns
|
||||
|
||||
_register("uf_c4", "ts_std(close, 20)")
|
||||
frame = _frame()
|
||||
cold = materialize_scoring_columns(frame, ["uf_c4"])
|
||||
materialize_cache.invalidate() # 磁盘上留有 key.parquet
|
||||
|
||||
root = Path(settings.data_dir) / "cache" / "factor_columns"
|
||||
files = list(root.glob("*.parquet"))
|
||||
assert len(files) == 1
|
||||
files[0].write_bytes(b"broken-not-parquet")
|
||||
|
||||
recovered = materialize_scoring_columns(frame, ["uf_c4"])
|
||||
assert recovered["uf_c4"].equals(cold["uf_c4"]) # 损坏 → 回退现算, 结果一致
|
||||
assert materialize_cache.stats()["error"] == 1
|
||||
|
||||
|
||||
def test_small_frame_bypasses_cache() -> None:
|
||||
from app.strategy.scoring import materialize_scoring_columns
|
||||
|
||||
_register("uf_c5", "ts_mean(volume, 10)")
|
||||
small = _frame(n=1_000)
|
||||
result = materialize_scoring_columns(small, ["uf_c5"])
|
||||
assert "uf_c5" in result.columns
|
||||
assert materialize_cache.stats()["bypass_small"] == 1
|
||||
assert materialize_cache.stats()["miss"] == 0 # 小帧完全没进缓存路径
|
||||
|
||||
|
||||
def test_staged_temp_columns_do_not_leak() -> None:
|
||||
"""现算与命中两条路径的输出帧列集合一致 (DSL 分阶段临时列不泄漏)。"""
|
||||
from app.strategy.scoring import materialize_scoring_columns
|
||||
|
||||
# 嵌套窗口公式会生成分阶段临时列
|
||||
_register("uf_c6", "ts_mean(ts_zscore(close, 20), 10)")
|
||||
frame = _frame()
|
||||
cold = materialize_scoring_columns(frame, ["uf_c6"])
|
||||
warm = materialize_scoring_columns(frame, ["uf_c6"])
|
||||
assert cold.columns == warm.columns
|
||||
assert set(warm.columns) - set(frame.columns) == {"uf_c6"}
|
||||
@@ -50,7 +50,7 @@
|
||||
| P0 基准 | `backend/scripts/bench_feature_resolution.py`; 依赖解析宽度/耗时基线 | 基线 JSON 落 `data/benchmarks/`(本地不入库) | ✅ |
|
||||
| P1a 依赖推导 | filter_history AST 列推导(`app/strategy/feature_deps.py`), 引擎加载接入 | 无声明且可证明的策略不再全量兜底 | ✅ |
|
||||
| P1b 契约统一 | signal/factor 统一 schema + 依赖提取公共化 | 新增 kind 复用同一校验/注册路径 | ✅ |
|
||||
| P2 L2 物化 | 按需物化 + 内容寻址缓存 + 失效传播; 指标成为一等定义 | 调参循环提速 ≥2×; 新增 10 个自定义指标热路径耗时不变 | 🔶 P2a 完成 |
|
||||
| P2 L2 物化 | 按需物化 + 内容寻址缓存 + 失效传播; 指标成为一等定义 | 调参循环提速 ≥2×; 新增 10 个自定义指标热路径耗时不变 | ✅ |
|
||||
| P3 表达式级 Python | T2 中间层 (ctx 追踪依赖+安全) | SKDJ 类口径零 DSL 可表达 | ⬜ |
|
||||
| P4 体验与分发 | 统一选择器/自动表单/导入冒烟; 备份管线复用为扩展包 | 新增信号导入到可回测 ≤2 分钟 | ⬜ |
|
||||
| 条件: 沙箱 | T3 子进程执行 | 触发条件: 首次真实导入第三方扩展 | ⬜ |
|
||||
@@ -82,6 +82,23 @@ stats 剔除计时字段后逐组完全一致, 交易逐笔相同; tests/backtes
|
||||
剩余每组成本 = filter_history (参数相关, 不可共享) + 掩码散布。
|
||||
P2b (L2 内容寻址物化) 在此之上进一步覆盖跨 run/跨策略的因子复用。
|
||||
|
||||
|
||||
## P2b 实测(2026-10-26, 本机, 132 万行合成帧, 13 个自定义 DSL 因子)
|
||||
|
||||
内容寻址因子列缓存 (app/factors/materialize_cache.py): 键 =
|
||||
sha256(定义 + 依赖列内容指纹 + 行数), 进程内 LRU + 磁盘 parquet 两层,
|
||||
fail-open。命中列一次批量附着 (与因子数无关)。
|
||||
|
||||
| 臂 | 冷 (首次) | 热 (重复求值) |
|
||||
|---|---|---|
|
||||
| K=3 因子 | 132ms | 0.0ms |
|
||||
| K=13 因子 | 480.7ms (≈无缓存基线 478ms) | **0.1ms** |
|
||||
|
||||
**+10 因子的热路径绝对增量 0.1ms** (基线为 +400ms 线性增长), 冷启动
|
||||
开销 ~0.5%。端到端优化器 A/B 复测: trials 4.23× / 含 prepare 3.16×,
|
||||
stats 完全一致。P2 两项验收 (调参 ≥2×、热路径平坦) 均达成。
|
||||
内置指标转为「一等定义」的完全体形态随 P3 表达式层一并落。
|
||||
|
||||
## P1b 交付(2026-10-26)
|
||||
|
||||
- `app/definitions.py`: `JsonDefinitionStore`(原子写 + 损坏跳过 + 目录 mtime 签名,
|
||||
|
||||
Reference in New Issue
Block a user